topology-aware: implement metrics collection.

Implement collection of per zone prometheus metrics. Currently we collect for each pool/zone the following - name, cpuset and memset - shared pool capacity, allocation, available amount - memory capacity, allocation, available amount - number of containers - number of containers in the shared pool Signed-off-by: Krisztian Litkey <[email protected]>
containers · Nov 13, 2024 · 4ea6ec6 · 4ea6ec6
1 parent 5cde4c1
commit 4ea6ec6
Show file tree

Hide file tree

Showing 3 changed files with 341 additions and 19 deletions.
diff --git a/cmd/plugins/topology-aware/policy/metrics.go b/cmd/plugins/topology-aware/policy/metrics.go
@@ -0,0 +1,311 @@
+// Copyright The NRI Plugins Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package topologyaware
+
+import (
+	"slices"
+	"strings"
+
+	libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory"
+	policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy"
+	"github.com/containers/nri-plugins/pkg/utils/cpuset"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type TopologyAwareMetrics struct {
+	p         *policy
+	ZoneNames []string
+	Zones     map[string]*Zone
+	Metrics   Metrics
+}
+
+type Zone struct {
+	Name                 string
+	Cpus                 cpuset.CPUSet
+	Mems                 libmem.NodeMask
+	SharedPool           cpuset.CPUSet
+	SharedAssigned       int
+	SharedAvailable      int
+	MemCapacity          int64
+	MemAssigned          int64
+	MemAvailable         int64
+	ContainerCount       int
+	SharedContainerCount int
+}
+
+type Metrics struct {
+	zone                 *prometheus.GaugeVec
+	cpuSharedCapacity    *prometheus.GaugeVec
+	cpuSharedAssigned    *prometheus.GaugeVec
+	cpuSharedAvailable   *prometheus.GaugeVec
+	memCapacity          *prometheus.GaugeVec
+	memAssigned          *prometheus.GaugeVec
+	memAvailable         *prometheus.GaugeVec
+	containerCount       *prometheus.GaugeVec
+	sharedContainerCount *prometheus.GaugeVec
+}
+
+func (p *policy) GetMetrics() policyapi.Metrics {
+	return p.metrics
+}
+
+func (p *policy) NewTopologyAwareMetrics() *TopologyAwareMetrics {
+	m := &TopologyAwareMetrics{
+		p:     p,
+		Zones: make(map[string]*Zone),
+		Metrics: Metrics{
+			zone: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_policy_zone_cpu_capacity",
+					Help: "A topology zone of CPUs.",
+				},
+				[]string{
+					"zone",
+					"cpus",
+					"mems",
+				},
+			),
+			cpuSharedCapacity: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_policy_zone_cpu_shared_capacity",
+					Help: "Capacity of shared CPU pool of a topology zone.",
+				},
+				[]string{
+					"zone",
+					"cpus",
+				},
+			),
+			cpuSharedAssigned: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_policy_zone_cpu_shared_assigned",
+					Help: "Assigned amount of shared CPU pool of a topology zone.",
+				},
+				[]string{
+					"zone",
+					"cpus",
+				},
+			),
+			cpuSharedAvailable: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_policy_zone_cpu_shared_available",
+					Help: "Available amount of shared CPU pool of a topology zone.",
+				},
+				[]string{
+					"zone",
+					"cpus",
+				},
+			),
+			memCapacity: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_zone_mem_capacity",
+					Help: "Memory capacity of a topology zone.",
+				},
+				[]string{
+					"zone",
+					"mems",
+				},
+			),
+			memAssigned: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_zone_mem_assigned",
+					Help: "Amount of assigned memory of a topology zone.",
+				},
+				[]string{
+					"zone",
+					"mems",
+				},
+			),
+			memAvailable: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_zone_mem_available",
+					Help: "Amount of available memory of a topology zone.",
+				},
+				[]string{
+					"zone",
+					"mems",
+				},
+			),
+			containerCount: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_zone_container_count",
+					Help: "Number of containers assigned to a topology zone.",
+				},
+				[]string{
+					"zone",
+				},
+			),
+			sharedContainerCount: prometheus.NewGaugeVec(
+				prometheus.GaugeOpts{
+					Name: "topologyaware_zone_shared_container_count",
+					Help: "Number of containers in the shared CPU pool of a topology zone.",
+				},
+				[]string{
+					"zone",
+				},
+			),
+		},
+	}
+
+	for _, pool := range p.pools {
+		var (
+			name = pool.Name()
+			mems = libmem.NewNodeMask(pool.GetMemset(memoryAll).Members()...)
+			capa = pool.GetSupply().(*supply)
+			cpus = capa.ReservedCPUs().Union(capa.IsolatedCPUs()).Union(capa.SharableCPUs())
+			zone = &Zone{
+				Name:        name,
+				Cpus:        cpus,
+				Mems:        mems,
+				MemCapacity: p.memAllocator.ZoneCapacity(mems),
+			}
+		)
+
+		m.Zones[name] = zone
+		m.ZoneNames = append(m.ZoneNames, name)
+
+		m.Metrics.zone.WithLabelValues(
+			zone.Name,
+			zone.Cpus.String(),
+			zone.Mems.String(),
+		).Set(float64(zone.Cpus.Size()))
+
+		m.Metrics.memCapacity.WithLabelValues(
+			zone.Name,
+			zone.Mems.String(),
+		).Set(float64(zone.MemCapacity))
+	}
+
+	slices.SortFunc(m.ZoneNames, func(a, b string) int {
+		poolA, poolB := p.nodes[a], p.nodes[b]
+		if diff := poolA.RootDistance() - poolB.RootDistance(); diff != 0 {
+			return diff
+		}
+		return strings.Compare(a, b)
+	})
+
+	m.Update()
+
+	return m
+}
+
+func (m *TopologyAwareMetrics) Describe(ch chan<- *prometheus.Desc) {
+	if m == nil {
+		return
+	}
+
+	m.Metrics.zone.Describe(ch)
+	m.Metrics.cpuSharedCapacity.Describe(ch)
+	m.Metrics.cpuSharedAssigned.Describe(ch)
+	m.Metrics.cpuSharedAvailable.Describe(ch)
+	m.Metrics.memCapacity.Describe(ch)
+	m.Metrics.memAssigned.Describe(ch)
+	m.Metrics.memAvailable.Describe(ch)
+	m.Metrics.containerCount.Describe(ch)
+	m.Metrics.sharedContainerCount.Describe(ch)
+}
+
+func (m *TopologyAwareMetrics) Collect(ch chan<- prometheus.Metric) {
+	if m == nil {
+		return
+	}
+
+	m.Update()
+
+	m.Metrics.zone.Collect(ch)
+	m.Metrics.cpuSharedCapacity.Collect(ch)
+	m.Metrics.cpuSharedAssigned.Collect(ch)
+	m.Metrics.cpuSharedAvailable.Collect(ch)
+	m.Metrics.memCapacity.Collect(ch)
+	m.Metrics.memAssigned.Collect(ch)
+	m.Metrics.memAvailable.Collect(ch)
+	m.Metrics.containerCount.Collect(ch)
+	m.Metrics.sharedContainerCount.Collect(ch)
+}
+
+// Update updates our metrics.
+func (m *TopologyAwareMetrics) Update() {
+	if m == nil {
+		return
+	}
+
+	p := m.p
+	for _, pool := range p.pools {
+		log.Debug("updating metrics for pool %s...", pool.Name())
+
+		var (
+			zone       = m.Zones[pool.Name()]
+			free       = pool.FreeSupply().(*supply)
+			mems       = libmem.NewNodeMask(pool.GetMemset(memoryAll).Members()...)
+			sharedPool = free.SharableCPUs().Union(free.ReservedCPUs())
+			containers = 0
+			sharedctrs = 0
+		)
+
+		if zone == nil {
+			log.Error("metrics zone not found for pool %s", pool.Name())
+			continue
+		}
+
+		for _, g := range p.allocations.grants {
+			if g.GetCPUNode().Name() == pool.Name() {
+				containers++
+				if g.ReservedPortion() != 0 || g.CPUPortion() != 0 {
+					sharedctrs++
+				}
+			}
+		}
+
+		zone.SharedPool = sharedPool
+		zone.SharedAssigned = free.GrantedReserved() + free.GrantedShared()
+		zone.SharedAvailable = free.AllocatableSharedCPU()
+		zone.MemAssigned = p.memAllocator.ZoneUsage(mems)
+		zone.MemAvailable = p.memAllocator.ZoneAvailable(mems)
+		zone.ContainerCount = containers
+		zone.SharedContainerCount = sharedctrs
+
+		m.Metrics.cpuSharedCapacity.WithLabelValues(
+			zone.Name,
+			zone.SharedPool.String(),
+		).Set(float64(zone.SharedPool.Size()))
+
+		m.Metrics.cpuSharedAssigned.WithLabelValues(
+			zone.Name,
+			zone.SharedPool.String(),
+		).Set(float64(zone.SharedAssigned) / 1000.0)
+
+		m.Metrics.cpuSharedAvailable.WithLabelValues(
+			zone.Name,
+			zone.SharedPool.String(),
+		).Set(float64(zone.SharedAvailable) / 1000.0)
+
+		m.Metrics.memAssigned.WithLabelValues(
+			zone.Name,
+			zone.Mems.MemsetString(),
+		).Set(float64(zone.MemAssigned))
+
+		m.Metrics.memAvailable.WithLabelValues(
+			zone.Name,
+			zone.Mems.MemsetString(),
+		).Set(float64(zone.MemAvailable))
+
+		m.Metrics.containerCount.WithLabelValues(
+			zone.Name,
+		).Set(float64(zone.ContainerCount))
+
+		m.Metrics.sharedContainerCount.WithLabelValues(
+			zone.Name,
+		).Set(float64(zone.SharedContainerCount))
+	}
+}
diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go
@@ -19,7 +19,6 @@ import (
 	"fmt"
 
 	"github.com/containers/nri-plugins/pkg/utils/cpuset"
-	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/api/resource"
 
 	cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware"
@@ -422,24 +421,6 @@ func (p *policy) ExportResourceData(c cache.Container) map[string]string {
 	return data
 }
 
-func (p *policy) GetMetrics() policyapi.Metrics {
-	return p.metrics
-}
-
-func (p *policy) NewTopologyAwareMetrics() *TopologyAwareMetrics {
-	return &TopologyAwareMetrics{}
-}
-
-type TopologyAwareMetrics struct{}
-
-func (*TopologyAwareMetrics) Describe(ch chan<- *prometheus.Desc) {
-	return
-}
-
-func (*TopologyAwareMetrics) Collect(ch chan<- prometheus.Metric) {
-	return
-}
-
 // reallocateResources reallocates the given containers using the given pool hints
 func (p *policy) reallocateResources(containers []cache.Container, pools map[string]string) error {
 	errs := []error{}

diff --git a/docs/resource-policy/policy/topology-aware.md b/docs/resource-policy/policy/topology-aware.md
@@ -146,6 +146,16 @@ behavior. These options can be supplied as part of the effective
     CPU allocations. For a more detailed discussion of CPU prioritization see
     the [cpu allocator](../developers-guide/cpu-allocator.md) documentation.
 
+Additionally, the following sub-configuration is available for instrumentation:
+
+- `instrumentation`: configures runtime instrumentation.
+  - `httpEndpoint`: the address the HTTP server listens on. Example:
+    `:8891`.
+  - `prometheusExport`: if set to True, metrics about system and topology zone
+     resource assignment are readable through `/metrics` from the configured
+     `httpEndpoint`.
+  - `reportPeriod`: `/metrics` aggregation interval for polled metrics.
+
 ## Policy CPU Allocation Preferences
 
 There are a number of workload properties this policy actively checks to decide
@@ -760,3 +770,23 @@ metadata:
 
 <!-- Links -->
 [configuration]: ../configuration.md
+
+## Metrics and Debugging
+
+In order to enable more verbose logging and metrics exporting from the
+topology-aware policy, enable instrumentation and policy debugging from
+the nri-resource-policy global config:
+
+```yaml
+instrumentation:
+  # The topology-aware policy can exports various system and topology
+  # zone utilisation metrics. Accessible in command line with
+  # curl --silent http://$localhost_or_pod_IP:8891/metrics
+  HTTPEndpoint: :8891
+  PrometheusExport: true
+  metrics:
+    enabled: # use '*' instead for all available metrics
+    - policy
+logger:
+  Debug: policy
+```