From 99e71209e75ae84a6b47c72764141aa888764e45 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 27 Sep 2024 13:15:51 +0200 Subject: [PATCH] Add metrics necessary for SELinuxMount feature This is a proof of concept for https://github.com/kubernetes/enhancements/pull/4843 * Report pod spec.securityContext as kube_pod_security_context. The only values I need are seLinuxOptions. * Report container securityContext as kube_pod_container_security_context. I need privileged flag + seLinuxOptions. * Report volumes used by *containers* as kube_pod_container_volume_mount. The existing kube_pod_spec_volumes_persistentvolumeclaims_info tracks volumes used in a *Pod*, while I need volumes used by *containers*. * Report CSIDriver object fields as kube_csidriver_info. I need just seLinuxMount flag. --- internal/store/builder.go | 5 ++ internal/store/csidriver.go | 84 +++++++++++++++++++++++++++++ internal/store/pod.go | 103 ++++++++++++++++++++++++++++++++++++ pkg/options/resource.go | 1 + 4 files changed, 193 insertions(+) create mode 100644 internal/store/csidriver.go diff --git a/internal/store/builder.go b/internal/store/builder.go index f7941794e4..d1d01a5993 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -316,6 +316,7 @@ var availableStores = map[string]func(f *Builder) []cache.Store{ "configmaps": func(b *Builder) []cache.Store { return b.buildConfigMapStores() }, "clusterrolebindings": func(b *Builder) []cache.Store { return b.buildClusterRoleBindingStores() }, "cronjobs": func(b *Builder) []cache.Store { return b.buildCronJobStores() }, + "csidrivers": func(b *Builder) []cache.Store { return b.buildCSIDriverStores() }, "daemonsets": func(b *Builder) []cache.Store { return b.buildDaemonSetStores() }, "deployments": func(b *Builder) []cache.Store { return b.buildDeploymentStores() }, "endpoints": func(b *Builder) []cache.Store { return b.buildEndpointsStores() }, @@ -461,6 +462,10 @@ func (b *Builder) buildStorageClassStores() []cache.Store { return b.buildStoresFunc(storageClassMetricFamilies(b.allowAnnotationsList["storageclasses"], b.allowLabelsList["storageclasses"]), &storagev1.StorageClass{}, createStorageClassListWatch, b.useAPIServerCache) } +func (b *Builder) buildCSIDriverStores() []cache.Store { + return b.buildStoresFunc(csiDriverMetricFamilies(b.allowAnnotationsList["csidrivers"], b.allowLabelsList["csidrivers"]), &storagev1.CSIDriver{}, createCSIDriverListWatch, b.useAPIServerCache) +} + func (b *Builder) buildPodStores() []cache.Store { return b.buildStoresFunc(podMetricFamilies(b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache) } diff --git a/internal/store/csidriver.go b/internal/store/csidriver.go new file mode 100644 index 0000000000..a4e31eff03 --- /dev/null +++ b/internal/store/csidriver.go @@ -0,0 +1,84 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + "context" + "strconv" + + basemetrics "k8s.io/component-base/metrics" + + "k8s.io/kube-state-metrics/v2/pkg/metric" + generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" + + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +var ( + defaultSELinuxMount = false + descCSIDriverLabelsDefaultLabels = []string{"csi_driver"} +) + +func csiDriverMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator { + return []generator.FamilyGenerator{ + *generator.NewFamilyGeneratorWithStability( + "kube_csidriver_info", + "Information about CSI drivers.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapCSIDriverFunc(func(c *storagev1.CSIDriver) *metric.Family { + if c.Spec.SELinuxMount == nil { + c.Spec.SELinuxMount = &defaultSELinuxMount + } + m := metric.Metric{ + LabelKeys: []string{"selinux_mount"}, + LabelValues: []string{strconv.FormatBool(*c.Spec.SELinuxMount)}, + Value: 1, + } + return &metric.Family{Metrics: []*metric.Metric{&m}} + }), + ), + } +} + +func wrapCSIDriverFunc(f func(*storagev1.CSIDriver) *metric.Family) func(interface{}) *metric.Family { + return func(obj interface{}) *metric.Family { + csiDriver := obj.(*storagev1.CSIDriver) + + metricFamily := f(csiDriver) + + for _, m := range metricFamily.Metrics { + m.LabelKeys, m.LabelValues = mergeKeyValues(descCSIDriverLabelsDefaultLabels, []string{csiDriver.Name}, m.LabelKeys, m.LabelValues) + } + + return metricFamily + } +} + +func createCSIDriverListWatch(kubeClient clientset.Interface, _ string, _ string) cache.ListerWatcher { + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + return kubeClient.StorageV1().CSIDrivers().List(context.TODO(), opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + return kubeClient.StorageV1().CSIDrivers().Watch(context.TODO(), opts) + }, + } +} diff --git a/internal/store/pod.go b/internal/store/pod.go index 8c39017283..6e24fd6933 100644 --- a/internal/store/pod.go +++ b/internal/store/pod.go @@ -57,6 +57,8 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodContainerStatusTerminatedReasonFamilyGenerator(), createPodContainerStatusWaitingFamilyGenerator(), createPodContainerStatusWaitingReasonFamilyGenerator(), + createPodContainerSecurityContextFamilyGenerator(), + createPodContainerVolumeMountFamilyGenerator(), createPodCreatedFamilyGenerator(), createPodDeletionTimestampFamilyGenerator(), createPodInfoFamilyGenerator(), @@ -96,6 +98,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat createPodNodeSelectorsFamilyGenerator(), createPodServiceAccountFamilyGenerator(), createPodSchedulerNameFamilyGenerator(), + createPodSecurityContextFamilyGenerator(), } } @@ -164,6 +167,49 @@ func createPodContainerInfoFamilyGenerator() generator.FamilyGenerator { ) } +func createPodContainerSecurityContextFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_security_context", + "Information about security context of a container in a pod.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + labelKeys := []string{"container", "privileged", "container_selinux_user", "container_selinux_role", "container_selinux_type", "container_selinux_level"} + + for _, c := range p.Spec.Containers { + sc := c.SecurityContext + if sc == nil { + // No security context set, report defaults. + ms = append(ms, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: []string{c.Name, "false", "", "", "", ""}, + Value: 1, + }) + continue + } + seLinuxOpts := &v1.SELinuxOptions{} + if sc.SELinuxOptions != nil { + seLinuxOpts = sc.SELinuxOptions + } + privileged := "false" + if sc.Privileged != nil && *sc.Privileged { + privileged = "true" + } + ms = append(ms, &metric.Metric{ + LabelKeys: labelKeys, + LabelValues: []string{c.Name, privileged, seLinuxOpts.User, seLinuxOpts.Role, seLinuxOpts.Type, seLinuxOpts.Level}, + Value: 1, + }) + } + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodContainerResourceLimitsFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_container_resource_limits", @@ -1309,6 +1355,32 @@ func createPodSpecVolumesPersistentVolumeClaimsReadonlyFamilyGenerator() generat ) } +func createPodContainerVolumeMountFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_container_volume_mount", + "Information about volumes mounted into containers in a pod.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + ms := []*metric.Metric{} + for _, c := range p.Spec.Containers { + for _, mount := range c.VolumeMounts { + ms = append(ms, &metric.Metric{ + LabelKeys: []string{"container", "volume"}, + LabelValues: []string{c.Name, mount.Name}, + Value: 1, + }) + } + } + + return &metric.Family{ + Metrics: ms, + } + }), + ) +} + func createPodStartTimeFamilyGenerator() generator.FamilyGenerator { return *generator.NewFamilyGeneratorWithStability( "kube_pod_start_time", @@ -1762,6 +1834,37 @@ func createPodSchedulerNameFamilyGenerator() generator.FamilyGenerator { ) } +func createPodSecurityContextFamilyGenerator() generator.FamilyGenerator { + return *generator.NewFamilyGeneratorWithStability( + "kube_pod_security_context", + "The security context of a pod.", + metric.Gauge, + basemetrics.ALPHA, + "", + wrapPodFunc(func(p *v1.Pod) *metric.Family { + m := metric.Metric{ + // TODO: Add SELinuxChangePolicy from https://github.com/kubernetes/enhancements/pull/4843 + LabelKeys: []string{"pod_selinux_user", "pod_selinux_role", "pod_selinux_type", "pod_selinux_level"}, + LabelValues: []string{"", "", "", ""}, + Value: 1, + } + + if p.Spec.SecurityContext == nil { + // No security context specified, use defaults + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + } + if opts := p.Spec.SecurityContext.SELinuxOptions; opts != nil { + m.LabelValues = []string{opts.User, opts.Role, opts.Type, opts.Level} + } + return &metric.Family{ + Metrics: []*metric.Metric{&m}, + } + }), + ) +} + func wrapPodFunc(f func(*v1.Pod) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { pod := obj.(*v1.Pod) diff --git a/pkg/options/resource.go b/pkg/options/resource.go index 09e0e0400c..e2ffbe385d 100644 --- a/pkg/options/resource.go +++ b/pkg/options/resource.go @@ -29,6 +29,7 @@ var ( "certificatesigningrequests": struct{}{}, "configmaps": struct{}{}, "cronjobs": struct{}{}, + "csidrivers": struct{}{}, "daemonsets": struct{}{}, "deployments": struct{}{}, "endpoints": struct{}{},