[Release-1.28] - Enable kube-router metrics

[brandond]

Backport fix for Enable kube-router metrics

• Enable kube-router metrics #9031

[VestigeJ]

##Environment Details
Reproduced using VERSION=v1.28.5+k3s1
Validated using COMMIT=6224ea62af78b44db3f2f881e55590c61463dbcd

Infrastructure

• Cloud

Node(s) CPU architecture, OS, and version:

Linux 5.14.21-150500.53-default x86_64 GNU/Linux
PRETTY_NAME="SUSE Linux Enterprise Server 15 SP5"

Cluster Configuration:

NAME                    STATUS   ROLES                       AGE    VERSION
node/ip-17-13-23-12     Ready    control-plane,etcd,master   3m5s   v1.28.5+k3s1

NAMESPACE     NAME                                          READY   STATUS      RESTARTS   AGE
kube-system   pod/coredns-6799fbcd5-klkhj                   1/1     Running     0          2m50s
kube-system   pod/helm-install-traefik-9l7n6                0/1     Completed   1          2m51s
kube-system   pod/helm-install-traefik-crd-266db            0/1     Completed   0          2m51s
kube-system   pod/local-path-provisioner-84db5d44d9-l8zmt   1/1     Running     0          2m50s
kube-system   pod/metrics-server-67c658944b-svztw           1/1     Running     0          2m50s
kube-system   pod/svclb-traefik-21d07f83-mkvzf              2/2     Running     0          2m36s
kube-system   pod/traefik-f4564c4f4-5w9cf                   1/1     Running     0          2m36s

Config.yaml:

write-kubeconfig-mode: 644
debug: true
cluster-init: true
token: YOUR_TOKEN_HERE

Reproduction && Validation

$ curl https://get.k3s.io --output install-"k3s".sh
$ sudo chmod +x install-"k3s".sh
$ sudo groupadd --system etcd && sudo useradd -s /sbin/nologin --system -g etcd etcd
$ sudo modprobe ip_vs_rr
$ sudo modprobe ip_vs_wrr
$ sudo modprobe ip_vs_sh
$ sudo printf "on_oovm.panic_on_oom=0 \nvm.overcommit_memory=1 \nkernel.panic=10 \nkernel.panic_ps=1 \nkernel.panic_on_oops=1 \n" > ~/90-kubelet.conf
$ sudo cp 90-kubelet.conf /etc/sysctl.d/
$ sudo systemctl restart systemd-sysctl
$ sudo INSTALL_K3S_VERSION=v1.28.5+k3s1 INSTALL_K3S_EXEC=server ./install-k3s.sh
$ kg no,po,svc -A //kubectl get nodes,pods,services -A
$ kg --raw /api/v1/nodes/ip-17-13-23-12/proxy/metrics //check for metrics in general before looking for new exposed metrics specifically
$ kg --raw /api/v1/nodes/ip-17-13-23-12/proxy/metrics | grep -F kube_router_ //nil output is expected on current releases
// upgrade the node watch for the pods and services to be re-deployed with the changes before querying the api again for the new metrics
$ sudo INSTALL_K3S_COMMIT=6224ea62af78b44db3f2f881e55590c61463dbcd INSTALL_K3S_EXEC=server ./install-k3s.sh
$ kg --raw /api/v1/nodes/ip-17-13-23-12/proxy/metrics | grep -F kube_router_

Results:

Existing releases have no output when grepping for this metric on the api endpoint.

New metrics output content is quite large

$ kg --raw /api/v1/nodes/ip-17-13-23-12/proxy/metrics | grep -F kube_router_

# HELP kube_router_build_info Expose version and other build information
# TYPE kube_router_build_info gauge
kube_router_build_info{goversion="go1.20.12",version="v2.0.0-20230925161250-364f994b140b"} 1
# HELP kube_router_controller_iptables_sync_time Time it took for controller to sync iptables
# TYPE kube_router_controller_iptables_sync_time histogram
kube_router_controller_iptables_sync_time_bucket{le="0.005"} 0
kube_router_controller_iptables_sync_time_bucket{le="0.01"} 0
kube_router_controller_iptables_sync_time_bucket{le="0.025"} 0
kube_router_controller_iptables_sync_time_bucket{le="0.05"} 1
kube_router_controller_iptables_sync_time_bucket{le="0.1"} 4
kube_router_controller_iptables_sync_time_bucket{le="0.25"} 8
kube_router_controller_iptables_sync_time_bucket{le="0.5"} 8
kube_router_controller_iptables_sync_time_bucket{le="1"} 8
kube_router_controller_iptables_sync_time_bucket{le="2.5"} 8
kube_router_controller_iptables_sync_time_bucket{le="5"} 8
kube_router_controller_iptables_sync_time_bucket{le="10"} 8
kube_router_controller_iptables_sync_time_bucket{le="+Inf"} 8
kube_router_controller_iptables_sync_time_sum 0.8283274270000001
kube_router_controller_iptables_sync_time_count 8
# HELP kube_router_controller_iptables_v4_restore_time Time it took for controller to restore IPv4 rules
# TYPE kube_router_controller_iptables_v4_restore_time histogram
kube_router_controller_iptables_v4_restore_time_bucket{le="0.005"} 3
kube_router_controller_iptables_v4_restore_time_bucket{le="0.01"} 5
kube_router_controller_iptables_v4_restore_time_bucket{le="0.025"} 6
kube_router_controller_iptables_v4_restore_time_bucket{le="0.05"} 7
kube_router_controller_iptables_v4_restore_time_bucket{le="0.1"} 8
kube_router_controller_iptables_v4_restore_time_bucket{le="0.25"} 8
kube_router_controller_iptables_v4_restore_time_bucket{le="0.5"} 8
kube_router_controller_iptables_v4_restore_time_bucket{le="1"} 8
kube_router_controller_iptables_v4_restore_time_bucket{le="2.5"} 8
kube_router_controller_iptables_v4_restore_time_bucket{le="5"} 8
kube_router_controller_iptables_v4_restore_time_bucket{le="10"} 8
kube_router_controller_iptables_v4_restore_time_bucket{le="+Inf"} 8
kube_router_controller_iptables_v4_restore_time_sum 0.13941654
kube_router_controller_iptables_v4_restore_time_count 8
# HELP kube_router_controller_iptables_v4_save_time Time it took controller to save IPv4 rules
# TYPE kube_router_controller_iptables_v4_save_time histogram
kube_router_controller_iptables_v4_save_time_bucket{le="0.005"} 7
kube_router_controller_iptables_v4_save_time_bucket{le="0.01"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="0.025"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="0.05"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="0.1"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="0.25"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="0.5"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="1"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="2.5"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="5"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="10"} 8
kube_router_controller_iptables_v4_save_time_bucket{le="+Inf"} 8
kube_router_controller_iptables_v4_save_time_sum 0.027213059999999997
kube_router_controller_iptables_v4_save_time_count 8
# HELP kube_router_controller_iptables_v6_restore_time Time it took for controller to restore IPv6 rules
# TYPE kube_router_controller_iptables_v6_restore_time histogram
kube_router_controller_iptables_v6_restore_time_bucket{le="0.005"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="0.01"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="0.025"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="0.05"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="0.1"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="0.25"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="0.5"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="1"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="2.5"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="5"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="10"} 0
kube_router_controller_iptables_v6_restore_time_bucket{le="+Inf"} 0
kube_router_controller_iptables_v6_restore_time_sum 0
kube_router_controller_iptables_v6_restore_time_count 0
# HELP kube_router_controller_iptables_v6_save_time Time to took for controller to save IPv6 rules
# TYPE kube_router_controller_iptables_v6_save_time histogram
kube_router_controller_iptables_v6_save_time_bucket{le="0.005"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="0.01"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="0.025"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="0.05"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="0.1"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="0.25"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="0.5"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="1"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="2.5"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="5"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="10"} 0
kube_router_controller_iptables_v6_save_time_bucket{le="+Inf"} 0
kube_router_controller_iptables_v6_save_time_sum 0
kube_router_controller_iptables_v6_save_time_count 0
# HELP kube_router_controller_policy_chains Active policy chains
# TYPE kube_router_controller_policy_chains gauge
kube_router_controller_policy_chains 0
# HELP kube_router_controller_policy_chains_sync_time Time it took for controller to sync policy chains
# TYPE kube_router_controller_policy_chains_sync_time histogram
kube_router_controller_policy_chains_sync_time_bucket{le="0.005"} 5
kube_router_controller_policy_chains_sync_time_bucket{le="0.01"} 5
kube_router_controller_policy_chains_sync_time_bucket{le="0.025"} 7
kube_router_controller_policy_chains_sync_time_bucket{le="0.05"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="0.1"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="0.25"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="0.5"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="1"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="2.5"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="5"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="10"} 8
kube_router_controller_policy_chains_sync_time_bucket{le="+Inf"} 8
kube_router_controller_policy_chains_sync_time_sum 0.09091909000000001
kube_router_controller_policy_chains_sync_time_count 8
# HELP kube_router_controller_policy_ipset_v4_restore_time Time it took for controller to restore IPv4 ipsets
# TYPE kube_router_controller_policy_ipset_v4_restore_time histogram
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="0.005"} 5
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="0.01"} 5
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="0.025"} 7
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="0.05"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="0.1"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="0.25"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="0.5"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="1"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="2.5"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="5"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="10"} 8
kube_router_controller_policy_ipset_v4_restore_time_bucket{le="+Inf"} 8
kube_router_controller_policy_ipset_v4_restore_time_sum 0.09086489799999999
kube_router_controller_policy_ipset_v4_restore_time_count 8
# HELP kube_router_controller_policy_ipset_v6_restore_time Time it took for controller to restore IPv6 ipsets
# TYPE kube_router_controller_policy_ipset_v6_restore_time histogram
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="0.005"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="0.01"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="0.025"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="0.05"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="0.1"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="0.25"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="0.5"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="1"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="2.5"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="5"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="10"} 0
kube_router_controller_policy_ipset_v6_restore_time_bucket{le="+Inf"} 0
kube_router_controller_policy_ipset_v6_restore_time_sum 0
kube_router_controller_policy_ipset_v6_restore_time_count 0
# HELP kube_router_controller_policy_ipsets Active policy ipsets
# TYPE kube_router_controller_policy_ipsets gauge
kube_router_controller_policy_ipsets 0