diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 33ffd4690815..ea5c4e39503d 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -1,4 +1,4 @@ -ARG GOLANG=golang:1.19.9-alpine3.17 +ARG GOLANG=golang:1.19.9-alpine3.18 FROM ${GOLANG} ARG http_proxy=$http_proxy diff --git a/Dockerfile.manifest b/Dockerfile.manifest index 8a7d6f107b66..d9fc02f11958 100644 --- a/Dockerfile.manifest +++ b/Dockerfile.manifest @@ -1,4 +1,4 @@ -ARG GOLANG=golang:1.19.9-alpine3.17 +ARG GOLANG=golang:1.19.9-alpine3.18 FROM ${GOLANG} COPY --from=plugins/manifest:1.2.3 /bin/* /bin/ diff --git a/Dockerfile.test b/Dockerfile.test index 2e5ec2349884..7b57e90293f7 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -ARG GOLANG=golang:1.19.9-alpine3.17 +ARG GOLANG=golang:1.19.9-alpine3.18 FROM ${GOLANG} as test-base RUN apk -U --no-cache add bash jq diff --git a/docs/adrs/core-controller-user.md b/docs/adrs/core-controller-user.md new file mode 100644 index 000000000000..91a693c19421 --- /dev/null +++ b/docs/adrs/core-controller-user.md @@ -0,0 +1,25 @@ +# Use a dedicated user for K3s core controllers + +Date: 2023-05-26 + +## Status + +Accepted + +## Context + +Users who collect audit logs from K3s currently have a hard time determining if an action was performed by an administrator, or by the K3s supervisor. +This is due to the K3s supervisor using the same `system:admin` user for both the admin kubeconfig, and the kubeconfig used by core Wrangler controllers that drive core functionality and the deploy/helm controllers. + +Users may have policies in place that prohibit use of the `system:admin` account, or that require service accounts to be distinct from user accounts. + +## Decision + +* We will add a new kubeconfig for the K3s supervisor controllers: core functionality, deploy (AddOns; aka the manifests directory), and helm (HelmChart/HelmChartConfig). +* Each of the three controllers will use a dedicated user-agent to further assist in discriminating between events, via both audit logs and resource ManageFields tracking. +* The new user account will use exisiting core Kubernetes group RBAC. + +## Consequences + +* K3s servers will create and manage an additional kubeconfig, client cert, and key that is intended only for use by the supervisor controllers. +* K3s supervisor controllers will use distinct user-agents to further discriminate between which component initiated the request. diff --git a/go.mod b/go.mod index 9195d2d0d3c6..b88f6c576824 100644 --- a/go.mod +++ b/go.mod @@ -89,10 +89,12 @@ replace ( require ( github.com/Mirantis/cri-dockerd v0.0.0-00010101000000-000000000000 github.com/cloudnativelabs/kube-router/v2 v2.0.0-00010101000000-000000000000 + github.com/containerd/aufs v1.0.0 github.com/containerd/cgroups v1.1.0 github.com/containerd/containerd v1.6.10 github.com/containerd/fuse-overlayfs-snapshotter v1.0.5 github.com/containerd/stargz-snapshotter v0.14.3 + github.com/containerd/zfs v1.0.0 github.com/coreos/go-iptables v0.6.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/docker/docker v23.0.3+incompatible @@ -106,7 +108,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/json-iterator/go v1.1.12 - github.com/k3s-io/helm-controller v0.14.0 + github.com/k3s-io/helm-controller v0.15.0 github.com/k3s-io/kine v0.10.1 github.com/klauspost/compress v1.16.5 github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 @@ -114,8 +116,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.15 github.com/minio/minio-go/v7 v7.0.33 github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/onsi/ginkgo/v2 v2.9.1 - github.com/onsi/gomega v1.27.4 + github.com/onsi/ginkgo/v2 v2.9.4 + github.com/onsi/gomega v1.27.6 github.com/opencontainers/runc v1.1.6 github.com/opencontainers/selinux v1.11.0 github.com/otiai10/copy v1.7.0 @@ -246,12 +248,12 @@ require ( github.com/fvbommel/sortorder v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.14 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect @@ -384,12 +386,12 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/mod v0.9.0 // indirect + golang.org/x/mod v0.10.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/term v0.6.0 // indirect + golang.org/x/term v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.8.0 // indirect golang.zx2c4.com/wireguard v0.0.0-20220117163742-e0b8f11489c5 // indirect golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211230205640-daad0b7ba671 // indirect google.golang.org/api v0.60.0 // indirect diff --git a/go.sum b/go.sum index a6e4be06157c..4216b99496b8 100644 --- a/go.sum +++ b/go.sum @@ -186,6 +186,7 @@ github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:P github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= +github.com/containerd/aufs v1.0.0 h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY= github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/btrfs/v2 v2.0.0 h1:FN4wsx7KQrYoLXN7uLP0vBV4oVWHOIKDRQ1G2Z0oL5M= github.com/containerd/btrfs/v2 v2.0.0/go.mod h1:swkD/7j9HApWpzl8OHfrHNxppPd9l44DFZdF94BUj9k= @@ -230,6 +231,7 @@ github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= +github.com/containerd/zfs v1.0.0 h1:cXLJbx+4Jj7rNsTiqVfm6i+RNLx6FFA2fMmDlEf+Wm8= github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= @@ -369,8 +371,9 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= @@ -387,8 +390,9 @@ github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -629,8 +633,8 @@ github.com/k3s-io/etcd/raft/v3 v3.5.7-k3s1 h1:C2FlzI9JcwUhxoT7KSvxcK3aLyzy9bMM5z github.com/k3s-io/etcd/raft/v3 v3.5.7-k3s1/go.mod h1:TflkAb/8Uy6JFBxcRaH2Fr6Slm9mCPVdI2efzxY96yU= github.com/k3s-io/etcd/server/v3 v3.5.7-k3s1 h1:T4VGL9jq4WvxAd7Jiilt7f3ZBPkqPTqa0GZD9t5j50A= github.com/k3s-io/etcd/server/v3 v3.5.7-k3s1/go.mod h1:gxBgT84issUVBRpZ3XkW1T55NjOb4vZZRI4wVvNhf4A= -github.com/k3s-io/helm-controller v0.14.0 h1:jVJtfNIBFvK98oHBjySIxPSP/Hr/iN6LXG0mZVkaTQI= -github.com/k3s-io/helm-controller v0.14.0/go.mod h1:Y8hkgBc37wKWOQBIpFNEJ9/UVGrnEKFaPHpaPwV6cBA= +github.com/k3s-io/helm-controller v0.15.0 h1:ZKpgkq4TX/EAebCoJhPfGRF7w1F8Kz1Kb4ugbkKZoic= +github.com/k3s-io/helm-controller v0.15.0/go.mod h1:BgCPBQblj/Ect4Q7/Umf86WvyDjdG/34D+n8wfXtoeM= github.com/k3s-io/kine v0.10.1 h1:HdpG84WaZvIYYzFB+kT8grNhtlsGzLLYN/MohV+r7+U= github.com/k3s-io/kine v0.10.1/go.mod h1:TU8mr4oByeEPKChQBUpSdGc1mZk+Txr2yZhRY+Jch1Q= github.com/k3s-io/klog v1.0.0-k3s2 h1:yyvD2bQbxG7m85/pvNctLX2bUDmva5kOBvuZ77tTGBA= @@ -891,8 +895,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= -github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= -github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -905,8 +909,8 @@ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2 github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk= -github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= -github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2-0.20210730191737-8e42a01fb1b7/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= @@ -1240,8 +1244,9 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1284,8 +1289,9 @@ golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1374,8 +1380,9 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/manifests/coredns.yaml b/manifests/coredns.yaml index abe733390d5f..cd7d3a736bf3 100644 --- a/manifests/coredns.yaml +++ b/manifests/coredns.yaml @@ -72,6 +72,7 @@ data: loop reload loadbalance + import /etc/coredns/custom/*.override } import /etc/coredns/custom/*.server --- diff --git a/manifests/metrics-server/metrics-server-deployment.yaml b/manifests/metrics-server/metrics-server-deployment.yaml index 43dcf8bfab43..24fafa4f969c 100644 --- a/manifests/metrics-server/metrics-server-deployment.yaml +++ b/manifests/metrics-server/metrics-server-deployment.yaml @@ -44,13 +44,14 @@ spec: emptyDir: {} containers: - name: metrics-server - image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-metrics-server:v0.6.2 + image: %{SYSTEM_DEFAULT_REGISTRY}%rancher/mirrored-metrics-server:v0.6.3 args: - --cert-dir=/tmp - --secure-port=10250 - --kubelet-preferred-address-types=%{PREFERRED_ADDRESS_TYPES}% - --kubelet-use-node-status-port - --metric-resolution=15s + - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 resources: requests: cpu: 100m diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index e4b4aa0a4a34..b4c32f0c7502 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -15,10 +15,10 @@ const ( ) type StartupHookArgs struct { - APIServerReady <-chan struct{} - KubeConfigAdmin string - Skips map[string]bool - Disables map[string]bool + APIServerReady <-chan struct{} + KubeConfigSupervisor string + Skips map[string]bool + Disables map[string]bool } type StartupHook func(context.Context, *sync.WaitGroup, StartupHookArgs) error diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index b61be1515b2e..bdbd3c3e9a84 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -64,7 +64,7 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) error { sc.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt") sc.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt") sc.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key") - sc.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig") + sc.ControlConfig.Runtime.KubeConfigSupervisor = filepath.Join(dataDir, "cred", "supervisor.kubeconfig") return nil } @@ -116,7 +116,7 @@ func save(app *cli.Context, cfg *cmds.Server) error { return err } - sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin) + sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigSupervisor) if err != nil { return err } @@ -150,7 +150,7 @@ func delete(app *cli.Context, cfg *cmds.Server) error { return err } - sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin) + sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigSupervisor) if err != nil { return err } @@ -256,7 +256,7 @@ func prune(app *cli.Context, cfg *cmds.Server) error { return err } - sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin) + sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigSupervisor) if err != nil { return err } diff --git a/pkg/cloudprovider/servicelb.go b/pkg/cloudprovider/servicelb.go index 6c24843afdeb..d0b5be97abfd 100644 --- a/pkg/cloudprovider/servicelb.go +++ b/pkg/cloudprovider/servicelb.go @@ -41,9 +41,12 @@ var ( ) const ( - Ready = condition.Cond("Ready") - DefaultLBNS = meta.NamespaceSystem - DefaultLBImage = "rancher/klipper-lb:v0.4.3" + Ready = condition.Cond("Ready") + DefaultLBNS = meta.NamespaceSystem +) + +var ( + DefaultLBImage = "rancher/klipper-lb:v0.4.4" ) func (k *k3s) Register(ctx context.Context, diff --git a/pkg/containerd/builtins_linux.go b/pkg/containerd/builtins_linux.go index 25ce3a3e4ce1..a0ea4dc49613 100644 --- a/pkg/containerd/builtins_linux.go +++ b/pkg/containerd/builtins_linux.go @@ -20,13 +20,16 @@ package containerd import ( + _ "github.com/containerd/aufs/plugin" _ "github.com/containerd/containerd/metrics" _ "github.com/containerd/containerd/metrics/cgroups" _ "github.com/containerd/containerd/metrics/cgroups/v2" _ "github.com/containerd/containerd/runtime/v2/runc/options" _ "github.com/containerd/containerd/snapshots/btrfs/plugin" + _ "github.com/containerd/containerd/snapshots/devmapper/plugin" _ "github.com/containerd/containerd/snapshots/native/plugin" _ "github.com/containerd/containerd/snapshots/overlay/plugin" _ "github.com/containerd/fuse-overlayfs-snapshotter/plugin" _ "github.com/containerd/stargz-snapshotter/service/plugin" + _ "github.com/containerd/zfs/plugin" ) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 43176b6e4116..e99f443fe574 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -16,6 +16,7 @@ import ( "github.com/rancher/wrangler/pkg/leader" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/client-go/tools/record" utilsnet "k8s.io/utils/net" ) @@ -295,6 +296,7 @@ type ControlRuntime struct { ServiceCurrentKey string KubeConfigAdmin string + KubeConfigSupervisor string KubeConfigController string KubeConfigScheduler string KubeConfigAPIServer string @@ -318,6 +320,8 @@ type ControlRuntime struct { ClientAdminCert string ClientAdminKey string + ClientSupervisorCert string + ClientSupervisorKey string ClientControllerCert string ClientControllerKey string ClientSchedulerCert string @@ -338,6 +342,7 @@ type ControlRuntime struct { ClientETCDKey string Core *core.Factory + Event record.EventRecorder EtcdConfig endpoint.ETCDConfig } diff --git a/pkg/daemons/control/deps/deps.go b/pkg/daemons/control/deps/deps.go index 6753e6c792ed..bfb69f6f278b 100644 --- a/pkg/daemons/control/deps/deps.go +++ b/pkg/daemons/control/deps/deps.go @@ -119,6 +119,7 @@ func CreateRuntimeCertFiles(config *config.Control) { runtime.ServiceCurrentKey = filepath.Join(config.DataDir, "tls", "service.current.key") runtime.KubeConfigAdmin = filepath.Join(config.DataDir, "cred", "admin.kubeconfig") + runtime.KubeConfigSupervisor = filepath.Join(config.DataDir, "cred", "supervisor.kubeconfig") runtime.KubeConfigController = filepath.Join(config.DataDir, "cred", "controller.kubeconfig") runtime.KubeConfigScheduler = filepath.Join(config.DataDir, "cred", "scheduler.kubeconfig") runtime.KubeConfigAPIServer = filepath.Join(config.DataDir, "cred", "api-server.kubeconfig") @@ -126,6 +127,8 @@ func CreateRuntimeCertFiles(config *config.Control) { runtime.ClientAdminCert = filepath.Join(config.DataDir, "tls", "client-admin.crt") runtime.ClientAdminKey = filepath.Join(config.DataDir, "tls", "client-admin.key") + runtime.ClientSupervisorCert = filepath.Join(config.DataDir, "tls", "client-supervisor.crt") + runtime.ClientSupervisorKey = filepath.Join(config.DataDir, "tls", "client-supervisor.key") runtime.ClientControllerCert = filepath.Join(config.DataDir, "tls", "client-controller.crt") runtime.ClientControllerKey = filepath.Join(config.DataDir, "tls", "client-controller.key") runtime.ClientCloudControllerCert = filepath.Join(config.DataDir, "tls", "client-"+version.Program+"-cloud-controller.crt") @@ -351,6 +354,16 @@ func genClientCerts(config *config.Control) error { } } + certGen, err = factory("system:"+version.Program+"-supervisor", []string{user.SystemPrivilegedGroup}, runtime.ClientSupervisorCert, runtime.ClientSupervisorKey) + if err != nil { + return err + } + if certGen { + if err := KubeConfig(runtime.KubeConfigSupervisor, apiEndpoint, runtime.ServerCA, runtime.ClientSupervisorCert, runtime.ClientSupervisorKey); err != nil { + return err + } + } + certGen, err = factory(user.KubeControllerManager, nil, runtime.ClientControllerCert, runtime.ClientControllerKey) if err != nil { return err diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 95b7960623b2..afed71228a5e 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -383,7 +383,7 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error { // If the CCM RBAC changes, the ResourceAttributes checked for by this function should // be modified to check for the most recently added privilege. func checkForCloudControllerPrivileges(ctx context.Context, runtime *config.ControlRuntime, timeout time.Duration) error { - return util.WaitForRBACReady(ctx, runtime.KubeConfigAdmin, timeout, authorizationv1.ResourceAttributes{ + return util.WaitForRBACReady(ctx, runtime.KubeConfigSupervisor, timeout, authorizationv1.ResourceAttributes{ Namespace: metav1.NamespaceSystem, Verb: "watch", Resource: "endpointslices", @@ -424,7 +424,7 @@ func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRu select { case <-ctx.Done(): return - case err := <-promise(func() error { return util.WaitForAPIServerReady(ctx, runtime.KubeConfigAdmin, 30*time.Second) }): + case err := <-promise(func() error { return util.WaitForAPIServerReady(ctx, runtime.KubeConfigSupervisor, 30*time.Second) }): if err != nil { logrus.Infof("Waiting for API server to become available") continue diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index 36269fcb9fee..74d52de6b9fd 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -111,7 +111,7 @@ func ccmYaml() (*asset, error) { return a, nil } -var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x57\x51\x6f\xdb\x38\x12\x7e\xf7\xaf\x20\x04\xe4\xe5\x70\x72\xe2\x0b\xda\xcb\xf1\x2d\x8d\xdd\x36\xb8\xc4\x35\x6c\xa7\x40\xb1\x58\x04\x34\x35\xb6\xb8\xa1\x38\x5c\x92\x72\xe2\xed\xe6\xbf\x2f\x28\xc9\x32\x69\x2b\x69\x92\xed\xfa\xc5\x92\x86\xf3\x0d\xf9\x71\xf8\xcd\x90\x69\xf1\x15\x8c\x15\xa8\x28\x59\x0f\x7a\x77\x42\x65\x94\xcc\xc0\xac\x05\x87\x73\xce\xb1\x54\xae\x57\x80\x63\x19\x73\x8c\xf6\x08\x51\xac\x00\x4a\x38\x1a\xc8\x94\x6d\xde\xad\x66\x1c\x28\xb9\x2b\x17\x90\xda\x8d\x75\x50\xf4\xd2\x34\xed\x85\xd0\x66\xc1\x78\x9f\x95\x2e\x47\x23\xfe\x60\x4e\xa0\xea\xdf\x9d\xd9\xbe\xc0\xe3\x36\xe8\x85\x2c\xad\x03\x33\x45\x09\x51\x44\xc9\x16\x20\xad\x7f\x22\x55\x08\xa3\xc0\x41\xe5\xba\x40\x74\xd6\x19\xa6\xb5\x50\xab\x3a\x46\x9a\xc1\x92\x95\xd2\xd9\x76\xaa\xf5\x84\xe8\x76\xc6\xa6\x94\x60\x69\x2f\x25\x4c\x8b\x4f\x06\x4b\x5d\x21\xa7\x24\x49\x7a\x84\x18\xb0\x58\x1a\x0e\xcd\x37\x50\x99\x46\xa1\x2a\xb0\x94\xd8\x9a\x94\xfa\x45\x63\x56\x3f\xb4\xeb\xf7\xaf\x6b\x30\x8b\xc6\x57\x0a\xeb\xaa\x87\x7b\xe6\x78\x7e\x18\x2f\x13\x96\xe3\x1a\xcc\xa6\xe1\xe1\x99\xe8\x52\xfc\x10\xfd\x6f\xb1\xfd\x41\xa8\x4c\xa8\x55\x44\x3a\x53\x0a\x5d\xe5\xd9\x30\xdf\x05\x19\x6d\x06\x2b\x1d\x96\x3a\x63\x0e\x28\x49\x9c\x29\x21\xf9\xf9\x7b\x87\x12\xa6\xb0\xac\xe6\xd7\xb0\xf9\xcc\x5a\x7b\x84\x1c\x26\xd6\x13\xc8\xb6\x5c\xfc\x06\xdc\x55\x89\xd1\x79\x04\xde\x9c\xf8\x3b\xc2\x51\x2d\xc5\xea\x9a\xe9\xb7\x1c\xa7\xed\xf0\x0b\x34\xb0\x14\x12\x28\xf9\xb3\xe2\xb4\x4f\xdf\x9d\x92\xef\xd5\xa3\xff\x81\x31\x68\x6c\xfb\x9a\x03\x93\x2e\x6f\x5f\x0d\xb0\x6c\xd3\xbe\xed\xb6\x83\x1c\x7d\xbf\xb8\xba\x99\xcd\x47\xd3\xdb\xe1\x97\xeb\xf3\xcb\xf1\xe3\x11\x11\x2a\x65\x59\x66\xfa\xcc\x68\x46\x84\x7e\x5f\x3f\xec\x22\x91\xea\x04\x10\xa1\x2c\xf0\xd2\x40\xf0\x7d\xc9\xa4\x74\xb9\xc1\x72\x95\x77\xa3\xb4\x63\x1f\x77\x13\x45\xeb\x2c\x39\x06\xc7\x8f\x1b\x2a\x8e\xc7\x98\xc1\xe7\xea\x73\x18\xd4\x39\x49\xde\x9f\x04\x1f\x0c\x48\x64\x19\x19\xbc\xb3\xdd\x53\xe8\x08\xa6\x0d\x16\xe0\x72\x28\x2d\xa1\xff\x1b\xbc\x3b\x6d\x0d\x4b\x34\xf7\xcc\x64\xa4\x5f\xcf\xc4\x1f\x47\xb9\xee\x73\x54\xcb\x76\x08\x67\x3c\x07\x72\xba\x9b\x81\x44\xd4\xbd\x78\x32\x81\x8d\x65\x0b\x26\x99\xe2\x35\x3f\xf5\x14\x44\xa1\xd1\xb8\x78\xb1\xbc\xb4\x0e\x8b\xe3\x7f\xf5\xbd\xc6\x80\x39\x48\x22\xa6\xb5\xdd\x1d\xdd\x21\x68\x89\x9b\x02\xde\xa6\xcc\x7b\x87\xf2\xcc\xa6\x4c\xeb\x66\x48\xed\xb8\x7f\x54\x6b\xe0\xc4\xe7\xde\x70\x3c\x4b\x7a\x56\x03\xa7\x95\x5e\xad\x85\x9f\xdf\x67\x61\x1d\x9a\xcd\x95\x28\x84\xa3\xc4\x73\xe3\x0f\xb6\x83\xd5\xa6\x8e\xe1\x36\x1a\x28\x99\xa2\x94\x42\xad\x6e\x2a\x89\xa8\x25\x25\xfc\x42\x1b\xda\x0a\xf6\x70\xa3\xd8\x9a\x09\xc9\x16\x3e\xcf\x07\x1e\x0e\x24\x70\x87\xa6\x1e\x53\x78\xc9\xbb\x0a\xd6\xd0\xbd\x0a\x07\x85\x96\x2d\x70\x48\x54\xb5\x37\x91\xff\x53\x3c\x6c\x57\x5a\xa7\x8d\x40\x23\xdc\xe6\x42\x32\x6b\xc7\x35\x25\x35\xa5\x29\xaf\x05\x26\xe5\x46\x38\xc1\x99\x4c\x1a\x17\x1b\x69\xc8\x78\x6f\x7f\x2a\x6a\x50\x82\x09\x65\xd6\xff\x52\x72\x07\x1b\x4f\x78\x03\x77\x9e\x65\xa8\xec\x17\x25\x37\x49\x90\xe4\xa8\xbd\x27\x1a\x4a\x92\xd1\x83\xb0\xce\x26\x07\x00\x0a\x33\x48\xbd\x68\xee\x49\x35\x47\xe5\x0c\xca\x54\x4b\xa6\xe0\x85\x98\x84\xc0\x72\x09\xdc\x51\x92\x8c\x71\xc6\x73\xc8\x4a\x09\x2f\x0f\x59\x30\xcf\xd0\xcf\x88\xe5\x23\xcc\xa2\x84\x38\xcc\x58\xb4\x94\x48\xa1\xca\x87\x96\x66\x8d\x12\x57\x9b\x99\xf6\x1a\x78\x81\xca\x27\xa8\x2f\xad\x21\xe9\x05\x7b\x98\xdd\xc1\x7d\x9d\x72\xdb\xdf\xd6\xf3\xff\x7e\x75\x71\x10\x2f\x5a\xfe\x68\x04\xa3\xef\x73\x50\x37\xca\x32\x27\xec\x52\xd4\xf9\x3b\xc4\x31\xba\xed\x1a\x82\xa1\x55\x02\x1e\xae\xe3\x89\x04\x7f\x3e\x4d\x09\xf1\x3b\xca\x84\x02\xd3\x7a\xa4\x07\x7a\x50\xff\x44\xc1\x56\x40\xc9\xd1\xf7\xd9\xb7\xd9\x7c\x74\x7d\x3b\x1c\x7d\x3c\xbf\xb9\x9a\xdf\x4e\x47\x9f\x2e\x67\xf3\xe9\xb7\xc7\x23\xc3\x14\xcf\xc1\x1c\x17\xc2\x57\x13\xc8\xd2\x06\x62\xfb\x4f\x07\xfd\xc1\x49\x7f\x10\x23\x4e\x4a\x29\x27\x28\x05\xdf\x50\x72\xb9\x1c\xa3\x9b\x18\xb0\x50\x15\xce\xfa\x17\x35\x37\x2d\x09\x5e\x32\xf6\x16\x59\x40\x81\x66\x43\xc9\xe0\xbf\x27\xd7\x22\x52\xfa\xdf\x4b\xb0\xfb\xa3\xb9\x2e\x29\x19\x9c\x9c\x14\x9d\x18\x11\x04\x33\x2b\x4b\xc9\x2f\x24\x49\xbd\xa4\x27\xff\x26\x49\xa4\xc1\xdb\xd2\x9a\x90\x5f\x5b\x97\x35\xca\xb2\x80\x6b\x7f\x7a\xa3\x54\xd9\x52\xeb\x2b\x7a\x5a\x0f\x0a\xe2\x17\x7e\xfc\x84\xb9\x9c\x46\x2a\x1f\xad\x85\x65\xfe\x3c\x53\xe2\x1b\xa5\x43\xe0\xaa\x1c\xa4\xaf\xc4\x6f\xaa\xc8\x8f\xc3\xf8\xfa\x13\x2d\xa7\xcd\x9e\x09\x1a\x47\x49\x50\x12\xb7\x55\x25\x9e\xbe\x36\xe8\x90\xa3\xa4\xe4\x66\x38\x79\x2d\x4e\xea\xb8\xee\xc4\x9a\x5f\x3c\x83\x15\x15\xea\x2d\x5a\x01\xce\x08\xde\x3d\xb3\x10\xad\xea\x51\xbc\x74\xa3\x72\xf0\xe0\xc2\x0c\x62\x52\xe2\xfd\xc4\x88\xb5\x90\xb0\x82\x91\xe5\x4c\x56\x72\x4c\x7d\x13\x61\x43\xd6\x39\xd3\x6c\x21\xa4\x70\x02\xf6\x72\x90\x65\x59\xfc\x21\x25\xe3\xd1\xfc\xf6\xc3\xe5\x78\x78\x3b\x1b\x4d\xbf\x5e\x5e\x8c\x22\x73\x66\x50\xef\x3b\x30\x29\x3b\x36\x6e\x8a\xe8\x3e\x0a\x09\x4d\xb7\x1a\x6f\xa3\x14\x6b\x50\x60\xed\xc4\xe0\x02\x42\xbc\xdc\x39\xfd\x09\x5c\x1c\x42\xd7\xf9\xb2\xd7\x12\x92\x26\x1d\x28\x39\x3b\x39\x3b\x89\x3e\x5b\x9e\x83\x27\xf9\xf3\x7c\x3e\x09\x0c\x42\x09\x27\x98\x1c\x82\x64\x9b\x19\x70\x54\x99\xa5\x71\x4b\xa6\xc1\x08\xcc\x5a\xdb\x20\xb4\x39\x51\x00\x96\x6e\x67\x0c\x6c\xb6\xe4\x1c\xac\x9d\xe7\x06\x6c\x8e\x32\x8b\xad\x4b\x26\x64\x69\x20\xb0\x9e\x46\x8d\xad\x78\x35\x15\x71\x3b\x1c\x30\x31\x38\x1b\xbc\x99\x89\x67\x88\xf8\xcf\x3f\xcc\x43\xa6\xec\x56\x81\x87\xf5\x45\xaa\x31\xd4\x02\xf2\x0a\x01\xe3\xdb\xab\x4a\xcc\x5b\x77\x41\xa9\xa8\x70\x50\xd8\xfd\x94\xae\x1a\x82\xad\xaa\xee\xd5\xb1\x7a\x0b\x3a\x8d\x8d\x63\xdb\xff\x77\x7a\x1e\x5a\x5f\xa8\x9d\x2f\x59\x5a\x7a\x20\xa4\xbe\x5b\xf1\xaa\xc0\x64\x73\x06\x9f\xbc\xe5\x35\xd7\xc6\x8e\xc6\x3c\xa8\xd8\x4f\x76\xe6\x07\xb7\xee\xdd\x5d\xc5\x77\x1c\x75\x7e\x26\x5e\x0b\x93\x0e\xb3\xe5\x86\xe9\x27\x6f\xdf\x2f\x68\xf4\xb7\x7d\x6c\xd3\xb7\x06\x48\x2f\xbd\x12\xc4\x9d\x7a\x57\xcc\x26\xc6\xe5\x84\x86\xd7\xce\xf1\xec\xf1\xa8\x17\x54\xa6\x74\xaf\xee\xe8\xb0\xa0\xec\x97\x9f\xb4\xa3\xb8\x3c\xe1\x50\x57\x85\xb4\xa3\x7e\xe8\xb8\xcc\xc4\x2e\x7f\x05\x00\x00\xff\xff\xfd\x41\xe7\x07\x25\x13\x00\x00") +var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x57\x51\x6f\xdb\x38\x12\x7e\xf7\xaf\x20\x04\xe4\xe5\x70\x72\xe2\x0b\xda\xcb\xf1\x2d\x8d\xdd\x36\xb8\xc4\x35\x6c\xa7\x40\xb1\x58\x04\x34\x39\xb6\xb8\xa1\x38\x5c\x92\x72\xe2\xed\xe6\xbf\x2f\x28\xc9\xb2\x68\x2b\x69\x92\xed\xfa\xc5\x92\x86\xf3\x0d\xf9\x71\xf8\xcd\x90\x19\xf9\x15\xac\x93\xa8\x29\x59\x0f\x7a\x77\x52\x0b\x4a\x66\x60\xd7\x92\xc3\x39\xe7\x58\x68\xdf\xcb\xc1\x33\xc1\x3c\xa3\x3d\x42\x34\xcb\x81\x12\x8e\x16\x84\x76\xf5\xbb\x33\x8c\x03\x25\x77\xc5\x02\x52\xb7\x71\x1e\xf2\x5e\x9a\xa6\xbd\x36\xb4\x5d\x30\xde\x67\x85\xcf\xd0\xca\x3f\x98\x97\xa8\xfb\x77\x67\xae\x2f\xf1\xb8\x09\x7a\xa1\x0a\xe7\xc1\x4e\x51\x41\x14\x51\xb1\x05\x28\x17\x9e\x48\x19\xc2\x6a\xf0\x50\xba\x2e\x10\xbd\xf3\x96\x19\x23\xf5\xaa\x8a\x91\x0a\x58\xb2\x42\x79\xd7\x4c\xb5\x9a\x10\xdd\xce\xd8\x16\x0a\x1c\xed\xa5\x84\x19\xf9\xc9\x62\x61\x4a\xe4\x94\x24\x49\x8f\x10\x0b\x0e\x0b\xcb\xa1\xfe\x06\x5a\x18\x94\xba\x04\x4b\x89\xab\x48\xa9\x5e\x0c\x8a\xea\xa1\x59\x7f\x78\x5d\x83\x5d\xd4\xbe\x4a\x3a\x5f\x3e\xdc\x33\xcf\xb3\xc3\x78\x42\x3a\x8e\x6b\xb0\x9b\x9a\x87\x67\xa2\x2b\xf9\x43\xf4\xbf\xc5\xf6\x07\xa9\x85\xd4\xab\x88\x74\xa6\x35\xfa\xd2\xb3\x66\xbe\x0b\x32\xda\x0c\x56\x78\x2c\x8c\x60\x1e\x28\x49\xbc\x2d\x20\xf9\xf9\x7b\x87\x0a\xa6\xb0\x2c\xe7\x57\xb3\xf9\xcc\x5a\x7b\x84\x1c\x26\xd6\x13\xc8\xae\x58\xfc\x06\xdc\x97\x89\xd1\x79\x04\xde\x9c\xf8\x3b\xc2\x51\x2f\xe5\xea\x9a\x99\xb7\x1c\xa7\xed\xf0\x0b\xb4\xb0\x94\x0a\x28\xf9\xb3\xe4\xb4\x4f\xdf\x9d\x92\xef\xe5\x63\xf8\x81\xb5\x68\x5d\xf3\x9a\x01\x53\x3e\x6b\x5e\x2d\x30\xb1\x69\xde\x76\xdb\x41\x8e\xbe\x5f\x5c\xdd\xcc\xe6\xa3\xe9\xed\xf0\xcb\xf5\xf9\xe5\xf8\xf1\x88\x48\x9d\x32\x21\x6c\x9f\x59\xc3\x88\x34\xef\xab\x87\x5d\x24\x52\x9e\x00\x22\xb5\x03\x5e\x58\x68\x7d\x5f\x32\xa5\x7c\x66\xb1\x58\x65\xdd\x28\xcd\xd8\xc7\xdd\x44\xd1\x79\x47\x8e\xc1\xf3\xe3\x9a\x8a\xe3\x31\x0a\xf8\x5c\x7e\x6e\x07\xf5\x5e\x91\xf7\x27\xad\x0f\x16\x14\x32\x41\x06\xef\x5c\xf7\x14\x3a\x82\x19\x8b\x39\xf8\x0c\x0a\x47\xe8\xff\x06\xef\x4e\x1b\xc3\x12\xed\x3d\xb3\x82\xf4\xab\x99\x84\xe3\xa8\xd6\x7d\x8e\x7a\xd9\x0c\xe1\x8c\x67\x40\x4e\x77\x33\x50\x88\xa6\x17\x4f\xa6\x65\x63\x62\xc1\x14\xd3\x7c\xc7\x8f\xcc\x0d\x5a\x1f\x2f\x95\x17\xce\x63\x7e\xfc\xaf\x7e\xd0\x03\x2b\x45\x35\xba\x9a\xf0\xb3\xe3\x83\x22\x81\x3d\x48\x39\x66\x8c\xdb\x1d\xf4\x21\x18\x85\x9b\x1c\xde\xa6\xe3\x7b\x47\xf8\xcc\xa5\xcc\x98\x7a\x48\xe5\xb8\x7f\xb0\x2b\xe0\x24\x64\xea\x70\x3c\x4b\x7a\xce\x00\xa7\xa5\xba\xad\x65\x98\xdf\x67\xe9\x3c\xda\xcd\x95\xcc\xa5\xa7\x24\x30\x19\x64\xc0\xc3\x6a\x53\xc5\xf0\x1b\x03\x94\x4c\x51\x29\xa9\x57\x37\xa5\xa0\x54\x02\xd4\xfe\x42\x6b\x42\x73\xf6\x70\xa3\xd9\x9a\x49\xc5\x16\xe1\x54\x0c\x02\x1c\x28\xe0\x1e\x6d\x35\x26\x0f\x02\x79\xd5\x5a\x43\xf7\x2a\x3c\xe4\x46\x35\xc0\x6d\xa2\xca\x9d\x8c\xfc\x9f\xe2\x61\xbb\xd2\x2a\xc9\x24\x5a\xe9\x37\x17\x8a\x39\x37\xae\x28\xa9\x28\x4d\x79\x25\x47\x29\xb7\xd2\x4b\xce\x54\x52\xbb\xb8\x48\x71\xc6\x7b\xfb\x53\x52\x83\x0a\x6c\x5b\x94\xc3\x2f\x25\x77\xb0\x09\x84\xd7\x70\xe7\x42\xa0\x76\x5f\xb4\xda\x24\xad\x23\x81\x26\x78\xa2\xa5\x24\x19\x3d\x48\xe7\x5d\x72\x00\xa0\x51\x40\x1a\x24\x76\x4f\xd8\x39\x6a\x6f\x51\xa5\x46\x31\x0d\x2f\xc4\x24\x04\x96\x4b\xe0\x9e\x92\x64\x8c\x33\x9e\x81\x28\x14\xbc\x3c\x64\xce\x02\x43\x3f\x23\x56\x88\x30\x8b\x12\xe2\x30\x63\xd1\x51\xa2\xa4\x2e\x1e\x1a\x9a\x0d\x2a\x5c\x6d\x66\x26\x28\xe6\x05\xea\x90\xa0\xa1\x10\xb7\x49\xcf\xd9\xc3\xec\x0e\xee\xab\x94\xdb\xfe\xb6\x9e\xff\x0f\xab\x8b\x83\x04\x89\x0b\x47\xa3\x35\xfa\x3e\x03\x7d\xa3\x1d\xf3\xd2\x2d\x65\x95\xbf\x43\x1c\xa3\xdf\xae\xa1\x35\xb4\x4c\xc0\xc3\x75\x3c\x91\xe0\xcf\xa7\x29\x21\x61\x47\x99\xd4\x60\x1b\x8f\xf4\x40\x0f\xaa\x9f\xcc\xd9\x0a\x28\x39\xfa\x3e\xfb\x36\x9b\x8f\xae\x6f\x87\xa3\x8f\xe7\x37\x57\xf3\xdb\xe9\xe8\xd3\xe5\x6c\x3e\xfd\xf6\x78\x64\x99\xe6\x19\xd8\xe3\x5c\x86\xda\x03\x22\xad\x21\xb6\xff\x74\xd0\x1f\x9c\xf4\x07\x31\xe2\xa4\x50\x6a\x82\x4a\xf2\x0d\x25\x97\xcb\x31\xfa\x89\x05\x07\x65\x99\xad\x7e\x51\x2b\xd4\x90\x10\x24\x63\x6f\x91\x39\xe4\x68\x37\x94\x0c\xfe\x7b\x72\x2d\xa3\xba\xf0\x7b\x01\x6e\x7f\x34\x37\x05\x25\x83\x93\x93\xbc\x13\x23\x82\x60\x76\xe5\x28\xf9\x85\x24\x69\x28\x00\xc9\xbf\x49\x12\x69\xf0\xb6\x10\x27\xe4\xd7\xc6\x65\x8d\xaa\xc8\xe1\x3a\x9c\xde\x28\x55\xb6\xd4\x86\xfa\x9f\x56\x83\x5a\xf1\xf3\x30\x7e\xc2\x7c\x46\x23\x95\x8f\xd6\xc2\x44\x38\xcf\x94\x84\xb6\xea\x10\xb8\x2c\x07\xe9\x2b\xf1\xeb\x2a\xf2\xe3\x30\xa1\xfe\x44\xcb\x69\xb2\x67\x82\xd6\x53\xd2\x2a\xa0\xdb\xaa\x12\x4f\xdf\x58\xf4\xc8\x51\x51\x72\x33\x9c\xbc\x16\x27\xf5\xdc\x74\x62\xcd\x2f\x9e\xc1\x8a\xca\xfa\x16\x2d\x07\x6f\x25\xef\x9e\x59\x1b\xad\xec\x68\x82\x74\xa3\xf6\xf0\xe0\xdb\x19\xc4\x94\xc2\xfb\x89\x95\x6b\xa9\x60\x05\x23\xc7\x99\x2a\xe5\x98\x86\x96\xc3\xb5\x59\xe7\xcc\xb0\x85\x54\xd2\x4b\xd8\xcb\x41\x26\x44\xfc\x21\x25\xe3\xd1\xfc\xf6\xc3\xe5\x78\x78\x3b\x1b\x4d\xbf\x5e\x5e\x8c\x22\xb3\xb0\x68\xf6\x1d\x98\x52\x1d\x1b\x37\x45\xf4\x1f\xa5\x82\xba\xb7\x8d\xb7\x51\xc9\x35\x68\x70\x6e\x62\x71\x01\x6d\xbc\xcc\x7b\xf3\x09\x7c\x1c\xc2\x54\xf9\xb2\xd7\x40\x92\x3a\x1d\x28\x39\x3b\x39\x3b\x89\x3e\x3b\x9e\x41\x20\xf9\xf3\x7c\x3e\x69\x19\xa4\x96\x5e\x32\x35\x04\xc5\x36\x33\xe0\xa8\x85\xa3\x71\x03\x67\xc0\x4a\x14\x8d\x6d\xd0\xb6\x79\x99\x03\x16\x7e\x67\x6c\xd9\x5c\xc1\x39\x38\x37\xcf\x2c\xb8\x0c\x95\x88\xad\x4b\x26\x55\x61\xa1\x65\x3d\x8d\xda\x60\xf9\x6a\x2a\xe2\xe6\xb9\xc5\xc4\xe0\x6c\xf0\x66\x26\x9e\x21\xe2\x3f\xff\x30\x0f\x42\xbb\xad\x02\x0f\xab\x6b\x57\x6d\xa8\x04\xe4\x15\x02\xc6\xb7\x17\x9b\x98\xb7\xee\x82\x52\x52\xe1\x21\x77\xfb\x29\x5d\x36\x04\x5b\x55\xdd\xab\x63\xd5\x16\x74\x1a\x6b\xc7\xe6\xb6\xd0\xe9\x79\x68\x7d\xa1\x76\xbe\x64\x69\xe9\x81\x90\x86\x6e\x25\xa8\x02\x53\xf5\x19\x7c\xf2\x4e\x58\x5f\x32\x3b\x1a\xf3\x56\xc5\x7e\xb2\x33\x3f\xb8\xa3\xef\x6e\x36\xa1\xe3\xa8\xf2\x33\x09\x5a\x98\x74\x98\x1d\xb7\xcc\x3c\x79\x57\x7f\x41\xa3\xbf\xed\x63\xeb\xbe\xb5\x85\xf4\xd2\x2b\x41\xdc\xa9\x77\xc5\xac\x63\x5c\x4e\x68\xfb\x92\x3a\x9e\x3d\x1e\xf5\x5a\x95\x29\xdd\xab\x3b\xa6\x5d\x50\xf6\xcb\x4f\xda\x51\x5c\x9e\x70\xa8\xaa\x42\xda\x51\x3f\x4c\x5c\x66\x62\x97\xbf\x02\x00\x00\xff\xff\x2b\x8d\x4e\x50\x53\x13\x00\x00") func corednsYamlBytes() ([]byte, error) { return bindataRead( @@ -231,7 +231,7 @@ func metricsServerMetricsApiserviceYaml() (*asset, error) { return a, nil } -var _metricsServerMetricsServerDeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x55\xdf\x4f\x1b\x47\x10\x7e\xf7\x5f\x31\x72\xc5\xe3\x61\x9b\x2a\x69\xb5\x12\x0f\x08\x9b\xa4\x12\x50\xcb\x67\x2a\xf1\x84\x96\xbd\x31\x5e\xb1\xbf\x3a\x33\xe7\x70\x45\xfc\xef\xd5\xfa\xc8\xe5\x8e\x40\x94\xaa\xc9\x3d\xce\x37\xf3\xcd\xb7\xdf\xee\xcc\x15\x45\x31\xd2\xc9\xfe\x85\xc4\x36\x06\x05\xbb\xd9\xe8\xde\x86\x4a\x41\x89\xb4\xb3\x06\x4f\x8c\x89\x75\x90\x91\x47\xd1\x95\x16\xad\x46\x00\x41\x7b\x54\xe0\x51\xc8\x1a\x2e\x18\x69\x87\xf4\x1c\xe6\xa4\x0d\x2a\xb8\xaf\x6f\xb1\xe0\x86\x05\xfd\xe8\x65\x07\x9d\x12\x4f\xba\x36\x73\x4c\x2e\x36\x1e\xff\x57\x0b\x00\xa7\x6f\xd1\x71\xae\x04\xb8\xff\x9d\x0b\x9d\xd2\x57\xe5\x9c\xd0\xe4\x0c\xc2\x9d\xcd\x52\x3e\x5a\x96\x48\xcd\xb9\xf5\x56\x14\x4c\x47\x00\x2c\xa4\x05\xef\x9a\x96\x47\x9a\x84\x0a\x56\xd1\x39\x1b\xee\xae\x52\xa5\x05\xf7\x71\xea\x47\xda\x54\x00\xaf\x1f\xae\x82\xde\x69\xeb\xf4\xad\x43\x05\xb3\x4c\x87\x0e\x8d\x44\x6a\x73\xbc\x16\xb3\x3d\xef\xe9\x7c\x5b\x29\x80\xa0\x4f\xae\xa3\xef\x3b\x93\xbf\x37\xdc\xc9\x9f\x1b\x34\xf8\x56\x0b\x80\xcf\x86\xe4\x2f\x91\x8d\x64\xa5\x39\x75\x9a\xf9\x72\xcf\x3f\x6e\xdd\x2d\x42\xac\xb0\x30\x64\xc5\x1a\xed\xc6\xcf\xf9\x3c\x78\x1e\x97\x6f\x0b\x92\xe8\x90\xb4\xd8\x18\x7a\xaa\x0a\xb8\xc7\x46\xc1\xf8\xf4\x99\xf5\xa4\xaa\x62\xe0\x3f\x83\x6b\xc6\x5d\x0e\x40\x4c\xb9\x32\x92\x82\xf1\xe2\xc1\xb2\xf0\xf8\x2b\x82\xbd\x36\x8a\x0e\x0f\xf3\x7b\xa0\x80\x82\x7c\x68\xe3\xc4\xc4\x20\x14\x5d\x91\x9c\x0e\xf8\x9d\x9c\x00\xb8\xd9\xa0\x11\x05\xe3\xcb\x58\x9a\x2d\x56\xb5\xc3\xef\x6f\xe9\x35\x0b\xd2\x8f\xe8\xb5\x8b\xae\xf6\xd8\xd9\xf5\x0b\xf8\xec\x31\xd8\x00\xe2\x13\x70\x84\x4f\x08\x46\x07\x60\xbd\x41\xd7\x40\xcd\x08\x1b\x8a\xbe\x60\x43\xf9\x8d\x81\xf5\xfa\x0e\x19\x74\xa8\x26\x91\x80\x50\x57\x45\x0c\xae\x81\x6c\x8a\xb6\x01\x89\x47\x9f\x8f\xd4\xbe\x24\xf1\xa9\xa8\x2c\x75\xea\xd0\x27\x69\xe6\x96\x14\x3c\x3e\x3d\x07\xbf\xd4\xaa\x17\xc5\xaf\xde\x3a\xb4\x22\x14\x1c\x3c\x96\xd7\xe5\x7a\x71\x71\x33\x5f\x9c\x9d\x5c\x9d\xaf\x6f\x56\x8b\x0f\x7f\x94\xeb\xd5\xf5\xd3\x01\xe9\x60\xb6\x48\x13\x6f\x89\x22\x61\x55\x0c\x99\xd4\x6e\x7a\xf8\xfe\xf0\xa8\x23\xd4\x74\x37\x78\x41\x45\x61\x90\x24\xeb\x3e\x9e\x88\x4f\x03\x84\xd1\xd4\x84\x45\x8a\x24\xc7\xb3\xe9\xd1\xbb\xe9\x00\xcd\xf7\xe6\x50\x8a\x44\xb8\x41\xca\x9d\x75\x55\x11\x32\x17\x79\xe4\xf9\xf8\xe0\x71\xb9\x5a\x9c\x2d\x56\xab\xc5\xfc\xe6\x64\x3e\x5f\x2d\xca\xf2\x66\x7d\xbd\x5c\x94\x4f\x07\xaf\xf2\xd4\x8c\xed\x90\xb0\x68\xa9\x79\xdf\x76\x90\xd8\x1e\xac\x20\xe4\xe8\xea\x3c\x0a\xc7\xb3\x77\xdc\x65\xe4\x70\x4d\x06\x7b\xa7\xcb\xc1\xbf\x6b\x64\x19\xc4\x00\x4c\xaa\x15\xcc\xa6\x53\x3f\x88\x7a\xf4\x91\x1a\x05\xbf\x4d\x2f\x6c\x07\x64\x11\x03\xbf\xda\xdb\xda\x8a\x24\xee\x55\x77\xf7\xba\x8c\x24\x99\xbb\x6f\x56\x5e\x0b\x51\xa2\x89\x4e\xc1\xfa\x74\xd9\x53\xac\x2b\x1b\x90\x79\x49\xf1\x16\xfb\x12\x33\xfd\x07\x94\xa1\xea\xa4\x65\xab\x60\x92\xab\x9a\x7f\x86\xc8\xbe\xe9\x4b\x4d\x00\x6c\xb6\x98\xd5\x7e\x5c\xaf\x97\x65\x0f\xb1\xc1\x8a\xd5\x6e\x8e\x4e\x37\x25\x9a\x18\x2a\x6e\x37\x77\x47\x88\x64\x63\xd5\x41\x47\x3d\x48\xac\xc7\x58\x4b\x87\xcd\x7a\x18\xd7\xc6\x20\xf3\x7a\x4b\xc8\xdb\xe8\xaa\x21\xba\xd1\xd6\xd5\x84\x3d\xf4\xd7\x0e\x75\x76\x87\xff\xd9\x89\x5c\xf4\x13\x8c\x78\xff\x0d\x27\x66\xd3\x9f\x6e\xc5\x7e\xe8\xf2\x2f\x24\x06\xc1\x07\x19\xbe\x66\x5d\xe5\xed\xbe\x8a\x51\xce\xac\xc3\xf6\xcf\xa2\x40\xa8\xc6\x7e\x5a\x1d\x4e\xf8\x32\x86\x9c\xf6\x3a\x78\xc5\x48\xfb\x09\xe8\x1f\x47\x3b\x17\x3f\x2d\xc9\xee\xac\xc3\x3b\x5c\xb0\xd1\x6e\xff\xc3\x51\xb0\xd1\x8e\xbf\x70\xb4\x7b\xf5\x22\x2f\xd3\x57\x26\xe3\xe5\x12\x84\x76\xed\x2e\xdb\x2b\xcb\x1b\xe6\xdf\x00\x00\x00\xff\xff\x14\xc8\x49\x02\x2c\x09\x00\x00") +var _metricsServerMetricsServerDeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x55\xc1\x6e\xdb\x46\x10\xbd\xeb\x2b\x06\x2a\x7c\x2b\x6d\x49\xa9\xdd\x80\x80\x0f\x82\xc4\x44\x01\x6c\x47\x10\xe5\x16\x3e\x09\xeb\xe5\xc8\x5a\x78\xb9\xbb\x9d\x19\x2a\x61\x83\xfc\x7b\xb1\xa2\x43\x93\x8e\x1d\xb8\x68\xc3\x03\x0f\x33\x6f\xde\x3c\xbe\x9d\xe5\x24\x49\x32\x50\xc1\xfc\x81\xc4\xc6\xbb\x14\xf6\xe3\xc1\xbd\x71\x45\x0a\x39\xd2\xde\x68\x9c\x6a\xed\x2b\x27\x83\x12\x45\x15\x4a\x54\x3a\x00\x70\xaa\xc4\x14\x4a\x14\x32\x9a\x13\x46\xda\x23\x3d\x84\x39\x28\x8d\x29\xdc\x57\xb7\x98\x70\xcd\x82\xe5\xe0\x69\x07\x15\x02\x9f\xb4\x6d\xe6\x18\xac\xaf\x4b\xfc\x4f\x2d\x00\xac\xba\x45\xcb\xb1\x12\xe0\xfe\x2d\x27\x2a\x84\xef\xca\x39\xa0\x8e\x08\xc2\xbd\x89\x52\x16\x86\xc5\x53\x7d\x61\x4a\x23\x29\x8c\x06\x00\x2c\xa4\x04\xef\xea\x86\x47\xea\x80\x29\xac\xbc\xb5\xc6\xdd\x5d\x87\x42\x09\x1e\xe2\xd4\x8d\x34\x50\x80\x52\x7d\xbe\x76\x6a\xaf\x8c\x55\xb7\x16\x53\x18\x47\x3a\xb4\xa8\xc5\x53\x83\x29\x95\xe8\xdd\x45\x47\xe7\xcb\x4a\x01\x04\xcb\x60\x5b\xfa\xae\x33\xf1\x79\xc1\x9d\xf8\xd8\x5e\x83\x1f\xb5\x00\xf8\x66\x48\x7c\x02\x19\x4f\x46\xea\x99\x55\xcc\x57\x07\xfe\x61\xe3\x6e\xe2\x7c\x81\x89\x26\x23\x46\x2b\x3b\x7c\xc0\x73\x6f\x3c\xae\x5e\x16\x24\xde\x22\x29\x31\xde\x75\x54\x25\x70\x8f\x75\x0a\xc3\xd9\x03\xeb\xb4\x28\xbc\xe3\x8f\xce\xd6\xc3\x16\x03\xe0\x43\xac\xf4\x94\xc2\x30\xfb\x6c\x58\x78\xf8\x1d\xc1\x41\x1b\x79\x8b\xc7\x71\x1e\xc8\xa1\x20\x1f\x1b\x7f\xa2\xbd\x13\xf2\x36\x09\x56\x39\x7c\x25\x27\x00\x6e\xb7\xa8\x25\x85\xe1\x95\xcf\xf5\x0e\x8b\xca\xe2\xeb\x5b\x96\x8a\x05\xe9\xff\xe8\xb5\xf7\xb6\x2a\xb1\xb5\xeb\x17\x28\xa3\xc7\x60\x1c\x48\x19\x80\x3d\x7c\x42\xd0\xca\x01\xab\x2d\xda\x1a\x2a\x46\xd8\x92\x2f\x13\xd6\x14\x67\x0c\x4c\xa9\xee\x90\x41\xb9\xe2\xc4\x13\x10\xaa\x22\xf1\xce\xd6\x10\x4d\x51\xc6\x21\xf1\xe0\xdb\x27\x35\x93\x24\x65\x48\x0a\x43\xad\x3a\x2c\x83\xd4\x73\x43\x29\x7c\xf9\xfa\x10\x7c\xac\x4d\x9f\x14\x3f\x7b\xea\xd0\x88\x48\xe1\xe8\x4b\x7e\x93\xaf\xb3\xcb\xcd\x3c\x7b\x37\xbd\xbe\x58\x6f\x56\xd9\xfb\x0f\xf9\x7a\x75\xf3\xf5\x88\x94\xd3\x3b\xa4\x93\xd2\x10\x79\xc2\x22\xe9\x33\xa5\xfb\xd1\xf1\xd9\xf1\x9b\x96\x50\xd1\x5d\x6f\x82\x92\x44\x23\x49\xd4\x7d\x7e\x22\x65\xe8\x65\x18\x75\x45\x98\x04\x4f\x72\x3e\x1e\x4d\x4e\x47\xbd\x6c\x3c\x37\x8b\x92\x04\xc2\x2d\x52\xec\xac\x8a\x82\x90\x39\x89\x57\x9e\xcf\x8f\xbe\x2c\x57\xd9\xbb\x6c\xb5\xca\xe6\x9b\xe9\x7c\xbe\xca\xf2\x7c\xb3\xbe\x59\x66\xf9\xd7\xa3\x67\x79\x2a\xc6\xe6\x92\xb0\x28\xa9\xf8\xd0\xb6\x07\x6c\x3e\x2c\x21\x64\x6f\xab\x78\x15\xce\xc7\xa7\xdc\x43\x88\xe5\x44\x9b\xb0\x43\x4a\xb8\x32\x82\x7c\xbe\xbe\xc8\x37\xd9\x6c\xbe\xc8\xe2\x3b\x9f\x6e\xfe\xfc\xb0\x5e\x6c\xa6\x59\xbe\x99\x9c\x9e\x6d\xde\xcf\x2e\x37\xf9\x62\xfa\xe6\xed\x6f\xbf\x3e\xe2\x56\xaf\x42\x3d\x61\x1b\x4f\xde\x7e\xc3\x4d\x4e\xcf\x5e\x62\x7b\x11\xd5\x61\x9b\x2d\xa6\xb3\xc5\x74\x32\xda\x2c\x3f\x5e\xdc\x8c\xdf\x8c\x4e\x9f\x23\xfb\x0e\xd4\xba\x10\xcd\xa9\x48\x63\xe7\x8c\x63\xf0\xaf\x0a\x59\x7a\x31\x00\x1d\xaa\x14\xc6\xa3\x51\xd9\x8b\x96\x58\x7a\xaa\x53\xf8\x7d\x74\x69\xda\x44\x3c\x8a\xde\xd4\x34\x33\xbb\x13\x09\xdc\xa9\x6e\xa7\x7b\xe9\x49\x22\x77\x77\x64\xe2\xcf\xd1\x8b\xd7\xde\xa6\xb0\x9e\x2d\x3b\x8a\x55\x61\x1c\x32\x2f\xc9\xdf\x62\x57\x62\xa4\x7f\x8f\xd2\x57\x1d\x94\xec\x52\x38\x89\x55\xf5\xdf\xfd\xcc\xa1\xe9\x53\x4d\x00\xac\x77\x18\xd5\x2e\xd6\xeb\x65\xde\xc9\x18\x67\xc4\x28\x3b\x47\xab\xea\x1c\xb5\x77\x05\x37\xfb\xab\x25\x44\x32\xbe\x68\x53\x93\x4e\x4a\x4c\x89\xbe\x92\x36\x37\xee\xe4\xb8\xd2\x1a\x99\xd7\x3b\x42\xde\x79\x5b\xf4\xb3\x5b\x65\x6c\x45\xd8\xc9\x3e\xde\x4d\x6b\xf6\xf8\xaf\x9d\x88\x45\x3f\xc1\x88\xb3\x1f\x38\x31\x1e\xfd\x74\x2b\x0e\xbf\x9e\xb8\x48\xbd\x13\xfc\x2c\xfd\x69\x56\x45\xdc\x71\x2b\xef\xe5\x9d\xb1\xd8\xec\xd7\x14\x84\x2a\xec\xc2\x2a\x37\xe5\x2b\xef\x22\xec\xf9\xe4\x35\x23\x1d\x6e\x40\xf7\x73\x94\xb5\xfe\xd3\x92\xcc\xde\x58\xbc\xc3\x8c\xb5\xb2\x87\xb5\x9b\xc2\x56\x59\x7e\xe4\x68\xb6\xcb\x65\x5c\x29\xcf\xdc\x8c\xa7\xab\x00\x9a\xe5\xb3\x6c\x8e\x2c\xfe\x67\xff\x09\x00\x00\xff\xff\x70\xb0\x51\x48\x32\x0a\x00\x00") func metricsServerMetricsServerDeploymentYamlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/nodepassword/nodepassword.go b/pkg/nodepassword/nodepassword.go index b52315c1e7e2..e1a460c08a72 100644 --- a/pkg/nodepassword/nodepassword.go +++ b/pkg/nodepassword/nodepassword.go @@ -15,13 +15,38 @@ import ( v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" ) var ( // Hasher provides the algorithm for generating and verifying hashes - Hasher = hash.NewSCrypt() + Hasher = hash.NewSCrypt() + ErrVerifyFailed = errVerifyFailed() ) +type passwordError struct { + node string + err error +} + +func (e *passwordError) Error() string { + return fmt.Sprintf("unable to verify password for node %s: %v", e.node, e.err) +} + +func (e *passwordError) Is(target error) bool { + switch target { + case ErrVerifyFailed: + return true + } + return false +} + +func (e *passwordError) Unwrap() error { + return e.err +} + +func errVerifyFailed() error { return &passwordError{} } + func getSecretName(nodeName string) string { return strings.ToLower(nodeName + ".node-password." + version.Program) } @@ -30,39 +55,34 @@ func verifyHash(secretClient coreclient.SecretClient, nodeName, pass string) err name := getSecretName(nodeName) secret, err := secretClient.Get(metav1.NamespaceSystem, name, metav1.GetOptions{}) if err != nil { - return err + return &passwordError{node: nodeName, err: err} } if hash, ok := secret.Data["hash"]; ok { if err := Hasher.VerifyHash(string(hash), pass); err != nil { - return errors.Wrapf(err, "unable to verify hash for node '%s'", nodeName) + return &passwordError{node: nodeName, err: err} } return nil } - return fmt.Errorf("unable to locate hash data for node secret '%s'", name) + return &passwordError{node: nodeName, err: errors.New("password hash not found in node secret")} } // Ensure will verify a node-password secret if it exists, otherwise it will create one func Ensure(secretClient coreclient.SecretClient, nodeName, pass string) error { - if err := verifyHash(secretClient, nodeName, pass); !apierrors.IsNotFound(err) { - return err - } - - hash, err := Hasher.CreateHash(pass) - if err != nil { - return errors.Wrapf(err, "unable to create hash for node '%s'", nodeName) - } - - immutable := true - _, err = secretClient.Create(&v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: getSecretName(nodeName), - Namespace: metav1.NamespaceSystem, - }, - Immutable: &immutable, - Data: map[string][]byte{"hash": []byte(hash)}, - }) - if apierrors.IsAlreadyExists(err) { - return verifyHash(secretClient, nodeName, pass) + err := verifyHash(secretClient, nodeName, pass) + if apierrors.IsNotFound(err) { + var hash string + hash, err = Hasher.CreateHash(pass) + if err != nil { + return &passwordError{node: nodeName, err: err} + } + _, err = secretClient.Create(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: getSecretName(nodeName), + Namespace: metav1.NamespaceSystem, + }, + Immutable: pointer.Bool(true), + Data: map[string][]byte{"hash": []byte(hash)}, + }) } return err } diff --git a/pkg/nodepassword/nodepassword_test.go b/pkg/nodepassword/nodepassword_test.go index 4639a5ee4298..ef8624ca91ba 100644 --- a/pkg/nodepassword/nodepassword_test.go +++ b/pkg/nodepassword/nodepassword_test.go @@ -1,6 +1,7 @@ package nodepassword import ( + "errors" "fmt" "log" "os" @@ -89,6 +90,13 @@ func Test_UnitMigrateFileNodes(t *testing.T) { assertNotEqual(t, Ensure(secretClient, newNode, "wrong-password"), nil) } +func Test_PasswordError(t *testing.T) { + err := &passwordError{node: "test", err: fmt.Errorf("inner error")} + assertEqual(t, errors.Is(err, ErrVerifyFailed), true) + assertEqual(t, errors.Is(err, fmt.Errorf("different error")), false) + assertNotEqual(t, errors.Unwrap(err), nil) +} + // -------------------------- // mock secret client interface diff --git a/pkg/server/context.go b/pkg/server/context.go index 0ad06230dc54..7af5122c90c9 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -2,29 +2,25 @@ package server import ( "context" - "fmt" - "os" - "runtime" helmcrd "github.com/k3s-io/helm-controller/pkg/crd" "github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io" addoncrd "github.com/k3s-io/k3s/pkg/crd" - "github.com/k3s-io/k3s/pkg/deploy" "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" - "github.com/rancher/wrangler/pkg/apply" "github.com/rancher/wrangler/pkg/crd" "github.com/rancher/wrangler/pkg/generated/controllers/apps" "github.com/rancher/wrangler/pkg/generated/controllers/batch" "github.com/rancher/wrangler/pkg/generated/controllers/core" "github.com/rancher/wrangler/pkg/generated/controllers/rbac" "github.com/rancher/wrangler/pkg/start" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/record" ) type Context struct { @@ -35,7 +31,7 @@ type Context struct { Auth *rbac.Factory Core *core.Factory K8s kubernetes.Interface - Apply apply.Apply + Event record.EventRecorder } func (c *Context) Start(ctx context.Context) error { @@ -47,16 +43,7 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) { if err != nil { return nil, err } - - // Construct a custom user-agent string for the apply client used by the deploy controller - // so that we can track which node's deploy controller most recently modified a resource. - nodeName := os.Getenv("NODE_NAME") - managerName := deploy.ControllerName + "@" + nodeName - if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength { - logrus.Warn("Deploy controller node name is empty or too long, and will not be tracked via server side apply field management") - managerName = deploy.ControllerName - } - restConfig.UserAgent = fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit) + restConfig.UserAgent = util.GetUserAgent(version.Program + "-supervisor") if err := crds(ctx, restConfig); err != nil { return nil, errors.Wrap(err, "failed to register CRDs") @@ -74,7 +61,7 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) { Apps: apps.NewFactoryFromConfigOrDie(restConfig), Batch: batch.NewFactoryFromConfigOrDie(restConfig), Core: core.NewFactoryFromConfigOrDie(restConfig), - Apply: apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup(), + Event: util.BuildControllerEventRecorder(k8s, version.Program+"-supervisor", metav1.NamespaceAll), }, nil } diff --git a/pkg/server/router.go b/pkg/server/router.go index b3473c938339..d9209cf0294b 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -26,10 +26,13 @@ import ( certutil "github.com/rancher/dynamiclistener/cert" coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" @@ -140,7 +143,7 @@ func cacerts(serverCA string) http.Handler { var err error ca, err = os.ReadFile(serverCA) if err != nil { - sendError(err, resp) + sendError(err, resp, req) return } } @@ -215,13 +218,13 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo nodeName, errCode, err := auth(req) if err != nil { - sendError(err, resp, errCode) + sendError(err, resp, req, errCode) return } caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey) if err != nil { - sendError(err, resp) + sendError(err, resp, req) return } @@ -231,7 +234,7 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo for _, v := range strings.Split(nodeIP, ",") { ip := net.ParseIP(v) if ip == nil { - sendError(fmt.Errorf("invalid IP address %s", ip), resp) + sendError(fmt.Errorf("invalid node IP address %s", ip), resp, req) return } ips = append(ips, ip) @@ -247,7 +250,7 @@ func servingKubeletCert(server *config.Control, keyFile string, auth nodePassBoo }, }, key, caCerts[0], caKey) if err != nil { - sendError(err, resp) + sendError(err, resp, req) return } @@ -271,13 +274,13 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot nodeName, errCode, err := auth(req) if err != nil { - sendError(err, resp, errCode) + sendError(err, resp, req, errCode) return } caCerts, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey) if err != nil { - sendError(err, resp) + sendError(err, resp, req) return } @@ -287,7 +290,7 @@ func clientKubeletCert(server *config.Control, keyFile string, auth nodePassBoot Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, }, key, caCerts[0], caKey) if err != nil { - sendError(err, resp) + sendError(err, resp, req) return } @@ -397,7 +400,7 @@ func serveStatic(urlPrefix, staticDir string) http.Handler { return http.StripPrefix(urlPrefix, http.FileServer(http.Dir(staticDir))) } -func sendError(err error, resp http.ResponseWriter, status ...int) { +func sendError(err error, resp http.ResponseWriter, req *http.Request, status ...int) { var code int if len(status) == 1 { code = status[0] @@ -405,9 +408,11 @@ func sendError(err error, resp http.ResponseWriter, status ...int) { if code == 0 || code == http.StatusOK { code = http.StatusInternalServerError } - logrus.Error(err) - resp.WriteHeader(code) - resp.Write([]byte(err.Error())) + logrus.Errorf("Sending HTTP %d response to %s: %v", code, req.RemoteAddr, err) + responsewriters.ErrorNegotiated( + apierrors.NewGenericServerResponse(code, req.Method, schema.GroupResource{}, req.URL.Path, err.Error(), 0, true), + scheme.Codecs.WithoutConversion(), schema.GroupVersion{}, resp, req, + ) } // nodePassBootstrapper returns a node name, or http error code and error @@ -457,12 +462,23 @@ func passwordBootstrap(ctx context.Context, config *Config) nodePassBootstrapper } } + // verify that the node exists, if using Node Identity auth if err := verifyNode(ctx, nodeClient, node); err != nil { return "", http.StatusUnauthorized, err } + // verify that the node password secret matches, or create it if it does not if err := nodepassword.Ensure(secretClient, node.Name, node.Password); err != nil { - return "", http.StatusForbidden, err + // if the verification failed, reject the request + if errors.Is(err, nodepassword.ErrVerifyFailed) { + return "", http.StatusForbidden, err + } + // If verification failed due to an error creating the node password secret, allow + // the request, but retry verification until the outage is resolved. This behavior + // allows nodes to join the cluster during outages caused by validating webhooks + // blocking secret creation - if the outage requires new nodes to join in order to + // run the webhook pods, we must fail open here to resolve the outage. + return verifyRemotePassword(ctx, config, &mu, deferredNodes, node) } return node.Name, http.StatusOK, nil @@ -489,7 +505,7 @@ func verifyLocalPassword(ctx context.Context, config *Config, mu *sync.Mutex, de } if err := nodepassword.Hasher.VerifyHash(passHash, node.Password); err != nil { - return "", http.StatusForbidden, errors.Wrapf(err, "unable to verify local password for node '%s'", node.Name) + return "", http.StatusForbidden, errors.Wrap(err, "unable to verify local node password") } mu.Lock() @@ -498,7 +514,7 @@ func verifyLocalPassword(ctx context.Context, config *Config, mu *sync.Mutex, de if _, ok := deferredNodes[node.Name]; !ok { deferredNodes[node.Name] = true go ensureSecret(ctx, config, node) - logrus.Debugf("Password verified locally for node '%s'", node.Name) + logrus.Infof("Password verified locally for node %s", node.Name) } return node.Name, http.StatusOK, nil @@ -511,7 +527,7 @@ func verifyRemotePassword(ctx context.Context, config *Config, mu *sync.Mutex, d if _, ok := deferredNodes[node.Name]; !ok { deferredNodes[node.Name] = true go ensureSecret(ctx, config, node) - logrus.Debugf("Password verification deferred for node '%s'", node.Name) + logrus.Infof("Password verification deferred for node %s", node.Name) } return node.Name, http.StatusOK, nil @@ -528,19 +544,25 @@ func verifyNode(ctx context.Context, nodeClient coreclient.NodeClient, node *nod func ensureSecret(ctx context.Context, config *Config, node *nodeInfo) { runtime := config.ControlConfig.Runtime - for { - select { - case <-ctx.Done(): - return - case <-time.After(1 * time.Second): - if runtime.Core != nil { - logrus.Debugf("Runtime core has become available, ensuring password secret for node '%s'", node.Name) - secretClient := runtime.Core.Core().V1().Secret() - if err := nodepassword.Ensure(secretClient, node.Name, node.Password); err != nil { - logrus.Warnf("Error ensuring node password secret for pre-validated node '%s': %v", node.Name, err) - } - return + wait.PollImmediateUntilWithContext(ctx, time.Second*5, func(ctx context.Context) (bool, error) { + if runtime.Core != nil { + secretClient := runtime.Core.Core().V1().Secret() + // This is consistent with events attached to the node generated by the kubelet + // https://github.com/kubernetes/kubernetes/blob/612130dd2f4188db839ea5c2dea07a96b0ad8d1c/pkg/kubelet/kubelet.go#L479-L485 + nodeRef := &corev1.ObjectReference{ + Kind: "Node", + Name: node.Name, + UID: types.UID(node.Name), + Namespace: "", + } + if err := nodepassword.Ensure(secretClient, node.Name, node.Password); err != nil { + runtime.Event.Eventf(nodeRef, corev1.EventTypeWarning, "NodePasswordValidationFailed", "Deferred node password secret validation failed: %v", err) + // Return true to stop polling if the password verification failed; only retry on secret creation errors. + return errors.Is(err, nodepassword.ErrVerifyFailed), nil } + runtime.Event.Event(nodeRef, corev1.EventTypeNormal, "NodePasswordValidationComplete", "Deferred node password secret validation complete") + return true, nil } - } + return false, nil + }) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 7cdd84226aca..722a2523719a 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -12,7 +12,7 @@ import ( "sync" "time" - helm "github.com/k3s-io/helm-controller/pkg/controllers/chart" + helmchart "github.com/k3s-io/helm-controller/pkg/controllers/chart" helmcommon "github.com/k3s-io/helm-controller/pkg/controllers/common" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" @@ -28,12 +28,15 @@ import ( "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" + "github.com/rancher/wrangler/pkg/apply" v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/rancher/wrangler/pkg/leader" "github.com/rancher/wrangler/pkg/resolvehome" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) const ( @@ -67,10 +70,10 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error { config.ControlConfig.Runtime.StartupHooksWg = wg shArgs := cmds.StartupHookArgs{ - APIServerReady: config.ControlConfig.Runtime.APIServerReady, - KubeConfigAdmin: config.ControlConfig.Runtime.KubeConfigAdmin, - Skips: config.ControlConfig.Skips, - Disables: config.ControlConfig.Disables, + APIServerReady: config.ControlConfig.Runtime.APIServerReady, + KubeConfigSupervisor: config.ControlConfig.Runtime.KubeConfigSupervisor, + Skips: config.ControlConfig.Skips, + Disables: config.ControlConfig.Disables, } for _, hook := range config.StartupHooks { if err := hook(ctx, wg, shArgs); err != nil { @@ -101,7 +104,7 @@ func startOnAPIServerReady(ctx context.Context, config *Config) { func runControllers(ctx context.Context, config *Config) error { controlConfig := &config.ControlConfig - sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigAdmin) + sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigSupervisor) if err != nil { return errors.Wrap(err, "failed to create new server context") } @@ -118,6 +121,7 @@ func runControllers(ctx context.Context, config *Config) error { controlConfig.Runtime.NodePasswdFile); err != nil { logrus.Warn(errors.Wrap(err, "error migrating node-password file")) } + controlConfig.Runtime.Event = sc.Event controlConfig.Runtime.Core = sc.Core for name, cb := range controlConfig.Runtime.ClusterControllerStarts { @@ -205,26 +209,42 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error { // apply SystemDefaultRegistry setting to Helm before starting controllers if config.ControlConfig.SystemDefaultRegistry != "" { - helm.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helm.DefaultJobImage + helmchart.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helmchart.DefaultJobImage } if !config.ControlConfig.DisableHelmController { - helm.Register(ctx, + restConfig, err := clientcmd.BuildConfigFromFlags("", config.ControlConfig.Runtime.KubeConfigSupervisor) + if err != nil { + return err + } + restConfig.UserAgent = util.GetUserAgent(helmcommon.Name) + + k8s, err := clientset.NewForConfig(restConfig) + if err != nil { + return err + } + + apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup() + helm := sc.Helm.WithAgent(restConfig.UserAgent) + batch := sc.Batch.WithAgent(restConfig.UserAgent) + auth := sc.Auth.WithAgent(restConfig.UserAgent) + core := sc.Core.WithAgent(restConfig.UserAgent) + helmchart.Register(ctx, metav1.NamespaceAll, helmcommon.Name, - sc.K8s, - sc.Apply, - util.BuildControllerEventRecorder(sc.K8s, helmcommon.Name, metav1.NamespaceAll), - sc.Helm.Helm().V1().HelmChart(), - sc.Helm.Helm().V1().HelmChart().Cache(), - sc.Helm.Helm().V1().HelmChartConfig(), - sc.Helm.Helm().V1().HelmChartConfig().Cache(), - sc.Batch.Batch().V1().Job(), - sc.Batch.Batch().V1().Job().Cache(), - sc.Auth.Rbac().V1().ClusterRoleBinding(), - sc.Core.Core().V1().ServiceAccount(), - sc.Core.Core().V1().ConfigMap(), - sc.Core.Core().V1().Secret()) + k8s, + apply, + util.BuildControllerEventRecorder(k8s, helmcommon.Name, metav1.NamespaceAll), + helm.V1().HelmChart(), + helm.V1().HelmChart().Cache(), + helm.V1().HelmChartConfig(), + helm.V1().HelmChartConfig().Cache(), + batch.V1().Job(), + batch.V1().Job().Cache(), + auth.V1().ClusterRoleBinding(), + core.V1().ServiceAccount(), + core.V1().ConfigMap(), + core.V1().Secret()) } if config.ControlConfig.EncryptSecrets { @@ -274,10 +294,24 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control) return err } + restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor) + if err != nil { + return err + } + restConfig.UserAgent = util.GetUserAgent("deploy") + + k8s, err := clientset.NewForConfig(restConfig) + if err != nil { + return err + } + + apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup() + k3s := sc.K3s.WithAgent(restConfig.UserAgent) + return deploy.WatchFiles(ctx, - sc.K8s, - sc.Apply, - sc.K3s.K3s().V1().Addon(), + k8s, + apply, + k3s.V1().Addon(), controlConfig.Disables, dataDir) } diff --git a/pkg/util/client.go b/pkg/util/client.go index 6c6fefd00060..561a5cbc0817 100644 --- a/pkg/util/client.go +++ b/pkg/util/client.go @@ -1,9 +1,15 @@ package util import ( + "fmt" + "os" + "runtime" "strings" "github.com/k3s-io/k3s/pkg/datadir" + "github.com/k3s-io/k3s/pkg/version" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/validation" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" ) @@ -30,6 +36,17 @@ func GetClientSet(file string) (clientset.Interface, error) { return clientset.NewForConfig(restConfig) } +// GetUserAgent builds a complete UserAgent string for a given controller, including the node name if possible. +func GetUserAgent(controllerName string) string { + nodeName := os.Getenv("NODE_NAME") + managerName := controllerName + "@" + nodeName + if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength { + logrus.Warnf("%s controller node name is empty or too long, and will not be tracked via server side apply field management", controllerName) + managerName = controllerName + } + return fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit) +} + // SplitStringSlice is a helper function to handle StringSliceFlag containing multiple values // By default, StringSliceFlag only supports repeated values, not multiple values // e.g. --foo="bar,car" --foo=baz will result in []string{"bar", "car". "baz"} diff --git a/scripts/airgap/image-list.txt b/scripts/airgap/image-list.txt index c6b0b6d818a4..93536794af1c 100644 --- a/scripts/airgap/image-list.txt +++ b/scripts/airgap/image-list.txt @@ -1,8 +1,8 @@ docker.io/rancher/klipper-helm:v0.8.0-build20230510 -docker.io/rancher/klipper-lb:v0.4.3 +docker.io/rancher/klipper-lb:v0.4.4 docker.io/rancher/local-path-provisioner:v0.0.24 docker.io/rancher/mirrored-coredns-coredns:1.10.1 docker.io/rancher/mirrored-library-busybox:1.34.1 docker.io/rancher/mirrored-library-traefik:2.9.10 -docker.io/rancher/mirrored-metrics-server:v0.6.2 +docker.io/rancher/mirrored-metrics-server:v0.6.3 docker.io/rancher/mirrored-pause:3.6 diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index c73631c1a65b..0e6f89fb03c5 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -342,17 +342,17 @@ var _ = Describe("Verify Create", Ordered, func() { }) It("Validates certificates", func() { const grepCert = "sudo ls -lt /var/lib/rancher/k3s/server/ | grep tls" - var expectResult = []string{"client-ca.crt", - "client-ca.key", - "client-ca.nochain.crt", + var expectResult = []string{ + "client-ca.crt", "client-ca.key", "client-ca.nochain.crt", + "client-supervisor.crt", "client-supervisor.key", "dynamic-cert.json", "peer-ca.crt", "peer-ca.key", "server-ca.crt", "server-ca.key", "request-header-ca.crt", "request-header-ca.key", "server-ca.crt", "server-ca.key", "server-ca.nochain.crt", "service.current.key", "service.key", - "apiserver-loopback-client__.crt", - "apiserver-loopback-client__.key", "", + "apiserver-loopback-client__.crt", "apiserver-loopback-client__.key", + "", } var finalResult string