diff --git a/go.mod b/go.mod index 2b6408be5e2..9d1577e4ca6 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,11 @@ replace ( github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34 github.com/golang/protobuf => github.com/k3s-io/protobuf v1.4.3-k3s1 github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f - github.com/k3s-io/helm-controller => github.com/k3s-io/helm-controller v0.9.0 github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.21.0-k3s1 github.com/matryer/moq => github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009 github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc93.0.20210414171415-3397a09ee932 github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210316141917-a8c4a9ee0f6b + github.com/rancher/wharfie => github.com/rancher/wharfie v0.3.5 go.etcd.io/etcd => github.com/k3s-io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884 google.golang.org/grpc => google.golang.org/grpc v1.27.1 @@ -55,19 +55,16 @@ replace ( require ( github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e - github.com/frankban/quicktest v1.12.0 // indirect - github.com/google/go-containerregistry v0.4.2-0.20210316173552-70c58c0e4786 - github.com/k3s-io/helm-controller v0.9.0 - github.com/klauspost/compress v1.11.7 - github.com/pierrec/lz4 v2.5.2+incompatible + github.com/google/go-containerregistry v0.5.0 + github.com/k3s-io/helm-controller v0.9.1 github.com/pkg/errors v0.9.1 - github.com/rancher/k3s v1.21.1-0.20210510220633-948295e8e842 + github.com/rancher/k3s v1.21.1-0.20210512014615-e10524a6b182 + github.com/rancher/wharfie v0.3.5 github.com/rancher/wrangler v0.6.2 github.com/rancher/wrangler-api v0.6.0 github.com/sirupsen/logrus v1.8.1 github.com/urfave/cli v1.22.5 google.golang.org/grpc v1.37.0 - gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.21.0 k8s.io/apimachinery v0.21.0 k8s.io/apiserver v0.21.0 diff --git a/go.sum b/go.sum index 64734a9f309..b2e882923e5 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/ github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.12.0 h1:qlT496aO6ryji0l8Of/hVQCRkqNVyIbJtUZ1zDp6xyw= -github.com/frankban/quicktest v1.12.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= +github.com/frankban/quicktest v1.12.1 h1:P6vQcHwZYgVGIpUzKB5DXzkEeYJppJOStPLuh9aB89c= +github.com/frankban/quicktest v1.12.1/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -394,6 +394,8 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v1.0.0/go.mod h1:zpM0OfbMCjPtd2qkTD/jX2MgiFCqklhSUFyDW44gVQE= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= @@ -413,8 +415,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.0.0-20190617215043-876b8855d23c/go.mod h1:yZAFP63pRshzrEYLXLGPmUt0Ay+2zdjmMN1loCnRLUk= -github.com/google/go-containerregistry v0.4.2-0.20210316173552-70c58c0e4786 h1:Y6e1srz/qpjdrel/H0ejC3Xm9u44u4WPAK9EDQRLZU4= -github.com/google/go-containerregistry v0.4.2-0.20210316173552-70c58c0e4786/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= +github.com/google/go-containerregistry v0.5.0 h1:eb9sinv4PKm0AUwQGov0mvIdA4pyBGjRofxN4tWnMwM= +github.com/google/go-containerregistry v0.5.0/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -537,8 +539,8 @@ github.com/k3s-io/cri v1.4.0-k3s.5/go.mod h1:fGPUUHMKQik/vIegSe05DtX/m4miovdtvVL github.com/k3s-io/cri-tools v1.21.0-k3s1/go.mod h1:Qsz54zxINPR+WVWX9Kc3CTmuDFB1dNLCNV8jE8lUbtU= github.com/k3s-io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea h1:7cwby0GoNAi8IsVrT0q+JfQpB6V76ZaEGhj6qts/mvU= github.com/k3s-io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -github.com/k3s-io/helm-controller v0.9.0 h1:ctqIjsW2qZdvpGJ7iUQlpe/DPSY4J9c9FZ9/Z/Tco9w= -github.com/k3s-io/helm-controller v0.9.0/go.mod h1:nZP8FH3KZrNNUf5r+SwwiMR63HS6lxdHdpHijgPfF74= +github.com/k3s-io/helm-controller v0.9.1 h1:qtHWTNHiuCPRbA2YZ7z7jTgSHo7Yc5He52oMga72yUk= +github.com/k3s-io/helm-controller v0.9.1/go.mod h1:nZP8FH3KZrNNUf5r+SwwiMR63HS6lxdHdpHijgPfF74= github.com/k3s-io/kine v0.6.0 h1:4l7wjgCxb2oD+7Hyf3xIhkGd/6s1sXpRFdQiyy+7Ki8= github.com/k3s-io/kine v0.6.0/go.mod h1:rzCs93+rQHZGOiewMd84PDrER92QeZ6eeHbWkfEy4+w= github.com/k3s-io/kubernetes v1.21.0-k3s1 h1:pIE2vo7wrwC7nuuislJ9PRWNL/48/Xka2z9MHcdzR7M= @@ -596,8 +598,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= -github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= @@ -774,8 +776,8 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= +github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -817,11 +819,13 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI= github.com/rancher/dynamiclistener v0.2.3 h1:FHn0Gkx+kIUqsFs3zMMR2QC9ufH/AoBLqO5zH5hbtqw= github.com/rancher/dynamiclistener v0.2.3/go.mod h1:9WusTANoiRr8cDWCTtf5txieulezHbpv4vhLADPp0zU= -github.com/rancher/k3s v1.21.1-0.20210510220633-948295e8e842 h1:9IQqFi1WpYrUM/cHbNip+eywfa1NFEIVD8M/QFRey6o= -github.com/rancher/k3s v1.21.1-0.20210510220633-948295e8e842/go.mod h1:Iw1ZUukfS3dxh70NmdO8xD3jvRaip0ZCaQ+gqphLEDo= +github.com/rancher/k3s v1.21.1-0.20210512014615-e10524a6b182 h1:fVrEh4tQrbqCj0vSMymGR+8JnQofkGf7UTfyKdOfoCw= +github.com/rancher/k3s v1.21.1-0.20210512014615-e10524a6b182/go.mod h1:/vPlXgTCxOi8BUCRyAbBgHGxza629KeiNWUWmUb5pms= github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009/go.mod h1:wpITyDPTi/Na5h73XkbuEf2AP9fbgrIGqqxVzFhYD6U= github.com/rancher/remotedialer v0.2.0 h1:xD7t3K6JYwTdAsxmGtTHQMkEkFgKouQ1foLxVW424Dc= github.com/rancher/remotedialer v0.2.0/go.mod h1:tkU8ZvrR5lRgaKWaX71nAy6daeqvPFx/lJEnbW7tXSI= +github.com/rancher/wharfie v0.3.5 h1:LIXjQNByEmO3NDh4OFb/eY2B0VxKV7K1OnABdfwDYd0= +github.com/rancher/wharfie v0.3.5/go.mod h1:cb8mSczpmw7ItbPF3K1W7crWuJLVdyV49sZZuaY4BS8= github.com/rancher/wrangler v0.1.4/go.mod h1:EYP7cqpg42YqElaCm+U9ieSrGQKAXxUH5xsr+XGpWyE= github.com/rancher/wrangler v0.4.0/go.mod h1:1cR91WLhZgkZ+U4fV9nVuXqKurWbgXcIReU4wnQvTN8= github.com/rancher/wrangler v0.6.0/go.mod h1:L4HtjPeX8iqLgsxfJgz+JjKMcX2q3qbRXSeTlC/CSd4= @@ -1084,8 +1088,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index 2151954da45..5e5e06397d5 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -1,13 +1,9 @@ package bootstrap import ( - "archive/tar" - "compress/bzip2" - "compress/gzip" "crypto/sha256" "encoding/hex" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -18,40 +14,27 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-containerregistry/pkg/v1/tarball" helmv1 "github.com/k3s-io/helm-controller/pkg/apis/helm.cattle.io/v1" - "github.com/k3s-io/helm-controller/pkg/helm" - "github.com/klauspost/compress/zstd" - "github.com/pierrec/lz4" "github.com/pkg/errors" - "github.com/rancher/k3s/pkg/agent/util" - "github.com/rancher/k3s/pkg/untar" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/daemons/agent" + daemonconfig "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/util" "github.com/rancher/rke2/pkg/images" + "github.com/rancher/wharfie/pkg/credentialprovider/plugin" + "github.com/rancher/wharfie/pkg/extract" + "github.com/rancher/wharfie/pkg/registries" + "github.com/rancher/wharfie/pkg/tarfile" "github.com/rancher/wrangler/pkg/merr" "github.com/rancher/wrangler/pkg/schemes" "github.com/sirupsen/logrus" - "github.com/urfave/cli" "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/apimachinery/pkg/util/intstr" ) var ( releasePattern = regexp.MustCompile("^v[0-9]") - flagsToValues = map[string]string{ - "cluster-cidr": "global.clusterCIDR", - "cluster-dns": "global.clusterDNS", - "cluster-domain": "global.clusterDomain", - "data-dir": "global.rke2DataDir", - "service-cidr": "global.serviceCIDR", - "system-default-registry": "global.systemDefaultRegistry", - } -) - -const ( - bufferSize = 4096 - extensionList = ".tar .tar.lz4 .tar.bz2 .tbz .tar.gz .tgz .tar.zst .tzst" // keep this in sync with the decompressor list ) // binDirForDigest returns the path to dataDir/data/refDigest/bin. @@ -96,9 +79,7 @@ func dirExists(dir string) bool { // Unique image detection is accomplished by hashing the image name and tag, or the image digest, // depending on what the runtime image reference points at. // If the bin directory already exists, or content is successfully extracted, the bin directory path is returned. -func Stage(clx *cli.Context, resolver *images.Resolver) (string, error) { - dataDir := clx.String("data-dir") - privateRegistry := clx.String("private-registry") +func Stage(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Agent) (string, error) { var img v1.Image ref, err := resolver.GetReference(images.Runtime) @@ -111,15 +92,15 @@ func Stage(clx *cli.Context, resolver *images.Resolver) (string, error) { return "", err } - refBinDir := binDirForDigest(dataDir, refDigest) - refChartsDir := chartsDirForDigest(dataDir, refDigest) - manifestsDir := manifestsDir(dataDir) + refBinDir := binDirForDigest(cfg.DataDir, refDigest) + refChartsDir := chartsDirForDigest(cfg.DataDir, refDigest) + imagesDir := imagesDir(cfg.DataDir) if dirExists(refBinDir) && dirExists(refChartsDir) { logrus.Infof("Runtime image %s bin and charts directories already exist; skipping extract", ref.Name()) } else { // Try to use configured runtime image from an airgap tarball - img, err = preloadBootstrapFromRuntime(dataDir, resolver) + img, err = preloadBootstrapFromRuntime(imagesDir, resolver) if err != nil { return "", err } @@ -127,14 +108,29 @@ func Stage(clx *cli.Context, resolver *images.Resolver) (string, error) { // If we didn't find the requested image in a tarball, pull it from the remote registry. // Note that this will fail (potentially after a long delay) if the registry cannot be reached. if img == nil { - registries, err := getPrivateRegistries(privateRegistry) + registry, err := registries.GetPrivateRegistries(nodeConfig.AgentConfig.PrivateRegistry) if err != nil { - return "", errors.Wrapf(err, "failed to load private registry configuration from %s", privateRegistry) + return "", errors.Wrapf(err, "failed to load private registry configuration from %s", nodeConfig.AgentConfig.PrivateRegistry) } - multiKeychain := authn.NewMultiKeychain(registries, authn.DefaultKeychain) + + // Prefer registries.yaml auth config + kcs := []authn.Keychain{registry} + + // Try to enable Kubelet image credential provider plugins; fall back to legacy docker credentials + if agent.ImageCredProvAvailable(&nodeConfig.AgentConfig) { + plugins, err := plugin.RegisterCredentialProviderPlugins(nodeConfig.AgentConfig.ImageCredProvConfig, nodeConfig.AgentConfig.ImageCredProvBinDir) + if err != nil { + return "", err + } + kcs = append(kcs, plugins) + } else { + kcs = append(kcs, authn.DefaultKeychain) + } + + multiKeychain := authn.NewMultiKeychain(kcs...) logrus.Infof("Pulling runtime image %s", ref.Name()) - img, err = remote.Image(registries.Rewrite(ref), remote.WithAuthFromKeychain(multiKeychain), remote.WithTransport(registries)) + img, err = remote.Image(registry.Rewrite(ref), remote.WithAuthFromKeychain(multiKeychain), remote.WithTransport(registry)) if err != nil { return "", errors.Wrapf(err, "failed to get runtime image %s", ref.Name()) } @@ -142,10 +138,10 @@ func Stage(clx *cli.Context, resolver *images.Resolver) (string, error) { // Extract binaries and charts extractPaths := map[string]string{ - "bin": refBinDir, - "charts": refChartsDir, + "/bin": refBinDir, + "/charts": refChartsDir, } - if err := extract(img, extractPaths); err != nil { + if err := extract.ExtractDirs(img, extractPaths); err != nil { return "", errors.Wrap(err, "failed to extract runtime image") } // Ensure correct permissions on bin dir @@ -154,75 +150,46 @@ func Stage(clx *cli.Context, resolver *images.Resolver) (string, error) { } } + // ignore errors on symlink rewrite + _ = os.RemoveAll(symlinkBinDir(cfg.DataDir)) + _ = os.Symlink(refBinDir, symlinkBinDir(cfg.DataDir)) + + return refBinDir, nil +} + +// UpdateManifests copies the staged manifests into the server's manifests dir, and applies +// passthrough CLI values to any HelmChart manifests found in the manifests directory. +func UpdateManifests(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { + ref, err := resolver.GetReference(images.Runtime) + if err != nil { + return err + } + + refDigest, err := releaseRefDigest(ref) + if err != nil { + return err + } + + refChartsDir := chartsDirForDigest(cfg.DataDir, refDigest) + manifestsDir := manifestsDir(cfg.DataDir) + // Ensure manifests directory exists if err := os.MkdirAll(manifestsDir, 0755); err != nil && !os.IsExist(err) { - return "", err + return err } // Recursively copy all charts into the manifests directory, since the K3s // deploy controller will delete them if they are disabled. if err := fs.CopyDir(manifestsDir, refChartsDir); err != nil { - return "", errors.Wrap(err, "failed to copy runtime charts") + return errors.Wrap(err, "failed to copy runtime charts") } // Fix up HelmCharts to pass through configured values. // This needs to be done every time in order to sync values from the CLI - if err := setChartValues(manifestsDir, clx); err != nil { - return "", errors.Wrap(err, "failed to rewrite HelmChart manifests to pass through CLI values") - } - - // ignore errors on symlink rewrite - _ = os.RemoveAll(symlinkBinDir(dataDir)) - _ = os.Symlink(refBinDir, symlinkBinDir(dataDir)) - - return refBinDir, nil -} - -// extract extracts image content to target directories using a tar interface. -// Only files within subdirectories present in the dirs map are extracted. -func extract(img v1.Image, dirs map[string]string) error { - reader := mutate.Extract(img) - defer reader.Close() - - t := tar.NewReader(reader) - for { - h, err := t.Next() - if err == io.EOF { - return nil - } else if err != nil { - return err - } - - sourceDir := filepath.Dir(h.Name) - targetDir, ok := dirs[sourceDir] - if !ok { - continue - } - - switch h.Typeflag { - case tar.TypeDir: - if err := os.MkdirAll(targetDir, 0755); err != nil { - return err - } - case tar.TypeReg: - targetName := filepath.Join(targetDir, filepath.Base(h.Name)) - logrus.Infof("Extracting file %s to %s", h.Name, targetName) - - mode := h.FileInfo().Mode() & 0755 - f, err := os.OpenFile(targetName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) - if err != nil { - return err - } - - if _, err = io.Copy(f, t); err != nil { - f.Close() - return err - } - if err := f.Close(); err != nil { - return err - } - } + if err := setChartValues(manifestsDir, nodeConfig, cfg); err != nil { + return errors.Wrap(err, "failed to rewrite HelmChart manifests to pass through CLI values") } + return nil } // releaseRefDigest returns a unique name for an image reference. @@ -247,7 +214,7 @@ func releaseRefDigest(ref name.Reference) (string, error) { // preloadBootstrapFromRuntime tries to load the runtime image from tarballs, using both the // default registry, and the user-configured registry (on the off chance they've retagged the // images in the tarball to match their private registry). -func preloadBootstrapFromRuntime(dataDir string, resolver *images.Resolver) (v1.Image, error) { +func preloadBootstrapFromRuntime(imagesDir string, resolver *images.Resolver) (v1.Image, error) { var refs []name.Reference runtimeRef, err := resolver.GetReference(images.Runtime) if err != nil { @@ -267,7 +234,7 @@ func preloadBootstrapFromRuntime(dataDir string, resolver *images.Resolver) (v1. } for _, ref := range refs { - img, err := preloadBootstrapImage(dataDir, ref) + img, err := tarfile.FindImage(imagesDir, ref) if img != nil { return img, err } @@ -278,126 +245,22 @@ func preloadBootstrapFromRuntime(dataDir string, resolver *images.Resolver) (v1. return nil, nil } -// preloadBootstrapImage attempts return an image matching the given reference from a tarball -// within imagesDir. -func preloadBootstrapImage(dataDir string, imageRef name.Reference) (v1.Image, error) { - imageTag, ok := imageRef.(name.Tag) - if !ok { - logrus.Debugf("No local image available for %s: reference is not a tag", imageRef.Name()) - return nil, nil - } - - imagesDir := imagesDir(dataDir) - if _, err := os.Stat(imagesDir); err != nil { - if os.IsNotExist(err) { - logrus.Debugf("No local image available for %s: directory %s does not exist", imageTag.Name(), imagesDir) - return nil, nil - } - return nil, err - } - - logrus.Infof("Checking local image archives in %s for %s", imagesDir, imageRef.Name()) - - // Walk the images dir to get a list of tar files - files := map[string]os.FileInfo{} - if err := filepath.Walk(imagesDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - files[path] = info - } - return nil - }); err != nil { - return nil, err - } - - // Try to find the requested tag in each file, moving on to the next if there's an error - for fileName := range files { - img, err := preloadFile(imageTag, fileName) - if img != nil { - logrus.Debugf("Found %s in %s", imageTag.Name(), fileName) - return img, nil - } - if err != nil { - logrus.Infof("Failed to check %s: %v", fileName, err) - } - } - logrus.Debugf("No local image available for %s: not found in any file in %s", imageTag.Name(), imagesDir) - return nil, nil -} - -// preloadFile handles loading images from a single tarball. -func preloadFile(imageTag name.Tag, fileName string) (v1.Image, error) { - var opener tarball.Opener - switch { - case util.HasSuffixI(fileName, ".txt"): - return nil, nil - case util.HasSuffixI(fileName, ".tar"): - opener = func() (io.ReadCloser, error) { - return os.Open(fileName) - } - case util.HasSuffixI(fileName, ".tar.lz4"): - opener = func() (io.ReadCloser, error) { - file, err := os.Open(fileName) - if err != nil { - return nil, err - } - zr := lz4.NewReader(file) - return SplitReadCloser(zr, file), nil - } - case util.HasSuffixI(fileName, ".tar.bz2", ".tbz"): - opener = func() (io.ReadCloser, error) { - file, err := os.Open(fileName) - if err != nil { - return nil, err - } - zr := bzip2.NewReader(file) - return SplitReadCloser(zr, file), nil - } - case util.HasSuffixI(fileName, ".tar.gz", ".tgz"): - opener = func() (io.ReadCloser, error) { - file, err := os.Open(fileName) - if err != nil { - return nil, err - } - zr, err := gzip.NewReader(file) - if err != nil { - return nil, err - } - return MultiReadCloser(zr, file), nil - } - case util.HasSuffixI(fileName, "tar.zst", ".tzst"): - opener = func() (io.ReadCloser, error) { - file, err := os.Open(fileName) - if err != nil { - return nil, err - } - zr, err := zstd.NewReader(file, zstd.WithDecoderMaxMemory(untar.MaxDecoderMemory)) - if err != nil { - return nil, err - } - return ZstdReadCloser(zr, file), nil - } - default: - return nil, fmt.Errorf("unhandled file type; supported extensions: " + extensionList) - } - - img, err := tarball.Image(opener, &imageTag) - if err != nil { - logrus.Debugf("Did not find %s in %s: %s", imageTag, fileName, err) - return nil, nil - } - return img, nil -} - // setChartValues scans the directory at manifestDir. It attempts to load all manifests // in that directory as HelmCharts. Any manifests that contain a HelmChart are modified to // pass through settings to both the Helm job and the chart values. // NOTE: This will probably fail if any manifest contains multiple documents. This should // not matter for any of our packaged components, but may prevent this from working on user manifests. -func setChartValues(manifestsDir string, clx *cli.Context) error { +func setChartValues(manifestsDir string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { serializer := json.NewSerializerWithOptions(json.DefaultMetaFactory, schemes.All, schemes.All, json.SerializerOptions{Yaml: true, Pretty: true, Strict: true}) + chartValues := map[string]string{ + "global.clusterCIDR": util.JoinIPNets(nodeConfig.AgentConfig.ClusterCIDRs), + "global.clusterDNS": util.JoinIPs(nodeConfig.AgentConfig.ClusterDNSs), + "global.clusterDomain": nodeConfig.AgentConfig.ClusterDomain, + "global.rke2DataDir": cfg.DataDir, + "global.serviceCIDR": util.JoinIPNets(nodeConfig.AgentConfig.ServiceCIDRs), + "global.systemDefaultRegistry": nodeConfig.AgentConfig.SystemDefaultRegistry, + } + files := map[string]os.FileInfo{} if err := filepath.Walk(manifestsDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -419,7 +282,7 @@ func setChartValues(manifestsDir string, clx *cli.Context) error { var errs []error for fileName, info := range files { - if err := rewriteChart(fileName, info, clx, serializer); err != nil { + if err := rewriteChart(fileName, info, chartValues, serializer); err != nil { errs = append(errs, err) } } @@ -429,7 +292,7 @@ func setChartValues(manifestsDir string, clx *cli.Context) error { // rewriteChart applies dataDir and systemDefaultRegistry settings to the file at fileName with associated info. // If the file cannot be decoded as a HelmChart, it is silently skipped. Any other IO error is considered // a failure. -func rewriteChart(fileName string, info os.FileInfo, clx *cli.Context, serializer *json.Serializer) error { +func rewriteChart(fileName string, info os.FileInfo, chartValues map[string]string, serializer *json.Serializer) error { chartChanged := false bytes, err := ioutil.ReadFile(fileName) @@ -458,33 +321,8 @@ func rewriteChart(fileName string, info os.FileInfo, clx *cli.Context, serialize chart.Spec.Set = map[string]intstr.IntOrString{} } - for cliKey, chartKey := range flagsToValues { - chartVal, chartSet := chart.Spec.Set[chartKey] - if clx.IsSet(cliKey) { - // if set on the CLI, ensure that the chart value matches - cliVal := clx.String(cliKey) - if chartVal.StrVal != cliVal { - chart.Spec.Set[chartKey] = intstr.FromString(cliVal) - chartChanged = true - } - } else { - // if not set on the CLI, ensure that it is not set on the chart either - if chartSet { - delete(chart.Spec.Set, chartKey) - chartChanged = true - continue - } - } - } - - jobImage := helm.DefaultJobImage - if systemDefaultRegistry := clx.String("system-default-registry"); systemDefaultRegistry != "" { - jobImage = systemDefaultRegistry + "/" + helm.DefaultJobImage - } - - if chart.Spec.JobImage != jobImage { - chart.Spec.JobImage = jobImage - chartChanged = true + for k, v := range chartValues { + chart.Spec.Set[k] = intstr.FromString(v) } if chartChanged { diff --git a/pkg/bootstrap/readcloser.go b/pkg/bootstrap/readcloser.go deleted file mode 100644 index 449c0905b82..00000000000 --- a/pkg/bootstrap/readcloser.go +++ /dev/null @@ -1,82 +0,0 @@ -package bootstrap - -import ( - "io" - - "github.com/klauspost/compress/zstd" - "github.com/rancher/wrangler/pkg/merr" -) - -// Explicit interface checks -var _ io.ReadCloser = &zstdReadCloser{} -var _ io.ReadCloser = &multiReadCloser{} -var _ io.ReadCloser = &splitReadCloser{} - -// ZstdReadCloser implements the ReadCloser interface for zstd. The zstd decompressor's Close() -// method doesn't have a return value and therefore doesn't match the ReadCloser interface, so we -// have to wrap it in our own ReadCloser that doesn't expect a return value. We also need to close -// the underlying filehandle. -func ZstdReadCloser(r *zstd.Decoder, c io.Closer) io.ReadCloser { - return zstdReadCloser{r, c} -} - -type zstdReadCloser struct { - r *zstd.Decoder - c io.Closer -} - -func (z zstdReadCloser) Read(p []byte) (int, error) { - return z.r.Read(p) -} - -func (z zstdReadCloser) Close() error { - z.r.Close() - return z.c.Close() -} - -// MultiReadCloser implements the ReadCloser interface for decompressors that need to be closed. -// Some decompressors implement a Close function that needs to be called to clean up resources or -// verify checksums, but we also need to ensure that the underlying file gets closed as well. -func MultiReadCloser(r io.ReadCloser, c io.Closer) io.ReadCloser { - return multiReadCloser{r, c} -} - -type multiReadCloser struct { - r io.ReadCloser - c io.Closer -} - -func (m multiReadCloser) Read(p []byte) (int, error) { - return m.r.Read(p) -} - -func (m multiReadCloser) Close() error { - var errs []error - if err := m.r.Close(); err != nil { - errs = append(errs, err) - } - if err := m.c.Close(); err != nil { - errs = append(errs, err) - } - return merr.NewErrors(errs...) -} - -// SplitReadCloser implements the ReadCloser interface for decompressors that don't need to be -// closed. Some decompressors don't implement a Close function, so we just need to ensure that the -// underlying file gets closed. -func SplitReadCloser(r io.Reader, c io.Closer) io.ReadCloser { - return splitReadCloser{r, c} -} - -type splitReadCloser struct { - r io.Reader - c io.Closer -} - -func (s splitReadCloser) Read(p []byte) (int, error) { - return s.r.Read(p) -} - -func (s splitReadCloser) Close() error { - return s.c.Close() -} diff --git a/pkg/bootstrap/registries.go b/pkg/bootstrap/registries.go deleted file mode 100644 index 3dbe66f12d3..00000000000 --- a/pkg/bootstrap/registries.go +++ /dev/null @@ -1,277 +0,0 @@ -package bootstrap - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/url" - "os" - "regexp" - "strings" - "time" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/pkg/errors" - "github.com/rancher/k3s/pkg/agent/templates" - "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" -) - -// registry stores information necessary to configure authentication and -// connections to remote registries, including overriding registry endpoints -type registry struct { - r *templates.Registry - t map[string]*http.Transport - w map[string]bool -} - -// Explicit interface checks -var _ authn.Keychain = ®istry{} -var _ http.RoundTripper = ®istry{} - -// getPrivateRegistries loads private registry configuration from registries.yaml -func getPrivateRegistries(path string) (*registry, error) { - registry := ®istry{ - r: &templates.Registry{}, - t: map[string]*http.Transport{}, - w: map[string]bool{}, - } - privRegistryFile, err := ioutil.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return registry, nil - } - return nil, err - } - logrus.Infof("Using registry config file at %s", path) - if err := yaml.Unmarshal(privRegistryFile, registry.r); err != nil { - return nil, err - } - return registry, nil -} - -// Rewrite applies repository rewrites to the given image reference. -func (r *registry) Rewrite(ref name.Reference) name.Reference { - host := ref.Context().RegistryStr() - rewrites := r.getRewritesForHost(host) - repository := ref.Context().RepositoryStr() - - for pattern, replace := range rewrites { - exp, err := regexp.Compile(pattern) - if err != nil { - logrus.Warnf("Failed to compile rewrite `%s` for %s", pattern, host) - continue - } - if rr := exp.ReplaceAllString(repository, replace); rr != repository { - newRepo, err := name.NewRepository(rr) - if err != nil { - logrus.Warnf("Invalid repository rewrite %s for %s", rr, host) - continue - } - if t, ok := ref.(name.Tag); ok { - t.Repository = newRepo - return t - } else if d, ok := ref.(name.Digest); ok { - d.Repository = newRepo - return d - } - } - } - - return ref -} - -// Resolve returns an authenticator for the authn.Keychain interface. The authenticator -// provides credentials to a registry by looking up configuration from mirror endpoints. -func (r *registry) Resolve(target authn.Resource) (authn.Authenticator, error) { - endpointURL, err := r.getEndpointForHost(target.RegistryStr()) - if err != nil { - return nil, err - } - return r.getAuthenticatorForHost(endpointURL.Host) -} - -// RoundTrip round-trips a HTTP request for the http.RoundTripper interface. The round-tripper -// overrides the Host in the headers and URL based on mirror endpoint configuration. It also -// configures TLS based on the endpoint's TLS config, if any. -func (r *registry) RoundTrip(req *http.Request) (*http.Response, error) { - endpointURL, err := r.getEndpointForHost(req.URL.Host) - if err != nil { - return nil, err - } - - originalURL := req.URL.String() - - // The default base path is /v2/; if a path is included in the endpoint, - // replace the /v2/ prefix from the request path with the endpoint path. - // This behavior is cribbed from containerd. - if strings.HasPrefix(req.URL.Path, "/v2/") && endpointURL.Path != "" { - req.URL.Path = endpointURL.Path + strings.TrimPrefix(req.URL.Path, "/v2/") - - // If either URL has RawPath set (due to the path including urlencoded - // characters), it also needs to be used to set the combined URL - if endpointURL.RawPath != "" || req.URL.RawPath != "" { - endpointPath := endpointURL.Path - if endpointURL.RawPath != "" { - endpointPath = endpointURL.RawPath - } - reqPath := req.URL.Path - if req.URL.RawPath != "" { - reqPath = req.URL.RawPath - } - req.URL.RawPath = endpointPath + strings.TrimPrefix(reqPath, "/v2/") - } - } - - // override request host and scheme - req.Host = endpointURL.Host - req.URL.Host = endpointURL.Host - req.URL.Scheme = endpointURL.Scheme - - if newURL := req.URL.String(); originalURL != newURL { - logrus.Debugf("Registry endpoint URL modified: %s => %s", originalURL, newURL) - } - - switch endpointURL.Scheme { - case "http": - return http.DefaultTransport.RoundTrip(req) - case "https": - // Create and cache transport if not found. - if _, ok := r.t[endpointURL.Host]; !ok { - tlsConfig, err := r.getTLSConfigForHost(endpointURL.Host) - if err != nil { - return nil, errors.Wrapf(err, "failed to get TLS config for endpoint %s", endpointURL.Host) - } - - r.t[endpointURL.Host] = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - TLSClientConfig: tlsConfig, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - } - return r.t[endpointURL.Host].RoundTrip(req) - } - return nil, fmt.Errorf("unsupported scheme %s in registry endpoint", endpointURL.Scheme) -} - -// getEndpointForHost gets endpoint configuration for a host. Because go-containerregistry's -// Keychain interface does not provide a good hook to check authentication for multiple endpoints, -// we only use the first endpoint from the mirror list. If no endpoint configuration is found, https -// and the default path are assumed. -func (r *registry) getEndpointForHost(host string) (*url.URL, error) { - keys := []string{host} - if host == name.DefaultRegistry { - keys = append(keys, "docker.io") - } - keys = append(keys, "*") - - for _, key := range keys { - if mirror, ok := r.r.Mirrors[key]; ok { - endpointCount := len(mirror.Endpoints) - switch { - case endpointCount > 1: - // Only warn about multiple endpoints once per host - if !r.w[host] { - logrus.Warnf("Found more than one endpoint for %s; only the first entry will be used", host) - r.w[host] = true - } - fallthrough - case endpointCount == 1: - return url.Parse(mirror.Endpoints[0]) - } - } - } - return url.Parse("https://" + host + "/v2/") -} - -// getAuthenticatorForHost returns an Authenticator for a given host. This should be the host from -// the mirror endpoint list, NOT the host from the request. If no configuration is present, -// Anonymous authentication is used. -func (r *registry) getAuthenticatorForHost(host string) (authn.Authenticator, error) { - if config, ok := r.r.Configs[host]; ok { - if config.Auth != nil { - return authn.FromConfig(authn.AuthConfig{ - Username: config.Auth.Username, - Password: config.Auth.Password, - Auth: config.Auth.Auth, - IdentityToken: config.Auth.IdentityToken, - }), nil - } - } - return authn.Anonymous, nil -} - -// getTLSConfigForHost returns TLS configuration for a given host. This should be the host from the -// mirror endpoint list, NOT the host from the request. This is cribbed from -// https://github.com/containerd/cri/blob/release/1.4/pkg/server/image_pull.go#L274 -func (r *registry) getTLSConfigForHost(host string) (*tls.Config, error) { - tlsConfig := &tls.Config{} - if config, ok := r.r.Configs[host]; ok { - if config.TLS != nil { - if config.TLS.CertFile != "" && config.TLS.KeyFile == "" { - return nil, errors.Errorf("cert file %q was specified, but no corresponding key file was specified", config.TLS.CertFile) - } - if config.TLS.CertFile == "" && config.TLS.KeyFile != "" { - return nil, errors.Errorf("key file %q was specified, but no corresponding cert file was specified", config.TLS.KeyFile) - } - if config.TLS.CertFile != "" && config.TLS.KeyFile != "" { - cert, err := tls.LoadX509KeyPair(config.TLS.CertFile, config.TLS.KeyFile) - if err != nil { - return nil, errors.Wrap(err, "failed to load cert file") - } - if len(cert.Certificate) != 0 { - tlsConfig.Certificates = []tls.Certificate{cert} - } - tlsConfig.BuildNameToCertificate() // nolint:staticcheck - } - - if config.TLS.CAFile != "" { - caCertPool, err := x509.SystemCertPool() - if err != nil { - return nil, errors.Wrap(err, "failed to get system cert pool") - } - caCert, err := ioutil.ReadFile(config.TLS.CAFile) - if err != nil { - return nil, errors.Wrap(err, "failed to load CA file") - } - caCertPool.AppendCertsFromPEM(caCert) - tlsConfig.RootCAs = caCertPool - } - - tlsConfig.InsecureSkipVerify = config.TLS.InsecureSkipVerify - } - } - - return tlsConfig, nil -} - -// getRewritesForHost gets the map of rewrite patterns for a given host. -func (r *registry) getRewritesForHost(host string) map[string]string { - keys := []string{host} - if host == name.DefaultRegistry { - keys = append(keys, "docker.io") - } - keys = append(keys, "*") - - for _, key := range keys { - if mirror, ok := r.r.Mirrors[key]; ok { - if len(mirror.Rewrites) > 0 { - return mirror.Rewrites - } - } - } - - return nil -} diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index b3e13ee7e7a..335d5d26457 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -18,39 +18,50 @@ var ( Usage: "(data) Folder to hold state", Default: rke2Path, }, - "token": copy, - "token-file": copy, - "disable-selinux": drop, - "node-name": copy, - "with-node-id": drop, - "node-label": copy, - "node-taint": copy, - "docker": drop, - "container-runtime-endpoint": copy, - "pause-image": drop, - "private-registry": copy, - "node-ip": copy, - "node-external-ip": copy, - "resolv-conf": copy, - "flannel-iface": drop, - "flannel-conf": drop, - "kubelet-arg": copy, - "kube-proxy-arg": drop, - "rootless": drop, - "server": copy, - "no-flannel": drop, - "cluster-secret": drop, - "protect-kernel-defaults": copy, - "snapshotter": copy, - "selinux": copy, - "lb-server-port": copy, - "airgap-extra-registry": copy, + "token": copy, + "token-file": copy, + "disable-selinux": drop, + "node-name": copy, + "with-node-id": drop, + "node-label": copy, + "node-taint": copy, + "image-credential-provider-bin-dir": copy, + "image-credential-provider-config": copy, + "docker": drop, + "container-runtime-endpoint": copy, + "pause-image": drop, + "private-registry": copy, + "node-ip": copy, + "node-external-ip": copy, + "resolv-conf": copy, + "flannel-iface": drop, + "flannel-conf": drop, + "kubelet-arg": copy, + "kube-proxy-arg": drop, + "rootless": drop, + "server": copy, + "no-flannel": drop, + "cluster-secret": drop, + "protect-kernel-defaults": copy, + "snapshotter": copy, + "selinux": copy, + "lb-server-port": copy, + "airgap-extra-registry": copy, }) + deprecatedFlags = []cli.Flag{ + &cli.StringFlag{ + Name: "system-default-registry", + Usage: "(deprecated) This flag is no longer supported on agents", + EnvVar: "RKE2_SYSTEM_DEFAULT_REGISTRY", + Hidden: true, + }, + } ) func NewAgentCommand() cli.Command { cmd := k3sAgentBase cmd.Flags = append(cmd.Flags, commonFlag...) + cmd.Flags = append(cmd.Flags, deprecatedFlags...) return cmd } diff --git a/pkg/cli/cmds/root.go b/pkg/cli/cmds/root.go index 287aa5f8ac5..d274fc048bf 100644 --- a/pkg/cli/cmds/root.go +++ b/pkg/cli/cmds/root.go @@ -22,12 +22,6 @@ var ( debug bool appName = filepath.Base(os.Args[0]) commonFlag = []cli.Flag{ - &cli.StringFlag{ - Name: "system-default-registry", - Usage: "(image) Private registry to be used for all system Docker images", - EnvVar: "RKE2_SYSTEM_DEFAULT_REGISTRY", - Destination: &config.Images.SystemDefaultRegistry, - }, &cli.StringFlag{ Name: images.KubeAPIServer, Usage: "(image) Override image to use for kube-apiserver", diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index d60a6a8d5eb..4901076f3e5 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -71,58 +71,61 @@ var ( "disable": { Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")", }, - "disable-selinux": drop, - "disable-scheduler": copy, - "disable-cloud-controller": copy, - "disable-network-policy": drop, - "disable-kube-proxy": drop, - "disable-api-server": copy, - "disable-controller-manager": copy, - "disable-etcd": copy, - "etcd-disable-snapshots": copy, - "etcd-snapshot-schedule-cron": copy, - "etcd-snapshot-retention": copy, - "etcd-snapshot-dir": copy, - "etcd-snapshot-name": copy, - "node-name": copy, - "with-node-id": drop, - "node-label": copy, - "node-taint": copy, - "docker": drop, - "container-runtime-endpoint": copy, - "pause-image": drop, - "private-registry": copy, - "node-ip": copy, - "node-external-ip": copy, - "resolv-conf": copy, - "flannel-iface": drop, - "flannel-conf": drop, - "kubelet-arg": copy, - "kube-proxy-arg": drop, - "rootless": drop, - "agent-token": copy, - "agent-token-file": copy, - "server": copy, - "secrets-encryption": copy, - "no-flannel": drop, - "no-deploy": drop, - "cluster-secret": drop, - "protect-kernel-defaults": copy, - "snapshotter": copy, - "selinux": copy, - "lb-server-port": copy, - "service-node-port-range": copy, - "etcd-expose-metrics": copy, - "airgap-extra-registry": copy, - "etcd-s3": copy, - "etcd-s3-endpoint": copy, - "etcd-s3-endpoint-ca": copy, - "etcd-s3-skip-ssl-verify": copy, - "etcd-s3-access-key": copy, - "etcd-s3-secret-key": copy, - "etcd-s3-bucket": copy, - "etcd-s3-region": copy, - "etcd-s3-folder": copy, + "disable-selinux": drop, + "disable-scheduler": copy, + "disable-cloud-controller": copy, + "disable-network-policy": drop, + "disable-kube-proxy": drop, + "disable-api-server": copy, + "disable-controller-manager": copy, + "disable-etcd": copy, + "etcd-disable-snapshots": copy, + "etcd-snapshot-schedule-cron": copy, + "etcd-snapshot-retention": copy, + "etcd-snapshot-dir": copy, + "etcd-snapshot-name": copy, + "node-name": copy, + "with-node-id": drop, + "node-label": copy, + "node-taint": copy, + "image-credential-provider-bin-dir": copy, + "image-credential-provider-config": copy, + "docker": drop, + "container-runtime-endpoint": copy, + "pause-image": drop, + "private-registry": copy, + "system-default-registry": copy, + "node-ip": copy, + "node-external-ip": copy, + "resolv-conf": copy, + "flannel-iface": drop, + "flannel-conf": drop, + "kubelet-arg": copy, + "kube-proxy-arg": drop, + "rootless": drop, + "agent-token": copy, + "agent-token-file": copy, + "server": copy, + "secrets-encryption": copy, + "no-flannel": drop, + "no-deploy": drop, + "cluster-secret": drop, + "protect-kernel-defaults": copy, + "snapshotter": copy, + "selinux": copy, + "lb-server-port": copy, + "service-node-port-range": copy, + "etcd-expose-metrics": copy, + "airgap-extra-registry": copy, + "etcd-s3": copy, + "etcd-s3-endpoint": copy, + "etcd-s3-endpoint-ca": copy, + "etcd-s3-skip-ssl-verify": copy, + "etcd-s3-access-key": copy, + "etcd-s3-secret-key": copy, + "etcd-s3-bucket": copy, + "etcd-s3-region": copy, + "etcd-s3-folder": copy, }) ) diff --git a/pkg/podexecutor/staticpod.go b/pkg/podexecutor/staticpod.go index 5ebdeccc3c9..982315f68f1 100644 --- a/pkg/podexecutor/staticpod.go +++ b/pkg/podexecutor/staticpod.go @@ -14,8 +14,10 @@ import ( "time" "github.com/rancher/k3s/pkg/cli/cmds" + daemonconfig "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/daemons/executor" "github.com/rancher/rke2/pkg/auth" + "github.com/rancher/rke2/pkg/bootstrap" "github.com/rancher/rke2/pkg/images" "github.com/rancher/rke2/pkg/staticpod" "github.com/sirupsen/logrus" @@ -46,6 +48,7 @@ type StaticPodConfig struct { AuditPolicyFile string KubeletPath string DisableETCD bool + IsServer bool } type CloudProviderConfig struct { @@ -53,6 +56,22 @@ type CloudProviderConfig struct { Path string } +// Bootstrap prepares the static executor to run components by staging the kubelet and containerd binaries. +// On servers, it also ensures that manifests are copied in to place and in sync with the system configuration. +func (s *StaticPodConfig) Bootstrap(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { + execPath, err := bootstrap.Stage(s.Resolver, nodeConfig, cfg) + if err != nil { + return err + } + if err := os.Setenv("PATH", execPath+":"+os.Getenv("PATH")); err != nil { + return err + } + if s.IsServer { + return bootstrap.UpdateManifests(s.Resolver, nodeConfig, cfg) + } + return nil +} + // Kubelet starts the kubelet in a subprocess with watching goroutine. func (s *StaticPodConfig) Kubelet(args []string) error { extraArgs := []string{ diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go index 02fa0297081..5ce800328db 100644 --- a/pkg/rke2/rke2.go +++ b/pkg/rke2/rke2.go @@ -23,7 +23,6 @@ import ( "github.com/rancher/k3s/pkg/daemons/executor" "github.com/rancher/k3s/pkg/etcd" rawServer "github.com/rancher/k3s/pkg/server" - "github.com/rancher/rke2/pkg/bootstrap" "github.com/rancher/rke2/pkg/cli/defaults" "github.com/rancher/rke2/pkg/images" "github.com/rancher/rke2/pkg/podexecutor" @@ -46,7 +45,7 @@ const ( ) func Server(clx *cli.Context, cfg Config) error { - if err := setup(clx, cfg); err != nil { + if err := setup(clx, cfg, true); err != nil { return err } @@ -80,7 +79,7 @@ func Server(clx *cli.Context, cfg Config) error { } func Agent(clx *cli.Context, cfg Config) error { - if err := setup(clx, cfg); err != nil { + if err := setup(clx, cfg, false); err != nil { return err } return agent.Run(clx) @@ -91,7 +90,7 @@ func EtcdSnapshot(clx *cli.Context, cfg Config) error { return etcdsnapshot.Run(clx) } -func setup(clx *cli.Context, cfg Config) error { +func setup(clx *cli.Context, cfg Config, isServer bool) error { dataDir := clx.String("data-dir") disableETCD := clx.Bool("disable-etcd") disableScheduler := clx.Bool("disable-scheduler") @@ -104,6 +103,7 @@ func setup(clx *cli.Context, cfg Config) error { auditPolicyFile = defaultAuditPolicyFile } + cfg.Images.SystemDefaultRegistry = clx.String("system-default-registry") resolver, err := images.NewResolver(cfg.Images) if err != nil { return err @@ -118,21 +118,6 @@ func setup(clx *cli.Context, cfg Config) error { return err } - // If system-default-registry is set, add the same value to airgap-extra-registry so that images - // imported from tarballs are tagged to appear to come from the same registry. - if cfg.Images.SystemDefaultRegistry != "" { - clx.Set("airgap-extra-registry", cfg.Images.SystemDefaultRegistry) - } - - execPath, err := bootstrap.Stage(clx, resolver) - if err != nil { - return err - } - - if err := os.Setenv("PATH", execPath+":"+os.Getenv("PATH")); err != nil { - return err - } - agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath) agentImagesDir := filepath.Join(dataDir, "agent", "images") @@ -169,6 +154,7 @@ func setup(clx *cli.Context, cfg Config) error { AuditPolicyFile: auditPolicyFile, KubeletPath: cfg.KubeletPath, DisableETCD: disableETCD, + IsServer: isServer, } executor.Set(&sp)