diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 47e625b2f8d5..e8df928c1462 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -559,6 +559,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N } nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd") nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log") + nodeConfig.Containerd.Registry = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "certs.d") + nodeConfig.Containerd.NoDefault = envInfo.ContainerdNoDefault nodeConfig.Containerd.Debug = envInfo.Debug applyContainerdStateAndAddress(nodeConfig) applyCRIDockerdAddress(nodeConfig) diff --git a/pkg/agent/containerd/config.go b/pkg/agent/containerd/config.go new file mode 100644 index 000000000000..9f7358470b83 --- /dev/null +++ b/pkg/agent/containerd/config.go @@ -0,0 +1,109 @@ +package containerd + +import ( + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/containerd/containerd/remotes/docker" + "github.com/k3s-io/k3s/pkg/agent/templates" + util2 "github.com/k3s-io/k3s/pkg/agent/util" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/version" + "github.com/sirupsen/logrus" +) + +// writeContainerdConfig renders and saves config.toml from the filled template +func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error { + var containerdTemplate string + containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template) + if err == nil { + logrus.Infof("Using containerd template at %s", cfg.Containerd.Template) + containerdTemplate = string(containerdTemplateBytes) + } else if os.IsNotExist(err) { + containerdTemplate = templates.ContainerdConfigTemplate + } else { + return err + } + parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig) + if err != nil { + return err + } + + return util2.WriteFile(cfg.Containerd.Config, parsedTemplate) +} + +// writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template +func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error { + registry := containerdConfig.PrivateRegistryConfig + hosts := map[string]templates.HostConfig{} + + for host, mirror := range registry.Mirrors { + defaultHost, _ := docker.DefaultHost(host) + config := templates.HostConfig{ + Host: defaultHost, + Program: version.Program, + } + if host == "*" { + host = "_default" + config.Host = "" + } else if containerdConfig.NoDefaultEndpoint { + config.Host = "" + } + // TODO: rewrites are currently copied from the mirror settings into each endpoint. + // In the future, we should allow for per-endpoint rewrites, instead of expecting + // all mirrors to have the same structure. This will require changes to the registries.yaml + // structure, which is defined in rancher/wharfie. + for _, endpoint := range mirror.Endpoints { + if endpointURL, err := url.Parse(endpoint); err == nil { + config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{ + OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"), + Config: registry.Configs[endpointURL.Host], + Rewrites: mirror.Rewrites, + URI: endpoint, + }) + } + } + hosts[host] = config + } + + for host, registry := range registry.Configs { + config, ok := hosts[host] + if !ok { + config = templates.HostConfig{ + Program: version.Program, + } + } + if len(config.Endpoints) == 0 { + config.Endpoints = []templates.RegistryEndpoint{ + { + Config: registry, + URI: "https://" + host, + }, + } + } + hosts[host] = config + } + + // Clean up previous configuration templates + os.RemoveAll(cfg.Containerd.Registry) + + // Write out new templates + for host, config := range hosts { + hostDir := filepath.Join(cfg.Containerd.Registry, host) + hostsFile := filepath.Join(hostDir, "hosts.toml") + hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config) + if err != nil { + return err + } + if err := os.MkdirAll(hostDir, 0700); err != nil { + return err + } + if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index 4a5cd21dfead..5750844c0ccf 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -13,7 +13,6 @@ import ( stargz "github.com/containerd/stargz-snapshotter/service" "github.com/docker/docker/pkg/parsers/kernel" "github.com/k3s-io/k3s/pkg/agent/templates" - util2 "github.com/k3s-io/k3s/pkg/agent/util" "github.com/k3s-io/k3s/pkg/cgroups" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/version" @@ -67,7 +66,6 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime) } - var containerdTemplate string containerdConfig := templates.ContainerdConfig{ NodeConfig: cfg, DisableCgroup: disableCgroup, @@ -77,6 +75,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { PrivateRegistryConfig: privRegistries.Registry, ExtraRuntimes: extraRuntimes, Program: version.Program, + NoDefaultEndpoint: cfg.Containerd.NoDefault, } selEnabled, selConfigured, err := selinuxStatus() @@ -90,21 +89,11 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { logrus.Warnf("SELinux is enabled for "+version.Program+" but process is not running in context '%s', "+version.Program+"-selinux policy may need to be applied", SELinuxContextType) } - containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template) - if err == nil { - logrus.Infof("Using containerd template at %s", cfg.Containerd.Template) - containerdTemplate = string(containerdTemplateBytes) - } else if os.IsNotExist(err) { - containerdTemplate = templates.ContainerdConfigTemplate - } else { - return err - } - parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig) - if err != nil { + if err := writeContainerdConfig(cfg, containerdConfig); err != nil { return err } - return util2.WriteFile(cfg.Containerd.Config, parsedTemplate) + return writeContainerdHosts(cfg, containerdConfig) } func Client(address string) (*containerd.Client, error) { diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go index 6efbb7a14836..f4826391e8b9 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -5,11 +5,9 @@ package containerd import ( "context" - "os" "github.com/containerd/containerd" "github.com/k3s-io/k3s/pkg/agent/templates" - util2 "github.com/k3s-io/k3s/pkg/agent/util" "github.com/k3s-io/k3s/pkg/daemons/config" util3 "github.com/k3s-io/k3s/pkg/util" "github.com/pkg/errors" @@ -38,31 +36,20 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error { logrus.Warn("SELinux isn't supported on windows") } - var containerdTemplate string - containerdConfig := templates.ContainerdConfig{ NodeConfig: cfg, DisableCgroup: true, SystemdCgroup: false, IsRunningInUserNS: false, PrivateRegistryConfig: privRegistries.Registry, + NoDefaultEndpoint: cfg.Containerd.NoDefault, } - containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template) - if err == nil { - logrus.Infof("Using containerd template at %s", cfg.Containerd.Template) - containerdTemplate = string(containerdTemplateBytes) - } else if os.IsNotExist(err) { - containerdTemplate = templates.ContainerdConfigTemplate - } else { - return err - } - parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig) - if err != nil { + if err := writeContainerdConfig(cfg, containerdConfig); err != nil { return err } - return util2.WriteFile(cfg.Containerd.Config, parsedTemplate) + return writeContainerdHosts(cfg, containerdConfig) } func Client(address string) (*containerd.Client, error) { diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index a9317ea8f13c..d920693d35ce 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -1,6 +1,9 @@ package templates import ( + "bytes" + "text/template" + "github.com/rancher/wharfie/pkg/registries" "github.com/k3s-io/k3s/pkg/daemons/config" @@ -17,7 +20,71 @@ type ContainerdConfig struct { SystemdCgroup bool IsRunningInUserNS bool EnableUnprivileged bool + NoDefaultEndpoint bool PrivateRegistryConfig *registries.Registry ExtraRuntimes map[string]ContainerdRuntimeConfig Program string } + +type RegistryEndpoint struct { + OverridePath bool + URI string + Rewrites map[string]string + Config registries.RegistryConfig +} + +type HostConfig struct { + Host string + Program string + Endpoints []RegistryEndpoint +} + +const HostsTomlTemplate = ` +{{- /* */ -}} +# File generated by {{ .Program }}. DO NOT EDIT. +{{ if .Host }}server = "https://{{ .Host }}"{{ end }} + +{{ range $e := .Endpoints -}} +[host."{{ $e.URI }}"] + capabilities = ["pull", "resolve"] + {{- if $e.OverridePath }} + override_path = true + {{- end }} +{{- if $e.Config.TLS }} + {{- if $e.Config.TLS.CAFile }} + ca = [{{ printf "%q" $e.Config.TLS.CAFile }}] + {{- end }} + {{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }} + client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]] + {{- end }} + {{- if $e.Config.TLS.InsecureSkipVerify }} + skip_verify = true + {{- end }} +{{ end }} +{{- if $e.Rewrites }} + [host."{{ $e.URI }}".rewrite] + {{- range $pattern, $replace := $e.Rewrites }} + "{{ $pattern }}" = "{{ $replace }}" + {{- end }} +{{ end }} +{{ end -}} +` + +func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { + out := new(bytes.Buffer) + t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer)) + template.Must(t.New("base").Parse(ContainerdConfigTemplate)) + if err := t.Execute(out, config); err != nil { + return "", err + } + return out.String(), nil +} + +func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { + out := new(bytes.Buffer) + t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer)) + if err := t.Execute(out, config); err != nil { + return "", err + } + return out.String(), nil +} diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index 66f8ee108084..0df107abaae7 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -3,11 +3,11 @@ package templates import ( - "bytes" "text/template" ) const ContainerdConfigTemplate = ` +{{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. version = 2 @@ -95,20 +95,10 @@ enable_keychain = true [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] SystemdCgroup = {{ .SystemdCgroup }} -{{ if .PrivateRegistryConfig }} -{{ if .PrivateRegistryConfig.Mirrors }} -[plugins."io.containerd.grpc.v1.cri".registry.mirrors]{{end}} -{{range $k, $v := .PrivateRegistryConfig.Mirrors }} -[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"] - endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] -{{if $v.Rewrites}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite] -{{range $pattern, $replace := $v.Rewrites}} - "{{$pattern}}" = "{{$replace}}" -{{end}} -{{end}} -{{end}} +[plugins."io.containerd.grpc.v1.cri".registry] + config_path = "{{ .NodeConfig.Containerd.Registry }}" +{{ if .PrivateRegistryConfig }} {{range $k, $v := .PrivateRegistryConfig.Configs }} {{ if $v.Auth }} [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth] @@ -117,13 +107,6 @@ enable_keychain = true {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} {{end}} -{{ if $v.TLS }} -[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".tls] - {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} - {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} - {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} - {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}} -{{end}} {{end}} {{end}} @@ -136,12 +119,9 @@ enable_keychain = true {{end}} ` -func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { - out := new(bytes.Buffer) - t := template.Must(template.New("compiled_template").Parse(templateBuffer)) - template.Must(t.New("base").Parse(ContainerdConfigTemplate)) - if err := t.Execute(out, config); err != nil { - return "", err - } - return out.String(), nil +// Linux config templates do not need fixups +var templateFuncs = template.FuncMap{ + "deschemify": func(s string) string { + return s + }, } diff --git a/pkg/agent/templates/templates_windows.go b/pkg/agent/templates/templates_windows.go index f52b5edbd7f0..5cccd7e43d08 100644 --- a/pkg/agent/templates/templates_windows.go +++ b/pkg/agent/templates/templates_windows.go @@ -4,16 +4,17 @@ package templates import ( - "bytes" "net/url" "strings" "text/template" ) const ContainerdConfigTemplate = ` +{{- /* */ -}} +# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. version = 2 -root = "{{ replace .NodeConfig.Containerd.Root }}" -state = "{{ replace .NodeConfig.Containerd.State }}" +root = {{ printf "%q" .NodeConfig.Containerd.Root }} +state = {{ printf "%q" .NodeConfig.Containerd.State }} plugin_dir = "" disabled_plugins = [] required_plugins = [] @@ -107,14 +108,15 @@ oom_score = 0 privileged_without_host_devices = false base_runtime_spec = "" [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "{{ replace .NodeConfig.AgentConfig.CNIBinDir }}" - conf_dir = "{{ replace .NodeConfig.AgentConfig.CNIConfDir }}" + bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }} + conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }} max_conf_num = 1 conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "" + config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }} + + {{ if .PrivateRegistryConfig }} {{range $k, $v := .PrivateRegistryConfig.Configs }} - [plugins."io.containerd.grpc.v1.cri".registry.auths] {{ if $v.Auth }} [plugins."io.containerd.grpc.v1.cri".registry.configs.auth."{{$k}}"] {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}} @@ -122,35 +124,15 @@ oom_score = 0 {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} {{end}} - [plugins."io.containerd.grpc.v1.cri".registry.configs] - {{ if $v.TLS }} - [plugins."io.containerd.grpc.v1.cri".registry.configs.tls."{{$k}}"] - {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}} - {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}} - {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}} - {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}} - {{end}} {{end}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - {{ if .PrivateRegistryConfig.Mirrors }} - {{range $k, $v := .PrivateRegistryConfig.Mirrors }} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"] - endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}] - {{if $v.Rewrites}} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite] - {{range $pattern, $replace := $v.Rewrites}} - "{{$pattern}}" = "{{$replace}}" - {{end}} - {{end}} {{end}} - {{end}} [plugins."io.containerd.grpc.v1.cri".image_decryption] key_model = "" [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] tls_cert_file = "" tls_key_file = "" [plugins."io.containerd.internal.v1.opt"] - path = "{{ replace .NodeConfig.Containerd.Opt }}" + path = {{ printf "%q" .NodeConfig.Containerd.Opt }} [plugins."io.containerd.internal.v1.restart"] interval = "10s" [plugins."io.containerd.metadata.v1.bolt"] @@ -161,27 +143,16 @@ oom_score = 0 default = ["windows", "windows-lcow"] ` -func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { - out := new(bytes.Buffer) - funcs := template.FuncMap{ - "replace": func(s string) string { - return strings.ReplaceAll(s, "\\", "\\\\") - }, - "deschemify": func(s string) string { - if strings.HasPrefix(s, "npipe:") { - u, err := url.Parse(s) - if err != nil { - return "" - } - return u.Path +// Windows config templates need named pipe addresses fixed up +var templateFuncs = template.FuncMap{ + "deschemify": func(s string) string { + if strings.HasPrefix(s, "npipe:") { + u, err := url.Parse(s) + if err != nil { + return "" } - return s - }, - } - t := template.Must(template.New("compiled_template").Funcs(funcs).Parse(templateBuffer)) - template.Must(t.New("base").Parse(ContainerdConfigTemplate)) - if err := t.Execute(out, config); err != nil { - return "", err - } - return out.String(), nil + return u.Path + } + return s + }, } diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index b36f22fe9db9..83c04d84ff6d 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -26,6 +26,7 @@ type Agent struct { PauseImage string Snapshotter string Docker bool + ContainerdNoDefault bool ContainerRuntimeEndpoint string DefaultRuntime string ImageServiceEndpoint string @@ -220,6 +221,11 @@ var ( Usage: "(agent/networking) (experimental) Disable the agent's client-side load-balancer and connect directly to the configured server address", Destination: &AgentConfig.DisableLoadBalancer, } + DisableDefaultRegistryEndpointFlag = &cli.BoolFlag{ + Name: "disable-default-registry-endpoint", + Usage: "(agent/containerd) Disables containerd's fallback default registry endpoint when a mirror is configured for that registry", + Destination: &AgentConfig.ContainerdNoDefault, + } ) func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { @@ -269,6 +275,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { PauseImageFlag, SnapshotterFlag, PrivateRegistryFlag, + DisableDefaultRegistryEndpointFlag, AirgapExtraRegistryFlag, NodeIPFlag, NodeExternalIPFlag, diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index b8330d629b2c..a79ece175144 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -493,6 +493,7 @@ var ServerFlags = []cli.Flag{ CRIEndpointFlag, DefaultRuntimeFlag, ImageServiceEndpointFlag, + DisableDefaultRegistryEndpointFlag, PauseImageFlag, SnapshotterFlag, PrivateRegistryFlag, diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 5a7fbcf050af..00204a787f30 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -67,6 +67,8 @@ type Containerd struct { Template string BlockIOConfig string RDTConfig string + Registry string + NoDefault bool SELinux bool Debug bool }