k3s-io · brandond · Jan 5, 2024 · Nov 30, 2023
@@ -559,6 +559,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log")
+	nodeConfig.Containerd.Registry = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "certs.d")
+	nodeConfig.Containerd.NoDefault = envInfo.ContainerdNoDefault
 	nodeConfig.Containerd.Debug = envInfo.Debug
 	applyContainerdStateAndAddress(nodeConfig)
 	applyCRIDockerdAddress(nodeConfig)

@@ -0,0 +1,109 @@
+package containerd
+
+import (
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/k3s-io/k3s/pkg/agent/templates"
+	util2 "github.com/k3s-io/k3s/pkg/agent/util"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/sirupsen/logrus"
+)
+
+// writeContainerdConfig renders and saves config.toml from the filled template
+func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	var containerdTemplate string
+	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
+	if err == nil {
+		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
+		containerdTemplate = string(containerdTemplateBytes)
+	} else if os.IsNotExist(err) {
+		containerdTemplate = templates.ContainerdConfigTemplate
+	} else {
+		return err
+	}
+	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
+	if err != nil {
+		return err
+	}
+
+	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+}
+
+// writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
+func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	registry := containerdConfig.PrivateRegistryConfig
+	hosts := map[string]templates.HostConfig{}
+
+	for host, mirror := range registry.Mirrors {
+		defaultHost, _ := docker.DefaultHost(host)
+		config := templates.HostConfig{
+			Host:    defaultHost,
+			Program: version.Program,
+		}
+		if host == "*" {
+			host = "_default"
+			config.Host = ""
+		} else if containerdConfig.NoDefaultEndpoint {
+			config.Host = ""
+		}
+		// TODO: rewrites are currently copied from the mirror settings into each endpoint.
+		// In the future, we should allow for per-endpoint rewrites, instead of expecting
+		// all mirrors to have the same structure. This will require changes to the registries.yaml
+		// structure, which is defined in rancher/wharfie.
+		for _, endpoint := range mirror.Endpoints {
+			if endpointURL, err := url.Parse(endpoint); err == nil {
+				config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
+					OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
+					Config:       registry.Configs[endpointURL.Host],
+					Rewrites:     mirror.Rewrites,
+					URI:          endpoint,
+				})
+			}
+		}
+		hosts[host] = config
+	}
+
+	for host, registry := range registry.Configs {
+		config, ok := hosts[host]
+		if !ok {
+			config = templates.HostConfig{
+				Program: version.Program,
+			}
+		}
+		if len(config.Endpoints) == 0 {
+			config.Endpoints = []templates.RegistryEndpoint{
+				{
+					Config: registry,
+					URI:    "https://" + host,
+				},
+			}
+		}
+		hosts[host] = config
+	}
+
+	// Clean up previous configuration templates
+	os.RemoveAll(cfg.Containerd.Registry)
+
+	// Write out new templates
+	for host, config := range hosts {
+		hostDir := filepath.Join(cfg.Containerd.Registry, host)
+		hostsFile := filepath.Join(hostDir, "hosts.toml")
+		hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
+		if err != nil {
+			return err
+		}
+		if err := os.MkdirAll(hostDir, 0700); err != nil {
+			return err
+		}
+		if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
@@ -13,7 +13,6 @@ import (
 	stargz "github.com/containerd/stargz-snapshotter/service"
 	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/k3s-io/k3s/pkg/agent/templates"
-	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/cgroups"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -67,7 +66,6 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime)
 	}
 
-	var containerdTemplate string
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         disableCgroup,
@@ -77,6 +75,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		PrivateRegistryConfig: privRegistries.Registry,
 		ExtraRuntimes:         extraRuntimes,
 		Program:               version.Program,
+		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}
 
 	selEnabled, selConfigured, err := selinuxStatus()
@@ -90,21 +89,11 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		logrus.Warnf("SELinux is enabled for "+version.Program+" but process is not running in context '%s', "+version.Program+"-selinux policy may need to be applied", SELinuxContextType)
 	}
 
-	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
-	if err == nil {
-		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
-		containerdTemplate = string(containerdTemplateBytes)
-	} else if os.IsNotExist(err) {
-		containerdTemplate = templates.ContainerdConfigTemplate
-	} else {
-		return err
-	}
-	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
-	if err != nil {
+	if err := writeContainerdConfig(cfg, containerdConfig); err != nil {
 		return err
 	}
 
-	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+	return writeContainerdHosts(cfg, containerdConfig)
 }
 
 func Client(address string) (*containerd.Client, error) {

@@ -5,11 +5,9 @@ package containerd
 
 import (
 	"context"
-	"os"
 
 	"github.com/containerd/containerd"
 	"github.com/k3s-io/k3s/pkg/agent/templates"
-	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	util3 "github.com/k3s-io/k3s/pkg/util"
 	"github.com/pkg/errors"
@@ -38,31 +36,20 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		logrus.Warn("SELinux isn't supported on windows")
 	}
 
-	var containerdTemplate string
-
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         true,
 		SystemdCgroup:         false,
 		IsRunningInUserNS:     false,
 		PrivateRegistryConfig: privRegistries.Registry,
+		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}
 
-	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
-	if err == nil {
-		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
-		containerdTemplate = string(containerdTemplateBytes)
-	} else if os.IsNotExist(err) {
-		containerdTemplate = templates.ContainerdConfigTemplate
-	} else {
-		return err
-	}
-	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
-	if err != nil {
+	if err := writeContainerdConfig(cfg, containerdConfig); err != nil {
 		return err
 	}
 
-	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+	return writeContainerdHosts(cfg, containerdConfig)
 }
 
 func Client(address string) (*containerd.Client, error) {

@@ -1,6 +1,9 @@
 package templates
 
 import (
+	"bytes"
+	"text/template"
+
 	"github.com/rancher/wharfie/pkg/registries"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
@@ -17,7 +20,71 @@ type ContainerdConfig struct {
 	SystemdCgroup         bool
 	IsRunningInUserNS     bool
 	EnableUnprivileged    bool
+	NoDefaultEndpoint     bool
 	PrivateRegistryConfig *registries.Registry
 	ExtraRuntimes         map[string]ContainerdRuntimeConfig
 	Program               string
 }
+
+type RegistryEndpoint struct {
+	OverridePath bool
+	URI          string
+	Rewrites     map[string]string
+	Config       registries.RegistryConfig
+}
+
+type HostConfig struct {
+	Host      string
+	Program   string
+	Endpoints []RegistryEndpoint
+}
+
+const HostsTomlTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT.
+{{ if .Host }}server = "https://{{ .Host }}"{{ end }}
+
+{{ range $e := .Endpoints -}}
+[host."{{ $e.URI }}"]
+  capabilities = ["pull", "resolve"]
+  {{- if $e.OverridePath }}
+  override_path = true
+  {{- end }}
+{{- if $e.Config.TLS }}
+  {{- if $e.Config.TLS.CAFile }}
+  ca = [{{ printf "%q" $e.Config.TLS.CAFile }}]
+  {{- end }}
+  {{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }}
+  client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]]
+  {{- end }}
+  {{- if $e.Config.TLS.InsecureSkipVerify }}
+  skip_verify = true
+  {{- end }}
+{{ end }}
+{{- if $e.Rewrites }}
+  [host."{{ $e.URI }}".rewrite]
+  {{- range $pattern, $replace := $e.Rewrites }}
+    "{{ $pattern }}" = "{{ $replace }}"
+  {{- end }}
+{{ end }}
+{{ end -}}
+`
+
+func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
@@ -3,11 +3,11 @@
 package templates
 
 import (
-	"bytes"
 	"text/template"
 )
 
 const ContainerdConfigTemplate = `
+{{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
 version = 2
 
@@ -95,20 +95,10 @@ enable_keychain = true
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
   SystemdCgroup = {{ .SystemdCgroup }}
 
-{{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{if $v.Rewrites}}
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite]
-{{range $pattern, $replace := $v.Rewrites}}
-    "{{$pattern}}" = "{{$replace}}"
-{{end}}
-{{end}}
-{{end}}
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
 
+{{ if .PrivateRegistryConfig }}
 {{range $k, $v := .PrivateRegistryConfig.Configs }}
 {{ if $v.Auth }}
 [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth]
@@ -117,13 +107,6 @@ enable_keychain = true
   {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
   {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
 {{end}}
-{{ if $v.TLS }}
-[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
-{{end}}
 {{end}}
 {{end}}
 
@@ -136,12 +119,9 @@ enable_keychain = true
 {{end}}
 `
 
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
-	out := new(bytes.Buffer)
-	t := template.Must(template.New("compiled_template").Parse(templateBuffer))
-	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
-	if err := t.Execute(out, config); err != nil {
-		return "", err
-	}
-	return out.String(), nil
+// Linux config templates do not need fixups
+var templateFuncs = template.FuncMap{
+	"deschemify": func(s string) string {
+		return s
+	},
 }