diff --git a/cmd/clusterctl/client/cluster/assets/topology-test/existing-my-cluster.yaml b/cmd/clusterctl/client/cluster/assets/topology-test/existing-my-cluster.yaml index 0e1b816fd68b..7864e8f4deec 100644 --- a/cmd/clusterctl/client/cluster/assets/topology-test/existing-my-cluster.yaml +++ b/cmd/clusterctl/client/cluster/assets/topology-test/existing-my-cluster.yaml @@ -31,6 +31,11 @@ spec: controlPlane: metadata: {} replicas: 1 + workers: + machinePools: + - class: "default-worker" + name: "mp-0" + replicas: 1 --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerCluster diff --git a/cmd/clusterctl/client/cluster/assets/topology-test/mock-CRDs.yaml b/cmd/clusterctl/client/cluster/assets/topology-test/mock-CRDs.yaml index 20fdfc1aade8..bfa58b2797c2 100644 --- a/cmd/clusterctl/client/cluster/assets/topology-test/mock-CRDs.yaml +++ b/cmd/clusterctl/client/cluster/assets/topology-test/mock-CRDs.yaml @@ -32,8 +32,24 @@ metadata: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + labels: + cluster.x-k8s.io/provider: control-plane-kubeadm + cluster.x-k8s.io/v1beta1: v1beta1 + name: kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-docker + cluster.x-k8s.io/v1beta1: v1beta1 + name: dockermachinetemplates.infrastructure.cluster.x-k8s.io +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: labels: cluster.x-k8s.io/provider: infrastructure-docker cluster.x-k8s.io/v1beta1: v1beta1 - name: dockermachinetemplates.infrastructure.cluster.x-k8s.io \ No newline at end of file + name: dockermachinepooltemplates.infrastructure.cluster.x-k8s.io \ No newline at end of file diff --git a/cmd/clusterctl/client/cluster/assets/topology-test/modified-CP-dockermachinepooltemplate.yaml b/cmd/clusterctl/client/cluster/assets/topology-test/modified-CP-dockermachinepooltemplate.yaml new file mode 100644 index 000000000000..26d1f88532ca --- /dev/null +++ b/cmd/clusterctl/client/cluster/assets/topology-test/modified-CP-dockermachinepooltemplate.yaml @@ -0,0 +1,14 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachinePoolTemplate +metadata: + name: "docker-worker-machinepooltemplate" + namespace: default +spec: + template: + metadata: + labels: + docker-machinepool-template: test-template-worker + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" diff --git a/cmd/clusterctl/client/cluster/assets/topology-test/my-cluster-class.yaml b/cmd/clusterctl/client/cluster/assets/topology-test/my-cluster-class.yaml index 5f580a85f425..b8a40a026608 100644 --- a/cmd/clusterctl/client/cluster/assets/topology-test/my-cluster-class.yaml +++ b/cmd/clusterctl/client/cluster/assets/topology-test/my-cluster-class.yaml @@ -22,6 +22,22 @@ spec: kind: DockerClusterTemplate name: my-cluster namespace: default + workers: + machinePools: + - class: "default-worker" + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: docker-worker-bootstraptemplate + namespace: default + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachinePoolTemplate + name: docker-worker-machinepooltemplate + namespace: default --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerClusterTemplate @@ -69,3 +85,26 @@ spec: extraMounts: - containerPath: "/var/run/docker.sock" hostPath: "/var/run/docker.sock" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachinePoolTemplate +metadata: + name: "docker-worker-machinepooltemplate" + namespace: default +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "docker-worker-bootstraptemplate" + namespace: default +spec: + template: + spec: + joinConfiguration: + nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use. diff --git a/cmd/clusterctl/client/cluster/assets/topology-test/new-clusterclass-and-cluster.yaml b/cmd/clusterctl/client/cluster/assets/topology-test/new-clusterclass-and-cluster.yaml index 1473bdb34d4e..fdcf5f56bcd6 100644 --- a/cmd/clusterctl/client/cluster/assets/topology-test/new-clusterclass-and-cluster.yaml +++ b/cmd/clusterctl/client/cluster/assets/topology-test/new-clusterclass-and-cluster.yaml @@ -62,6 +62,31 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerMachineTemplate name: docker-worker-machinetemplate + machinePools: + - class: "default-worker" + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: docker-worker-bootstraptemplate + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachinePoolTemplate + name: docker-worker-machinepooltemplate + - class: "default-worker-2" + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: docker-worker-bootstraptemplate + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachinePoolTemplate + name: docker-worker-machinepooltemplate --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: DockerClusterTemplate @@ -123,6 +148,17 @@ spec: preLoadImages: - gcr.io/kakaraparthy-devel/kindest/kindnetd:0.5.4 --- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachinePoolTemplate +metadata: + name: "docker-worker-machinepooltemplate" + namespace: default +spec: + template: + spec: + preLoadImages: + - gcr.io/kakaraparthy-devel/kindest/kindnetd:0.5.4 +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: @@ -163,3 +199,10 @@ spec: - class: "default-worker" name: "md-1" replicas: 1 + machinePools: + - class: "default-worker" + name: "mp-0" + replicas: 1 + - class: "default-worker" + name: "mp-1" + replicas: 1 diff --git a/cmd/clusterctl/client/cluster/topology_test.go b/cmd/clusterctl/client/cluster/topology_test.go index ac8a88903cab..f9994ccad051 100644 --- a/cmd/clusterctl/client/cluster/topology_test.go +++ b/cmd/clusterctl/client/cluster/topology_test.go @@ -52,10 +52,14 @@ var ( //go:embed assets/topology-test/modified-my-cluster.yaml modifiedMyClusterYAML []byte - // modifiedDockerMachineTemplateYAML adds metadat to the docker machine used by the control plane template.. + // modifiedDockerMachineTemplateYAML adds metadata to the docker machine used by the control plane template.. //go:embed assets/topology-test/modified-CP-dockermachinetemplate.yaml modifiedDockerMachineTemplateYAML []byte + // modifiedDockerMachinePoolTemplateYAML adds metadata to the docker machine pool used by the control plane template.. + //go:embed assets/topology-test/modified-CP-dockermachinepooltemplate.yaml + modifiedDockerMachinePoolTemplateYAML []byte + //go:embed assets/topology-test/objects-in-different-namespaces.yaml objsInDifferentNamespacesYAML []byte ) @@ -97,11 +101,15 @@ func Test_topologyClient_Plan(t *testing.T) { {kind: "DockerMachineTemplate", namespace: "default", namePrefix: "my-cluster-md-0-"}, {kind: "DockerMachineTemplate", namespace: "default", namePrefix: "my-cluster-md-1-"}, {kind: "DockerMachineTemplate", namespace: "default", namePrefix: "my-cluster-"}, + {kind: "DockerMachinePool", namespace: "default", namePrefix: "my-cluster-mp-0-"}, + {kind: "DockerMachinePool", namespace: "default", namePrefix: "my-cluster-mp-1-"}, {kind: "KubeadmConfigTemplate", namespace: "default", namePrefix: "my-cluster-md-0-"}, {kind: "KubeadmConfigTemplate", namespace: "default", namePrefix: "my-cluster-md-1-"}, {kind: "KubeadmControlPlane", namespace: "default", namePrefix: "my-cluster-"}, {kind: "MachineDeployment", namespace: "default", namePrefix: "my-cluster-md-0-"}, {kind: "MachineDeployment", namespace: "default", namePrefix: "my-cluster-md-1-"}, + {kind: "MachinePool", namespace: "default", namePrefix: "my-cluster-mp-0-"}, + {kind: "MachinePool", namespace: "default", namePrefix: "my-cluster-mp-1-"}, }, modified: []item{ {kind: "Cluster", namespace: "default", namePrefix: "my-cluster"}, @@ -205,6 +213,35 @@ func Test_topologyClient_Plan(t *testing.T) { }, wantErr: false, }, + { + name: "Modifying an existing DockerMachinePoolTemplate. Affects multiple clusters. Target Cluster not specified.", + existingObjects: mustToUnstructured( + mockCRDsYAML, + existingMyClusterClassYAML, + existingMyClusterYAML, + existingMySecondClusterYAML, + ), + args: args{ + in: &TopologyPlanInput{ + Objs: mustToUnstructured(modifiedDockerMachinePoolTemplateYAML), + }, + }, + want: out{ + affectedClusters: func() []client.ObjectKey { + cluster := client.ObjectKey{Namespace: "default", Name: "my-cluster"} + cluster2 := client.ObjectKey{Namespace: "default", Name: "my-second-cluster"} + return []client.ObjectKey{cluster, cluster2} + }(), + affectedClusterClasses: func() []client.ObjectKey { + cc := client.ObjectKey{Namespace: "default", Name: "my-cluster-class"} + return []client.ObjectKey{cc} + }(), + modified: []item{}, + created: []item{}, + reconciledCluster: nil, + }, + wantErr: false, + }, { name: "Modifying an existing DockerMachineTemplate. Affects multiple clusters. Target Cluster specified.", existingObjects: mustToUnstructured( @@ -241,6 +278,37 @@ func Test_topologyClient_Plan(t *testing.T) { }, wantErr: false, }, + { + name: "Modifying an existing DockerMachinePoolTemplate. Affects multiple clusters. Target Cluster specified.", + existingObjects: mustToUnstructured( + mockCRDsYAML, + existingMyClusterClassYAML, + existingMyClusterYAML, + existingMySecondClusterYAML, + ), + args: args{ + in: &TopologyPlanInput{ + Objs: mustToUnstructured(modifiedDockerMachinePoolTemplateYAML), + TargetClusterName: "my-cluster", + }, + }, + want: out{ + affectedClusters: func() []client.ObjectKey { + cluster := client.ObjectKey{Namespace: "default", Name: "my-cluster"} + cluster2 := client.ObjectKey{Namespace: "default", Name: "my-second-cluster"} + return []client.ObjectKey{cluster, cluster2} + }(), + affectedClusterClasses: func() []client.ObjectKey { + cc := client.ObjectKey{Namespace: "default", Name: "my-cluster-class"} + return []client.ObjectKey{cc} + }(), + created: []item{ + {kind: "DockerMachinePool", namespace: "default", namePrefix: "my-cluster-"}, + }, + reconciledCluster: &client.ObjectKey{Namespace: "default", Name: "my-cluster"}, + }, + wantErr: false, + }, { name: "Input with objects in different namespaces should return error", args: args{ diff --git a/test/extension/handlers/topologymutation/handler_test.go b/test/extension/handlers/topologymutation/handler_test.go index 1b693ee064f3..faf3f9fd6771 100644 --- a/test/extension/handlers/topologymutation/handler_test.go +++ b/test/extension/handlers/topologymutation/handler_test.go @@ -34,6 +34,7 @@ import ( controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" + infraexpv1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/exp/api/v1beta1" ) var ( @@ -204,7 +205,7 @@ func Test_patchKubeadmConfigTemplate(t *testing.T) { expectedErr bool }{ { - name: "fails if builtin.machineDeployment.class is not set", + name: "fails if builtin variable is not set", template: &bootstrapv1.KubeadmConfigTemplate{}, variables: nil, expectedTemplate: &bootstrapv1.KubeadmConfigTemplate{}, @@ -273,6 +274,69 @@ func Test_patchKubeadmConfigTemplate(t *testing.T) { }, expectedTemplate: &bootstrapv1.KubeadmConfigTemplate{}, }, + { + name: "no op for MachinePool class != default-worker", + template: &bootstrapv1.KubeadmConfigTemplate{}, + variables: map[string]apiextensionsv1.JSON{ + runtimehooksv1.BuiltinsName: {Raw: toJSON(runtimehooksv1.Builtins{ + MachinePool: &runtimehooksv1.MachinePoolBuiltins{ + Class: "another-class", + }, + })}, + }, + expectedTemplate: &bootstrapv1.KubeadmConfigTemplate{}, + }, + { + name: "fails if builtin.machinePool.version is not set for MachinePool class == default-worker", + template: &bootstrapv1.KubeadmConfigTemplate{}, + variables: map[string]apiextensionsv1.JSON{ + runtimehooksv1.BuiltinsName: {Raw: toJSON(runtimehooksv1.Builtins{ + MachinePool: &runtimehooksv1.MachinePoolBuiltins{ + Class: "default-worker", + }, + })}, + }, + expectedTemplate: &bootstrapv1.KubeadmConfigTemplate{}, + expectedErr: true, + }, + { + name: "set KubeletExtraArgs[cgroup-driver] to cgroupfs for Kubernetes < 1.24 and MachinePool class == default-worker", + template: &bootstrapv1.KubeadmConfigTemplate{}, + variables: map[string]apiextensionsv1.JSON{ + runtimehooksv1.BuiltinsName: {Raw: toJSON(runtimehooksv1.Builtins{ + MachinePool: &runtimehooksv1.MachinePoolBuiltins{ + Class: "default-worker", + Version: "v1.23.0", + }, + })}, + }, + expectedTemplate: &bootstrapv1.KubeadmConfigTemplate{ + Spec: bootstrapv1.KubeadmConfigTemplateSpec{ + Template: bootstrapv1.KubeadmConfigTemplateResource{ + Spec: bootstrapv1.KubeadmConfigSpec{ + JoinConfiguration: &bootstrapv1.JoinConfiguration{ + NodeRegistration: bootstrapv1.NodeRegistrationOptions{ + KubeletExtraArgs: map[string]string{"cgroup-driver": "cgroupfs"}, + }, + }, + }, + }, + }, + }, + }, + { + name: "do not set KubeletExtraArgs[cgroup-driver] to cgroupfs for Kubernetes >= 1.24 and MachinePool class == default-worker", + template: &bootstrapv1.KubeadmConfigTemplate{}, + variables: map[string]apiextensionsv1.JSON{ + runtimehooksv1.BuiltinsName: {Raw: toJSON(runtimehooksv1.Builtins{ + MachinePool: &runtimehooksv1.MachinePoolBuiltins{ + Class: "default-worker", + Version: "v1.24.0", + }, + })}, + }, + expectedTemplate: &bootstrapv1.KubeadmConfigTemplate{}, + }, } for _, tt := range tests { t.Run(tt.name, func(*testing.T) { @@ -338,6 +402,63 @@ func Test_patchDockerMachineTemplate(t *testing.T) { } } +func Test_patchDockerMachinePoolTemplate(t *testing.T) { + g := NewWithT(t) + + tests := []struct { + name string + template *infraexpv1.DockerMachinePoolTemplate + variables map[string]apiextensionsv1.JSON + expectedTemplate *infraexpv1.DockerMachinePoolTemplate + expectedErr bool + }{ + { + name: "fails if builtin.controlPlane.version nor builtin.machinePool.version is not set", + template: &infraexpv1.DockerMachinePoolTemplate{}, + variables: nil, + expectedTemplate: &infraexpv1.DockerMachinePoolTemplate{}, + expectedErr: true, + }, + { + name: "sets customImage for templates linked to ControlPlane", + template: &infraexpv1.DockerMachinePoolTemplate{}, + variables: map[string]apiextensionsv1.JSON{ + runtimehooksv1.BuiltinsName: {Raw: toJSON(runtimehooksv1.Builtins{ + ControlPlane: &runtimehooksv1.ControlPlaneBuiltins{ + Version: "v1.23.0", + }, + MachinePool: &runtimehooksv1.MachinePoolBuiltins{ + Class: "default-worker", + Version: "v1.23.0", + }, + })}, + }, + expectedTemplate: &infraexpv1.DockerMachinePoolTemplate{ + Spec: infraexpv1.DockerMachinePoolTemplateSpec{ + Template: infraexpv1.DockerMachinePoolTemplateResource{ + Spec: infraexpv1.DockerMachinePoolSpec{ + Template: infraexpv1.DockerMachinePoolMachineTemplate{ + CustomImage: "kindest/node:v1.23.0", + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(*testing.T) { + err := patchDockerMachinePoolTemplate(context.Background(), tt.template, tt.variables) + if tt.expectedErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + g.Expect(tt.template).To(BeComparableTo(tt.expectedTemplate)) + }) + } +} + // Note: given that we are testing functions used for modifying templates inside GeneratePatches it is not // required to test GeneratePatches for all the sub-cases; we are only testing that everything comes together as expected. // NOTE: custom RuntimeExtension must test specif logic added to GeneratePatches, if any. @@ -363,6 +484,14 @@ func TestHandler_GeneratePatches(t *testing.T) { }, }), } + machinePoolVars123 := []runtimehooksv1.Variable{ + newVariable(runtimehooksv1.BuiltinsName, runtimehooksv1.Builtins{ + MachinePool: &runtimehooksv1.MachinePoolBuiltins{ + Class: "default-worker", + Version: "v1.23.0", + }, + }), + } kubeadmControlPlaneTemplate := controlplanev1.KubeadmControlPlaneTemplate{ TypeMeta: metav1.TypeMeta{ Kind: "KubeadmControlPlaneTemplate", @@ -375,6 +504,12 @@ func TestHandler_GeneratePatches(t *testing.T) { APIVersion: infrav1.GroupVersion.String(), }, } + dockerMachinePoolTemplate := infraexpv1.DockerMachinePoolTemplate{ + TypeMeta: metav1.TypeMeta{ + Kind: "DockerMachinePoolTemplate", + APIVersion: infrav1.GroupVersion.String(), + }, + } dockerClusterTemplate := infrav1.DockerClusterTemplate{ TypeMeta: metav1.TypeMeta{ Kind: "DockerClusterTemplate", @@ -400,6 +535,7 @@ func TestHandler_GeneratePatches(t *testing.T) { requestItem("3", dockerMachineTemplate, machineDeploymentVars123), requestItem("4", dockerClusterTemplate, imageRepositoryVar), requestItem("5", kubeadmConfigTemplate, machineDeploymentVars123), + requestItem("6", dockerMachinePoolTemplate, machinePoolVars123), }, expectedResponse: &runtimehooksv1.GeneratePatchesResponse{ CommonResponse: runtimehooksv1.CommonResponse{ @@ -423,6 +559,9 @@ func TestHandler_GeneratePatches(t *testing.T) { ]`), responseItem("5", `[ {"op":"add","path":"/spec/template/spec/joinConfiguration","value":{"discovery":{},"nodeRegistration":{"kubeletExtraArgs":{"cgroup-driver":"cgroupfs"}}}} +]`), + responseItem("6", `[ +{"op":"add","path":"/spec/template/spec/customImage","value":"kindest/node:v1.23.0"} ]`), }, },