diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index eb7304e6a5e1..e5577a030a2d 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -10,7 +10,8 @@ - [Generating a Kubeconfig](./tasks/certs/generate-kubeconfig.md) - [Auto Rotate Certificates in KCP](./tasks/certs/auto-rotate-certificates-in-kcp.md) - [Bootstrap](./tasks/bootstrap/index.md) - - [Kubeadm based bootstrap](./tasks/bootstrap/kubeadm-bootstrap.md) + - [Kubeadm based bootstrap](./tasks/bootstrap/kubeadm-bootstrap/index.md) + - [Kubelet configuration](./tasks/bootstrap/kubeadm-bootstrap/kubelet-config.md) - [MicroK8s based bootstrap](./tasks/bootstrap/microk8s-bootstrap.md) - [Upgrading management and workload clusters](./tasks/upgrading-clusters.md) - [External etcd](./tasks/external-etcd.md) diff --git a/docs/book/src/tasks/bootstrap/kubeadm-bootstrap.md b/docs/book/src/tasks/bootstrap/kubeadm-bootstrap/index.md similarity index 100% rename from docs/book/src/tasks/bootstrap/kubeadm-bootstrap.md rename to docs/book/src/tasks/bootstrap/kubeadm-bootstrap/index.md diff --git a/docs/book/src/tasks/bootstrap/kubeadm-bootstrap/kubelet-config.md b/docs/book/src/tasks/bootstrap/kubeadm-bootstrap/kubelet-config.md new file mode 100644 index 000000000000..dda906c56cc1 --- /dev/null +++ b/docs/book/src/tasks/bootstrap/kubeadm-bootstrap/kubelet-config.md @@ -0,0 +1,354 @@ +# Kubelet Configuration + +CAPBK has several ways to configure kubelet. + +- [Pass `KubeletConfiguration` file via `KubeadmConfigSpec.files`](#pass-kubeletconfiguration-file-via-kubeadmconfigspecfiles) +- [Set kubelet flags via `KubeadmConfigSpec.kubeletExtraArgs`](#set-kubelet-flags-via-kubeadmconfigspeckubeletextraargs) +- [`kubeletconfiguration` patch target](#use-the-kubeletconfiguration-patch-target) + +## Pass `KubeletConfiguration` file via `KubeadmConfigSpec.files` + +You can use `KubeadmConfigSpec.files` to put any files on nodes. This example puts a `KubeletConfiguration` file on nodes via `KubeadmConfigSpec.files`, and makes kubelet use it via `KubeadmConfigSpec.kubeletExtraArgs`. You can check available configurations of `KubeletConfiguration` on [Kubelet Configuration (v1beta1) | Kubernetes](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration). + +This method is easy to replace the whole kubelet configuration generated by kubeadm, but it is not easy to replace only a part of the kubelet configuration. + +### KubeadmControlPlaneTemplate + +```yaml +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: cloudinit-control-plane + namespace: default +spec: + template: + spec: + kubeadmConfigSpec: + files: + # We put a KubeletConfiguration file on nodes via KubeadmConfigSpec.files + # In this example, we directly put the file content in the KubeadmConfigSpec.files.content field. + - path: /etc/kubernetes/kubelet/config.yaml + owner: "root:root" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + kubeReserved: + cpu: "1" + memory: "2Gi" + ephemeral-storage: "1Gi" + systemReserved: + cpu: "500m" + memory: "1Gi" + ephemeral-storage: "1Gi" + evictionHard: + memory.available: "500Mi" + nodefs.available: "10%" + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.128.0.10 + clusterDomain: cluster.local + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionPressureTransitionPeriod: 0s + fileCheckFrequency: 0s + healthzBindAddress: 127.0.0.1 + healthzPort: 10248 + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + rotateCertificates: true + runtimeRequestTimeout: 0s + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + volumeStatsAggPeriod: 0s + clusterConfiguration: + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + initConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Here we configure kubelet to use the KubeletConfiguration file we put on nodes via KubeadmConfigSpec.files + kubeletExtraArgs: + config: "/etc/kubernetes/kubelet/config.yaml" + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Here we configure kubelet to use the KubeletConfiguration file we put on nodes via KubeadmConfigSpec.files + kubeletExtraArgs: + config: "/etc/kubernetes/kubelet/config.yaml" +``` + +### KubeadmConfigTemplate + +```yaml +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: cloudinit-default-worker-bootstraptemplate + namespace: default +spec: + template: + spec: + files: + # We puts a KubeletConfiguration file on nodes via KubeadmConfigSpec.files + # In this example, we directly put the file content in the KubeadmConfigSpec.files.content field. + - path: /etc/kubernetes/kubelet/config.yaml + owner: "root:root" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + kubeReserved: + cpu: "1" + memory: "2Gi" + ephemeral-storage: "1Gi" + systemReserved: + cpu: "500m" + memory: "1Gi" + ephemeral-storage: "1Gi" + evictionHard: + memory.available: "500Mi" + nodefs.available: "10%" + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.128.0.10 + clusterDomain: cluster.local + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionPressureTransitionPeriod: 0s + fileCheckFrequency: 0s + healthzBindAddress: 127.0.0.1 + healthzPort: 10248 + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + rotateCertificates: true + runtimeRequestTimeout: 0s + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + volumeStatsAggPeriod: 0s + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Here we configure kubelet to use the KubeletConfiguration file we put on nodes via KubeadmConfigSpec.files + kubeletExtraArgs: + config: "/etc/kubernetes/kubelet/config.yaml" +``` + +## Set kubelet flags via `KubeadmConfigSpec.kubeletExtraArgs` + +We can pass kubelet command-line flags via `KubeadmConfigSpec.kubeletExtraArgs`. This example is equivalent to setting `--kube-reserved`, `--system-reserved`, and `--eviction-hard` flags for the kubelet command. + +This method is useful when you want to set kubelet flags that are not configurable via the `KubeletConfiguration` file, however, it is not recommended to use this method to set flags that are configurable via the `KubeletConfiguration` file. + +### KubeadmControlPlaneTemplate + +```yaml +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: kubelet-extra-args-control-plane + namespace: default +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + initConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Set kubelet flags via KubeadmConfigSpec.kubeletExtraArgs + kubeletExtraArgs: + kube-reserved: cpu=1,memory=2Gi,ephemeral-storage=1Gi + system-reserved: cpu=500m,memory=1Gi,ephemeral-storage=1Gi + eviction-hard: memory.available<500Mi,nodefs.available<10% + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Set kubelet flags via KubeadmConfigSpec.kubeletExtraArgs + kubeletExtraArgs: + kube-reserved: cpu=1,memory=2Gi,ephemeral-storage=1Gi + system-reserved: cpu=500m,memory=1Gi,ephemeral-storage=1Gi + eviction-hard: memory.available<500Mi,nodefs.available<10% +``` + +### KubeadmConfigTemplate + +```yaml +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: kubelet-extra-args-default-worker-bootstraptemplate + namespace: default +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Set kubelet flags via KubeadmConfigSpec.kubeletExtraArgs + kubeletExtraArgs: + kube-reserved: cpu=1,memory=2Gi,ephemeral-storage=1Gi + system-reserved: cpu=500m,memory=1Gi,ephemeral-storage=1Gi + eviction-hard: memory.available<500Mi,nodefs.available<10% +``` + +## Use kubeadm's `kubeletconfiguration` patch target + +We can use kubeadm's `kubeletconfiguration` patch target to patch the kubelet configuration file. In this example, we put a patch file for `kubeletconfiguration` target in `strategic` `patchtype` on nodes via `KubeadmConfigSpec.files`. For more details, see [Customizing components with the kubeadm API | Kubernetes](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/#patches) + +This method is useful when you want to change the kubelet configuration file partially on specific nodes. For example, you can deploy a partially patched kubelet configuration file on specific nodes based on the default configuration used for `kubeadm init` or `kubeadm join`. + +### KubeadmControlPlaneTemplate + +```yaml +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: kubeadm-config-template-control-plane + namespace: default +spec: + template: + spec: + kubeadmConfigSpec: + files: + # Here we put a patch file for kubeletconfiguration target in strategic patchtype on nodes via KubeadmConfigSpec.files + # The naming convention of the patch file is kubeletconfiguration{suffix}+{patchtype}.json where {suffix} is an string and {patchtype} is one of the following: strategic, merge, json. + # {suffix} determines the order of the patch files. The patches are applied in the alpha-numerical order of the {suffix}. + - path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json + owner: "root:root" + permissions: "0644" + content: | + { + "apiVersion": "kubelet.config.k8s.io/v1beta1", + "kind": "KubeletConfiguration", + "kubeReserved": { + "cpu": "1", + "memory": "2Gi", + "ephemeral-storage": "1Gi", + }, + "systemReserved": { + "cpu": "500m", + "memory": "1Gi", + "ephemeral-storage": "1Gi", + }, + "evictionHard": { + "memory.available": "500Mi", + "nodefs.available": "10%", + }, + } + clusterConfiguration: + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + initConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Here we specify the directory that contains the patch files + patches: + directory: /etc/kubernetes/patches + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Here we specify the directory that contains the patch files + patches: + directory: /etc/kubernetes/patches +``` + +### KubeadmConfigTemplate + +```yaml +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: kubeadm-config-template-default-worker-bootstraptemplate + namespace: default +spec: + template: + spec: + files: + # Here we put a patch file for kubeletconfiguration target in strategic patchtype on nodes via KubeadmConfigSpec.files + # The naming convention of the patch file is kubeletconfiguration{suffix}+{patchtype}.json where {suffix} is an string and {patchtype} is one of the following: strategic, merge, json. + # {suffix} determines the order of the patch files. The patches are applied in the alpha-numerical order of the {suffix}. + - path: /etc/kubernetes/patches/kubeletconfiguration0+strategic.json + owner: "root:root" + permissions: "0644" + content: | + { + "apiVersion": "kubelet.config.k8s.io/v1beta1", + "kind": "KubeletConfiguration", + "kubeReserved": { + "cpu": "1", + "memory": "2Gi", + "ephemeral-storage": "1Gi", + }, + "systemReserved": { + "cpu": "500m", + "memory": "1Gi", + "ephemeral-storage": "1Gi", + }, + "evictionHard": { + "memory.available": "500Mi", + "nodefs.available": "10%", + }, + } + joinConfiguration: + nodeRegistration: + criSocket: unix:///var/run/containerd/containerd.sock + # Here we specify the directory that contains the patch files + patches: + directory: /etc/kubernetes/patches +``` \ No newline at end of file