Migrate most Kubelet flags to KubeletConfiguration file

* Add a KubeletConfiguration file to replace most Kubelet
flags, to prepare for upcoming changes
* Pass Kubelet the --config flag to specify the location of
the KubeletConfiguration
* Remove flsgs / configuration where it matches the defaults
  * Remove --cgroups-per-qos, defaults to true
  * Remove --container-runtime, defaults to remote
  * Remove enforce-node-allocatable=pods, defaults to pods

Rel:

* https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/
* https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/

aws/fedora-coreos/kubernetes/butane/controller.yaml

@@ -85,28 +85,13 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
           --provider-id=aws:///$${AFTERBURN_AWS_AVAILABILITY_ZONE}/$${AFTERBURN_AWS_INSTANCE_ID} \
-          --read-only-port=0 \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -144,6 +129,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

aws/fedora-coreos/kubernetes/workers/butane/worker.yaml

@@ -58,19 +58,9 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/node \
           %{~ for label in split(",", node_labels) ~}
@@ -79,12 +69,7 @@ systemd:
           %{~ for taint in split(",", node_taints) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --provider-id=aws:///$${AFTERBURN_AWS_AVAILABILITY_ZONE}/$${AFTERBURN_AWS_INSTANCE_ID}
-          --provider-id=aws:///$${AFTERBURN_AWS_AVAILABILITY_ZONE}/$${AFTERBURN_AWS_INSTANCE_ID} \
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -113,6 +98,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       contents:
         inline: |

aws/flatcar-linux/kubernetes/butane/controller.yaml

@@ -84,26 +84,13 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
           --provider-id=aws:///$${COREOS_EC2_AVAILABILITY_ZONE}/$${COREOS_EC2_INSTANCE_ID} \
-          --read-only-port=0 \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -143,6 +130,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

aws/flatcar-linux/kubernetes/workers/butane/worker.yaml

@@ -59,17 +59,9 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/node \
           %{~ for label in split(",", node_labels) ~}
@@ -78,12 +70,7 @@ systemd:
           %{~ for taint in split(",", node_taints) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --provider-id=aws:///$${COREOS_EC2_AVAILABILITY_ZONE}/$${COREOS_EC2_INSTANCE_ID}
-          --provider-id=aws:///$${COREOS_EC2_AVAILABILITY_ZONE}/$${COREOS_EC2_INSTANCE_ID} \
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -111,6 +98,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       mode: 0644
       contents:

azure/fedora-coreos/kubernetes/butane/controller.yaml

@@ -81,27 +81,12 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -139,6 +124,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

azure/fedora-coreos/kubernetes/workers/butane/worker.yaml

@@ -54,32 +54,17 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
-          --node-labels=node.kubernetes.io/node \
           %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
           %{~ endfor ~}
           %{~ for taint in split(",", node_taints) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --node-labels=node.kubernetes.io/node
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -108,6 +93,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       contents:
         inline: |

azure/flatcar-linux/kubernetes/butane/controller.yaml

@@ -81,25 +81,12 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -139,6 +126,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

azure/flatcar-linux/kubernetes/workers/butane/worker.yaml

@@ -56,30 +56,17 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
-          --node-labels=node.kubernetes.io/node \
           %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
           %{~ endfor ~}
           %{~ for taint in split(",", node_taints) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --node-labels=node.kubernetes.io/node
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -107,6 +94,31 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       mode: 0644
       contents:

bare-metal/fedora-coreos/kubernetes/butane/controller.yaml

@@ -80,28 +80,13 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=${domain_name} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -149,6 +134,31 @@ storage:
       contents:
         inline:
           ${domain_name}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

bare-metal/fedora-coreos/kubernetes/butane/worker.yaml

@@ -53,33 +53,18 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=${domain_name} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
-          --node-labels=node.kubernetes.io/node \
           %{~ for label in compact(split(",", node_labels)) ~}
           --node-labels=${label} \
           %{~ endfor ~}
           %{~ for taint in compact(split(",", node_taints)) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --node-labels=node.kubernetes.io/node
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -104,6 +89,31 @@ storage:
       contents:
         inline:
           ${domain_name}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       contents:
         inline: |

bare-metal/flatcar-linux/kubernetes/butane/controller.yaml

@@ -89,26 +89,13 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=${domain_name} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -150,6 +137,31 @@ storage:
       contents:
         inline:
           ${domain_name}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

bare-metal/flatcar-linux/kubernetes/butane/worker.yaml

@@ -64,17 +64,9 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=${domain_name} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/node \
@@ -84,11 +76,7 @@ systemd:
           %{~ for taint in compact(split(",", node_taints)) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --node-labels=node.kubernetes.io/node
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -107,6 +95,31 @@ storage:
       contents:
         inline:
           ${domain_name}
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       mode: 0644
       contents:

digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml

@@ -83,28 +83,13 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -146,6 +131,31 @@ storage:
     - path: /etc/kubernetes
     - path: /opt/bootstrap
   files:
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml

@@ -61,23 +61,11 @@ systemd:
           --authentication-token-webhook \
           --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
-          --node-labels=node.kubernetes.io/node \
+          --node-labels=node.kubernetes.io/node
-          --pod-manifest-path=/etc/kubernetes/manifests \
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -110,6 +98,31 @@ storage:
   directories:
     - path: /etc/kubernetes
   files:
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       contents:
         inline: |

digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml

@@ -92,26 +92,13 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -148,6 +135,31 @@ storage:
     - path: /etc/kubernetes
       mode: 0755
   files:
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml

@@ -67,25 +67,12 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
-          --node-labels=node.kubernetes.io/node \
+          --node-labels=node.kubernetes.io/node
-          --pod-manifest-path=/etc/kubernetes/manifests \
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -111,6 +98,31 @@ storage:
     - path: /etc/kubernetes
       mode: 0755
   files:
+    - path: /etc/kubernetes/kubelet.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       mode: 0644
       contents:

google-cloud/fedora-coreos/kubernetes/butane/controller.yaml

@@ -81,27 +81,12 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -139,6 +124,30 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml

@@ -54,32 +54,17 @@ systemd:
           --volume /var/run/lock:/var/run/lock:z \
           --volume /opt/cni/bin:/opt/cni/bin:z \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --cgroups-per-qos=true \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --enforce-node-allocatable=pods \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
-          --node-labels=node.kubernetes.io/node \
           %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
           %{~ endfor ~}
           %{~ for taint in split(",", node_taints) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --node-labels=node.kubernetes.io/node
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStop=-/usr/bin/podman stop kubelet
         Delegate=yes
         Restart=always
@@ -108,6 +93,30 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       contents:
         inline: |

google-cloud/flatcar-linux/kubernetes/butane/controller.yaml

@@ -81,25 +81,12 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -139,6 +126,30 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /opt/bootstrap/layout
       mode: 0544
       contents:

google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml

@@ -56,30 +56,17 @@ systemd:
           -v /var/log:/var/log \
           -v /opt/cni/bin:/opt/cni/bin \
           $${KUBELET_IMAGE} \
-          --anonymous-auth=false \
-          --authentication-token-webhook \
-          --authorization-mode=Webhook \
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
-          --cgroup-driver=systemd \
+          --config=/etc/kubernetes/kubelet.yaml \
-          --container-runtime=remote \
           --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-          --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns=${cluster_dns_service_ip} \
-          --cluster_domain=${cluster_domain_suffix} \
-          --healthz-port=0 \
           --kubeconfig=/var/lib/kubelet/kubeconfig \
-          --node-labels=node.kubernetes.io/node \
           %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
           %{~ endfor ~}
           %{~ for taint in split(",", node_taints) ~}
           --register-with-taints=${taint} \
           %{~ endfor ~}
-          --pod-manifest-path=/etc/kubernetes/manifests \
+          --node-labels=node.kubernetes.io/node
-          --read-only-port=0 \
-          --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --rotate-certificates \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
         ExecStop=docker stop kubelet
         ExecStopPost=docker rm kubelet
@@ -107,6 +94,30 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /etc/kubernetes/kubelet.yaml
+      contents:
+        inline: |
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          kind: KubeletConfiguration
+          authentication:
+            anonymous:
+              enabled: false
+            webhook:
+              enabled: true
+            x509:
+              clientCAFile: /etc/kubernetes/ca.crt
+          authorization:
+            mode: Webhook
+          cgroupDriver: systemd
+          clusterDNS:
+            - ${cluster_dns_service_ip}
+          clusterDomain: ${cluster_domain_suffix}
+          healthzPort: 0
+          rotateCertificates: true
+          staticPodPath: /etc/kubernetes/manifests
+          readOnlyPort: 0
+          resolvConf: /run/systemd/resolve/resolv.conf
+          volumePluginDir: /var/lib/kubelet/volumeplugins
     - path: /etc/sysctl.d/max-user-watches.conf
       mode: 0644
       contents: