Change Flatcar kubelet.service container from rkt to docker

* Use docker to run the `kubelet.service` container
* Update Kubelet mounts to match Fedora CoreOS
* Remove unused `/etc/ssl/certs` mount (see
https://github.com/poseidon/typhoon/pull/810)
* Remove unused `/usr/share/ca-certificates` mount
* Remove `/etc/resolv.conf` mount, Docker default is ok
* Change `delete-node.service` to use docker instead of rkt
and inline ExecStart, as was done on Fedora CoreOS
* Fix permission denied on shutdown `delete-node`, caused
by the kubeconfig mount changing with the introduction of
node TLS bootstrap

Background

* podmand, rkt, and runc daemonless container process runners
provide advantages over the docker daemon for system containers.
Docker requires workarounds for use in systemd units where the
ExecStart must tail logs so systemd can monitor the daemonized
container. https://github.com/moby/moby/issues/6791
* Why switch then? On Flatcar Linux, podman isn't shipped. rkt
works, but isn't developing while container standards continue
to move forward. Typhoon has used runc for the Kubelet runner
before in Fedora Atomic, but its more low-level. So we're left
with Docker, which is less than ideal, but shipped in Flatcar
* Flatcar Linux appears to be shifting system components to
use docker, which does provide some limited guards against
breakages (e.g. Flatcar cannot enable docker live restore)

CHANGES.md

@@ -6,6 +6,12 @@ Notable changes between versions.
 
 * Remove `asset_dir` variable (default off in [v1.17.0](https://github.com/poseidon/typhoon/pull/595), deprecated in [v1.18.0](https://github.com/poseidon/typhoon/pull/678))
 
+### Flatcar Linux
+
+* Change `kubelet.service` container runner from rkt to docker ([#855](https://github.com/poseidon/typhoon/pull/855))
+* Change `delete-node.service` to be inlined and use docker ([#855](https://github.com/poseidon/typhoon/pull/855))
+  * Fix mount to restore permission to delete the local node on shutdown (cloud-only)
+
 ## v1.19.3
 
 * Update Cilium from v1.8.3 to [v1.8.4](https://github.com/cilium/cilium/releases/tag/v1.8.4)

aws/container-linux/kubernetes/cl/controller.yaml

@@ -50,9 +50,11 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -60,39 +62,24 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -111,7 +98,9 @@ systemd:
           --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=10
         [Install]

aws/container-linux/kubernetes/workers/cl/worker.yaml

@@ -23,9 +23,11 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -33,39 +35,27 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        # Podman, rkt, or runc run container processes, whereas docker run
+        # is a client to a daemon and requires workarounds to use within a
+        # systemd unit. https://github.com/moby/moby/issues/6791
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -86,21 +76,24 @@ systemd:
           --read-only-port=0 \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=5
         [Install]
         WantedBy=multi-user.target
     - name: delete-node.service
-      enable: true
+      enabled: true
       contents: |
         [Unit]
-        Description=Waiting to delete Kubernetes node on shutdown
+        Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/etc/kubernetes/delete-node
+        ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:
@@ -117,22 +110,6 @@ storage:
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
-    - path: /etc/kubernetes/delete-node
-      filesystem: root
-      mode: 0744
-      contents:
-        inline: |
-          #!/bin/bash
-          set -e
-          exec /usr/bin/rkt run \
-            --trust-keys-from-https \
-            --volume config,kind=host,source=/etc/kubernetes \
-            --mount volume=config,target=/etc/kubernetes \
-            --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.19.3 \
-            --net=host \
-            --dns=host \
-            --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
 passwd:
   users:
     - name: core

azure/container-linux/kubernetes/cl/controller.yaml

@@ -50,9 +50,11 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -60,39 +62,24 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -111,7 +98,9 @@ systemd:
           --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=10
         [Install]

azure/container-linux/kubernetes/workers/cl/worker.yaml

@@ -23,9 +23,11 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -33,39 +35,27 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        # Podman, rkt, or runc run container processes, whereas docker run
+        # is a client to a daemon and requires workarounds to use within a
+        # systemd unit. https://github.com/moby/moby/issues/6791
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -86,7 +76,9 @@ systemd:
           --read-only-port=0 \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=5
         [Install]
@@ -95,12 +87,13 @@ systemd:
       enabled: true
       contents: |
         [Unit]
-        Description=Waiting to delete Kubernetes node on shutdown
+        Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/etc/kubernetes/delete-node
+        ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:
@@ -117,22 +110,6 @@ storage:
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
-    - path: /etc/kubernetes/delete-node
-      filesystem: root
-      mode: 0744
-      contents:
-        inline: |
-          #!/bin/bash
-          set -e
-          exec /usr/bin/rkt run \
-            --trust-keys-from-https \
-            --volume config,kind=host,source=/etc/kubernetes \
-            --mount volume=config,target=/etc/kubernetes \
-            --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.19.3 \
-            --net=host \
-            --dns=host \
-            --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]')
 passwd:
   users:
     - name: core

bare-metal/container-linux/kubernetes/cl/controller.yaml

@@ -58,9 +58,11 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -68,43 +70,26 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          --volume etc-iscsi,kind=host,source=/etc/iscsi \
-          --mount volume=etc-iscsi,target=/etc/iscsi \
-          --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
-          --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \
-          $${KUBELET_IMAGE} -- \
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          -v /etc/iscsi:/etc/iscsi \
+          -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -124,7 +109,9 @@ systemd:
           --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=10
         [Install]

bare-metal/container-linux/kubernetes/cl/worker.yaml

@@ -31,9 +31,11 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -41,43 +43,29 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          --volume etc-iscsi,kind=host,source=/etc/iscsi \
-          --mount volume=etc-iscsi,target=/etc/iscsi \
-          --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
-          --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \
-          $${KUBELET_IMAGE} -- \
+        # Podman, rkt, or runc run container processes, whereas docker run
+        # is a client to a daemon and requires workarounds to use within a
+        # systemd unit. https://github.com/moby/moby/issues/6791
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          -v /etc/iscsi:/etc/iscsi \
+          -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -102,7 +90,9 @@ systemd:
           --read-only-port=0 \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=5
         [Install]

digital-ocean/container-linux/kubernetes/cl/controller.yaml

@@ -58,11 +58,13 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Requires=coreos-metadata.service
         After=coreos-metadata.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         EnvironmentFile=/run/metadata/coreos
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -70,39 +72,24 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -121,7 +108,9 @@ systemd:
           --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=10
         [Install]

digital-ocean/container-linux/kubernetes/cl/worker.yaml

@@ -31,11 +31,13 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Requires=coreos-metadata.service
         After=coreos-metadata.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         EnvironmentFile=/run/metadata/coreos
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -43,39 +45,27 @@ systemd:
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        # Podman, rkt, or runc run container processes, whereas docker run
+        # is a client to a daemon and requires workarounds to use within a
+        # systemd unit. https://github.com/moby/moby/issues/6791
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -93,7 +83,9 @@ systemd:
           --read-only-port=0 \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=5
         [Install]
@@ -102,12 +94,13 @@ systemd:
       enabled: true
       contents: |
         [Unit]
-        Description=Waiting to delete Kubernetes node on shutdown
+        Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/etc/kubernetes/delete-node
+        ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:
@@ -122,19 +115,3 @@ storage:
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
-    - path: /etc/kubernetes/delete-node
-      filesystem: root
-      mode: 0744
-      contents:
-        inline: |
-          #!/bin/bash
-          set -e
-          exec /usr/bin/rkt run \
-            --trust-keys-from-https \
-            --volume config,kind=host,source=/etc/kubernetes \
-            --mount volume=config,target=/etc/kubernetes \
-            --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.19.3 \
-            --net=host \
-            --dns=host \
-            --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

docs/architecture/operating-systems.md

@@ -36,7 +36,7 @@ Together, they diversify Typhoon to support a range of container technologies.
 | kubelet image     | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary |
 | control plane images | upstream images | upstream images |
 | on-host etcd      | rkt-fly   | podman |
-| on-host kubelet   | rkt-fly   | podman |
+| on-host kubelet   | docker    | podman |
 | CNI plugins       | calico, cilium, flannel | calico, cilium, flannel |
 | coordinated drain & OS update | [FLUO](https://github.com/kinvolk/flatcar-linux-update-operator) addon | [fleetlock](https://github.com/poseidon/fleetlock) |
 

google-cloud/container-linux/kubernetes/cl/controller.yaml

@@ -50,48 +50,35 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -109,7 +96,9 @@ systemd:
           --read-only-port=0 \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=10
         [Install]

google-cloud/container-linux/kubernetes/workers/cl/worker.yaml

@@ -23,48 +23,38 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet
+        Requires=docker.service
+        After=docker.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStart=/usr/bin/rkt run \
-          --uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --stage1-from-dir=stage1-fly.aci \
-          --hosts-entry host \
-          --insecure-options=image \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
-          --mount volume=etc-machine-id,target=/etc/machine-id \
-          --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
-          --mount volume=etc-os-release,target=/etc/os-release \
-          --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-          --mount volume=etc-resolv,target=/etc/resolv.conf \
-          --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-          --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
-          --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
-          --mount volume=lib-modules,target=/lib/modules \
-          --volume run,kind=host,source=/run \
-          --mount volume=run,target=/run \
-          --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-          --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
-          --mount volume=var-lib-calico,target=/var/lib/calico \
-          --volume var-lib-docker,kind=host,source=/var/lib/docker \
-          --mount volume=var-lib-docker,target=/var/lib/docker \
-          --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
-          --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=opt-cni-bin,target=/opt/cni/bin \
-          $${KUBELET_IMAGE} -- \
+        # Podman, rkt, or runc run container processes, whereas docker run
+        # is a client to a daemon and requires workarounds to use within a
+        # systemd unit. https://github.com/moby/moby/issues/6791
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
           --anonymous-auth=false \
           --authentication-token-webhook \
           --authorization-mode=Webhook \
@@ -84,7 +74,9 @@ systemd:
           --read-only-port=0 \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
         Restart=always
         RestartSec=5
         [Install]
@@ -93,12 +85,13 @@ systemd:
       enabled: true
       contents: |
         [Unit]
-        Description=Waiting to delete Kubernetes node on shutdown
+        Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/etc/kubernetes/delete-node
+        ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:
@@ -115,22 +108,6 @@ storage:
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
-    - path: /etc/kubernetes/delete-node
-      filesystem: root
-      mode: 0744
-      contents:
-        inline: |
-          #!/bin/bash
-          set -e
-          exec /usr/bin/rkt run \
-            --trust-keys-from-https \
-            --volume config,kind=host,source=/etc/kubernetes \
-            --mount volume=config,target=/etc/kubernetes \
-            --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.19.3 \
-            --net=host \
-            --dns=host \
-            --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
 passwd:
   users:
     - name: core