Fix delete node permission on Fedora CoreOS node shutdown

* On cloud platforms, `delete-node.service` tries to delete the
local node (not always possible depending on preemption time)
* Since v1.18.3, kubelet TLS bootstrap generates a kubeconfig
in `/var/lib/kubelet` which should be used with kubectl in
the delete-node oneshot

CHANGES.md

@@ -4,13 +4,17 @@ Notable changes between versions.
 
 ## Latest
 
-* Remove `asset_dir` variable (default off in [v1.17.0](https://github.com/poseidon/typhoon/pull/595), deprecated in [v1.18.0](https://github.com/poseidon/typhoon/pull/678))
+* Remove `asset_dir` variable (defaulted off in [v1.17.0](https://github.com/poseidon/typhoon/pull/595), deprecated in [v1.18.0](https://github.com/poseidon/typhoon/pull/678))
+
+### Fedora CoreOS
+
+* Fix local node delete oneshot on node shutdown ([#856](https://github.com/poseidon/typhoon/pull/855))
 
 ### Flatcar Linux
 
 * Change `kubelet.service` container runner from rkt to docker ([#855](https://github.com/poseidon/typhoon/pull/855))
-* Change `delete-node.service` to be inlined and use docker ([#855](https://github.com/poseidon/typhoon/pull/855))
-  * Fix mount to restore permission to delete the local node on shutdown (cloud-only)
+* Change `delete-node.service` to use docker and an inline ExecStart ([#855](https://github.com/poseidon/typhoon/pull/855))
+* Fix local node delete oneshot on node shutdown ([#855](https://github.com/poseidon/typhoon/pull/855))
 
 ## v1.19.3
 

aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -86,10 +86,11 @@ systemd:
         [Unit]
         Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.19.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /var/lib/kubelet:/var/lib/kubelet:ro,z --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:

azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -85,10 +85,11 @@ systemd:
         [Unit]
         Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.19.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /var/lib/kubelet:/var/lib/kubelet:ro,z --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:

digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -95,10 +95,11 @@ systemd:
         [Unit]
         Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.19.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /var/lib/kubelet:/var/lib/kubelet:ro,z --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:

google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -85,10 +85,11 @@ systemd:
         [Unit]
         Description=Delete Kubernetes node on shutdown
         [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.19.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /var/lib/kubelet:/var/lib/kubelet:ro,z --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage: