Change Kubelet container image publishing

* Build Kubelet container images internally and publish
to Quay and Dockerhub (new) as an alternative in case of
registry outage or breach
* Use our infra to provide single and multi-arch (default)
Kublet images for possible future use
* Docs: Show how to use alternative Kubelet images via
snippets and a systemd dropin (builds on #737)

Changes:

* Update docs with changes to Kubelet image building
* If you prefer to trust images built by Quay/Dockerhub,
automated image builds are still available with unique
tags (albeit with some limitations):
  * Quay automated builds are tagged `build-{short_sha}`
  (limit: only amd64)
  * Dockerhub automated builts are tagged `build-{tag}`
  and `build-master` (limit: only amd64, no shas)

Links:

* Kubelet: https://github.com/poseidon/kubelet
* Docs: https://typhoon.psdn.io/topics/security/#container-images
* Registries:
  * quay.io/poseidon/kubelet
  * docker.io/psdn/kubelet

CHANGES.md

@@ -4,6 +4,14 @@ Notable changes between versions.
 
 ## Latest
 
+* Update Kubelet image build infra and publishing ([#749](https://github.com/poseidon/typhoon/pull/749))
+  * Publish Kubelet images from internal infra to Quay and Dockerhub
+    * [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) (official)
+    * [docker.io/psdn/kubelet](https://hub.docker.com/r/psdn/kubelet) (fallback)
+  * Document use of alternate Kubelet images during registry incidents
+  * For those preferring to trust images built by Quay/Dockerhub,
+  automated image builds are still available with an alternate tag
+  strategy (see [docs](https://typhoon.psdn.io/topics/security/#container-images))
 * Update Calico from v3.14.0 to [v3.14.1](https://docs.projectcalico.org/v3.14/release-notes/)
 
 ### Addons

aws/container-linux/kubernetes/cl/controller.yaml

@@ -52,7 +52,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -137,7 +137,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/apply

aws/container-linux/kubernetes/workers/cl/worker.yaml

@@ -25,7 +25,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -130,7 +130,7 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

aws/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -54,7 +54,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -126,7 +126,7 @@ systemd:
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             --entrypoint=/apply \
-            quay.io/poseidon/kubelet:v1.18.3
+            quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:

aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -24,7 +24,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -90,7 +90,7 @@ systemd:
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:

azure/container-linux/kubernetes/cl/controller.yaml

@@ -52,7 +52,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -135,7 +135,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/apply

azure/container-linux/kubernetes/workers/cl/worker.yaml

@@ -25,7 +25,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -128,7 +128,7 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]')

azure/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -54,7 +54,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -126,7 +126,7 @@ systemd:
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             --entrypoint=/apply \
-            quay.io/poseidon/kubelet:v1.18.3
+            quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:

azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -24,7 +24,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -90,7 +90,7 @@ systemd:
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:

bare-metal/container-linux/kubernetes/cl/controller.yaml

@@ -60,7 +60,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -150,7 +150,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/apply

bare-metal/container-linux/kubernetes/cl/worker.yaml

@@ -33,7 +33,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests

bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -53,7 +53,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -137,7 +137,7 @@ systemd:
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             --entrypoint=/apply \
-            quay.io/poseidon/kubelet:v1.18.3
+            quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:

bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -23,7 +23,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin

digital-ocean/container-linux/kubernetes/cl/controller.yaml

@@ -62,7 +62,7 @@ systemd:
         After=coreos-metadata.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         EnvironmentFile=/run/metadata/coreos
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -147,7 +147,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/apply

digital-ocean/container-linux/kubernetes/cl/worker.yaml

@@ -35,7 +35,7 @@ systemd:
         After=coreos-metadata.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         EnvironmentFile=/run/metadata/coreos
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -134,7 +134,7 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -55,7 +55,7 @@ systemd:
         After=afterburn.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         EnvironmentFile=/run/metadata/afterburn
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -138,7 +138,7 @@ systemd:
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             --entrypoint=/apply \
-            quay.io/poseidon/kubelet:v1.18.3
+            quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:

digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -26,7 +26,7 @@ systemd:
         After=afterburn.service
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         EnvironmentFile=/run/metadata/afterburn
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -100,7 +100,7 @@ systemd:
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage:

docs/advanced/customization.md

@@ -174,3 +174,34 @@ module "nemo" {
 
 To customize low-level Kubernetes control plane bootstrapping, see the [poseidon/terraform-render-bootstrap](https://github.com/poseidon/terraform-render-bootstrap) Terraform module.
 
+## Kubelet
+
+Typhoon publishes Kubelet [container images](/topics/security.md#container-images) to Quay.io (default) and to Dockerhub (in case of a Quay [outage](https://github.com/poseidon/typhoon/issues/735) or breach). Quay automated builds also provide the option for fully verifiable tagged images (`build-{short_sha}`).
+
+To set an alternative Kubelet image, use a snippet to set a systemd dropin.
+
+```
+# host-image-override.yaml
+variant: fcos           <- remove for Flatcar Linux
+version: 1.0.0          <- remove for Flatcar Linux
+systemd:
+  units:
+    - name: kubelet.service
+      dropins:
+        - name: 10-image-override.conf
+          contents: |
+            [Service]
+            Environment=KUBELET_IMAGE=docker.io/psdn/kubelet:v1.18.3
+```
+
+```
+module "nemo" {
+  ...
+
+  worker_snippets = [
+    file("./snippets/host-image-override.yaml")
+  ]
+  ...
+}
+```
+

docs/topics/security.md

@@ -52,7 +52,19 @@ Typhoon uses upstream container images (where possible) and upstream binaries.
 !!! note
     Kubernetes releases `kubelet` as a binary for distros to package, either as a DEB/RPM on traditional distros or as a container image for container-optimized operating systems.
 
-Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet) for use in Typhoon. The upstream Kubelet binary is checksummed and packaged directly. Quay automated builds provide verifiability and confidence in image contents.
+Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet). Builds fetch the upstream Kubelet binary and verify its checksum.
+
+The Kubelet image is published to Quay.io and Dockerhub.
+
+* [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet) (official)
+* [docker.io/psdn/kubelet](https://hub.docker.com/r/psdn/kubelet) (fallback)
+
+Two tag styles indicate the build strategy used.
+
+* Typhoon internal infra publishes single and multi-arch images (e.g. `v1.18.3`, `v1.18.3-amd64`, `v1.18.3-arm64`, `v1.18.3-2-g23228e6-amd64`, `v1.18.3-2-g23228e6-arm64`)
+* Quay and Dockerhub automated builds publish verifiable images (e.g. `build-SHA` on Quay, `build-TAG` on Dockerhub)
+
+The Typhoon-built Kubelet image is used as the official image. Automated builds provide an alternative image for those preferring to trust images built by Quay/Dockerhub (albeit lacking multi-arch). To use the fallback registry or an alternative tag, see [customization](/advanced/customization.md#kubelet).
 
 ## Disclosures
 

google-cloud/container-linux/kubernetes/cl/controller.yaml

@@ -52,7 +52,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -135,7 +135,7 @@ systemd:
             --volume script,kind=host,source=/opt/bootstrap/apply \
             --mount volume=script,target=/apply \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/apply

google-cloud/container-linux/kubernetes/workers/cl/worker.yaml

@@ -25,7 +25,7 @@ systemd:
         Description=Kubelet
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -128,7 +128,7 @@ storage:
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
             --insecure-options=image \
-            docker://quay.io/poseidon/kubelet:v1.18.3 \
+            docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
             --net=host \
             --dns=host \
             --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -54,7 +54,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -126,7 +126,7 @@ systemd:
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             --entrypoint=/apply \
-            quay.io/poseidon/kubelet:v1.18.3
+            quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:

google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -24,7 +24,7 @@ systemd:
         Description=Kubelet (System Container)
         Wants=rpc-statd.service
         [Service]
-        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
@@ -90,7 +90,7 @@ systemd:
         Type=oneshot
         RemainAfterExit=true
         ExecStart=/bin/true
-        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
+        ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
         [Install]
         WantedBy=multi-user.target
 storage: