Switch Fedora CoreOS from docker-shim to containerd

* Migrate from `docker-shim` to `containerd` in preparation
for Kubernetes v1.24.0 dropping `docker-shim` support
* Much consideration was given to the container runtime
choice. https://github.com/poseidon/typhoon/issues/899
provides relevant rationales

CHANGES.md

@@ -4,6 +4,12 @@ Notable changes between versions.
 
 ## Latest
 
+### Fedora CoreOS
+
+* Switch Kubernetes Container Runtime from `docker` to `containerd` ([#1101](https://github.com/poseidon/typhoon/pull/1101))
+
+## v1.23.1
+
 * Kubernetes [v1.23.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.23.md#v1231)
 * Workaround Terraform v1.1 regression in `file` provisioner ([#1093](https://github.com/poseidon/typhoon/pull/1093))
 

aws/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -29,7 +29,7 @@ systemd:
         LimitNOFILE=40000
         [Install]
         WantedBy=multi-user.target
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -74,7 +74,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -86,6 +86,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -220,6 +222,25 @@ storage:
           ETCD_PEER_CLIENT_CERT_AUTH=true
           ETCD_UNSUPPORTED_ARCH=arm64
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core

aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -3,7 +3,7 @@ variant: fcos
 version: 1.4.0
 systemd:
   units:
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -47,7 +47,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -59,6 +59,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -131,9 +133,27 @@ storage:
           DefaultMemoryAccounting=yes
           DefaultBlockIOAccounting=yes
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core
       ssh_authorized_keys:
         - ${ssh_authorized_key}
-

azure/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -29,7 +29,7 @@ systemd:
         LimitNOFILE=40000
         [Install]
         WantedBy=multi-user.target
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -70,7 +70,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -82,6 +82,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -214,6 +216,25 @@ storage:
           ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
           ETCD_PEER_CLIENT_CERT_AUTH=true
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core

azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -3,7 +3,7 @@ variant: fcos
 version: 1.4.0
 systemd:
   units:
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -43,7 +43,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -55,6 +55,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -126,10 +128,28 @@ storage:
           DefaultMemoryAccounting=yes
           DefaultBlockIOAccounting=yes
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core
       ssh_authorized_keys:
         - ${ssh_authorized_key}
 
-

bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -29,7 +29,7 @@ systemd:
         LimitNOFILE=40000
         [Install]
         WantedBy=multi-user.target
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -69,7 +69,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -81,6 +81,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -224,6 +226,25 @@ storage:
           ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
           ETCD_PEER_CLIENT_CERT_AUTH=true
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core

bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -3,7 +3,7 @@ variant: fcos
 version: 1.4.0
 systemd:
   units:
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -42,7 +42,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -54,6 +54,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -122,6 +124,25 @@ storage:
           DefaultMemoryAccounting=yes
           DefaultBlockIOAccounting=yes
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core

digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -29,7 +29,7 @@ systemd:
         LimitNOFILE=40000
         [Install]
         WantedBy=multi-user.target
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -72,7 +72,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -84,6 +84,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -221,4 +223,23 @@ storage:
           ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
           ETCD_PEER_CLIENT_CERT_AUTH=true
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 

digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -3,7 +3,7 @@ variant: fcos
 version: 1.4.0
 systemd:
   units:
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -46,7 +46,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -58,6 +58,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -128,3 +130,22 @@ storage:
           DefaultMemoryAccounting=yes
           DefaultBlockIOAccounting=yes
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true

google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -29,7 +29,7 @@ systemd:
         LimitNOFILE=40000
         [Install]
         WantedBy=multi-user.target
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -70,7 +70,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -82,6 +82,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -214,6 +216,25 @@ storage:
           ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
           ETCD_PEER_CLIENT_CERT_AUTH=true
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core

google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -3,7 +3,7 @@ variant: fcos
 version: 1.4.0
 systemd:
   units:
-    - name: docker.service
+    - name: containerd.service
       enabled: true
     - name: wait-for-dns.service
       enabled: true
@@ -43,7 +43,7 @@ systemd:
           --volume /run:/run \
           --volume /sys/fs/cgroup:/sys/fs/cgroup \
           --volume /var/lib/calico:/var/lib/calico:ro \
-          --volume /var/lib/docker:/var/lib/docker \
+          --volume /var/lib/containerd:/var/lib/containerd \
           --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \
           --volume /var/log:/var/log \
           --volume /var/run/lock:/var/run/lock:z \
@@ -55,6 +55,8 @@ systemd:
           --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
           --cgroup-driver=systemd \
           --cgroups-per-qos=true \
+          --container-runtime=remote \
+          --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
           --enforce-node-allocatable=pods \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${cluster_dns_service_ip} \
@@ -126,6 +128,25 @@ storage:
           DefaultMemoryAccounting=yes
           DefaultBlockIOAccounting=yes
     - path: /etc/fedora-coreos/iptables-legacy.stamp
+    - path: /etc/containerd/config.toml
+      overwrite: true
+      contents:
+        inline: |
+          version = 2
+          root = "/var/lib/containerd"
+          state = "/run/containerd"
+          subreaper = true
+          oom_score = -999
+          [grpc]
+          address = "/run/containerd/containerd.sock"
+          uid = 0
+          gid = 0
+          [plugins."io.containerd.grpc.v1.cri"]
+          enable_selinux = true
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          runtime_type = "io.containerd.runc.v2"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+          SystemdCgroup = true
 passwd:
   users:
     - name: core