Migrate Fedora CoreOS bare-metal to static pod control plane

* Run a kube-apiserver, kube-scheduler, and kube-controller-manager
static pod on each controller node. Previously, kube-apiserver was
self-hosted as a DaemonSet across controllers and kube-scheduler
and kube-controller-manager were a Deployment (with 2 or
controller_count many replicas).
* Remove bootkube bootstrap and pivot to self-hosted
* Remove pod-checkpointer manifests (no longer needed)

bare-metal/fedora-coreos/kubernetes/README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features <a href="https://www.cncf.io/certification/software-conformance/"><img align="right" src="https://storage.googleapis.com/poseidon/certified-kubernetes.png"></a>
 
-* Kubernetes v1.15.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.15.3 (upstream)
 * Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#container-linux) customization

bare-metal/fedora-coreos/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=98cc19f80f2c4c3ddc63fc7aea6320e74bec561a"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=6e59af71138bc5f784453873074de16e7ee150eb"
 
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]

bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -118,33 +118,48 @@ systemd:
         PathExists=/etc/kubernetes/kubeconfig
         [Install]
         WantedBy=multi-user.target
-    - name: bootkube.service
+    - name: bootstrap.service
       contents: |
         [Unit]
-        Description=Bootstrap a Kubernetes control plane
-        ConditionPathExists=!/opt/bootkube/init_bootkube.done
+        Description=Kubernetes control plane
+        ConditionPathExists=!/opt/bootstrap/bootstrap.done
         [Service]
         Type=oneshot
         RemainAfterExit=true
-        WorkingDirectory=/opt/bootkube
-        ExecStart=/usr/bin/bash -c 'set -x && \
-          [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* && exec podman run --name bootkube --privileged \
+        WorkingDirectory=/opt/bootstrap
+        ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*'
+        ExecStart=/usr/bin/podman run --name bootstrap \
             --network host \
-            --volume /opt/bootkube/assets:/assets \
-            --volume /etc/kubernetes:/etc/kubernetes \
-            quay.io/coreos/bootkube:v0.14.0 \
-            /bootkube start --asset-dir=/assets'
-        ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done
+            --volume /opt/bootstrap/assets:/assets:ro,Z \
+            --volume /opt/bootstrap/apply:/apply:ro,Z \
+            k8s.gcr.io/hyperkube:v1.15.3 \
+            /apply
+        ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
+        ExecStartPost=-/usr/bin/podman stop bootstrap
 storage:
   directories:
     - path: /etc/kubernetes
-    - path: /opt/bootkube
+    - path: /opt/bootstrap
   files:
     - path: /etc/hostname
       mode: 0644
       contents:
         inline:
           ${domain_name}
+    - path: /opt/bootstrap/apply
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          export KUBECONFIG=/assets/auth/kubeconfig
+          until kubectl version; do
+            echo "Waiting for static pod control plane"
+            sleep 5
+          done
+          until kubectl apply -f /assets/manifests -R; do
+             echo "Retry applying manifests"
+             sleep 5
+          done
     - path: /etc/sysctl.d/reverse-path-filter.conf
       contents:
         inline: |

bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -89,7 +89,6 @@ systemd:
 storage:
   directories:
     - path: /etc/kubernetes
-    - path: /opt/bootkube
   files:
     - path: /etc/hostname
       mode: 0644

bare-metal/fedora-coreos/kubernetes/ssh.tf

@@ -1,4 +1,4 @@
-# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service
+# Secure copy assets to controllers. Activates kubelet.service
 resource "null_resource" "copy-controller-secrets" {
   count = length(var.controller_names)
 
@@ -7,6 +7,7 @@ resource "null_resource" "copy-controller-secrets" {
   depends_on = [
     matchbox_group.controller,
     matchbox_group.worker,
+    module.bootkube,
   ]
 
   connection {
@@ -55,6 +56,11 @@ resource "null_resource" "copy-controller-secrets" {
     content     = module.bootkube.etcd_peer_key
     destination = "$HOME/etcd-peer.key"
   }
+  
+  provisioner "file" {
+    source      = var.asset_dir
+    destination = "$HOME/assets"
+  }
 
   provisioner "remote-exec" {
     inline = [
@@ -67,6 +73,11 @@ resource "null_resource" "copy-controller-secrets" {
       "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
       "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
       "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
+      "sudo mv $HOME/assets /opt/bootstrap/assets",
+      "sudo mkdir -p /etc/kubernetes/bootstrap-secrets",
+      "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/",
+      "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/",
+      "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/"
     ]
   }
 }
@@ -101,9 +112,8 @@ resource "null_resource" "copy-worker-secrets" {
   }
 }
 
-# Secure copy bootkube assets to ONE controller and start bootkube to perform
-# one-time self-hosted cluster bootstrapping.
-resource "null_resource" "bootkube-start" {
+# Connect to a controller to perform one-time cluster bootstrap.
+resource "null_resource" "bootstrap" {
   # Without depends_on, this remote-exec may start before the kubeconfig copy.
   # Terraform only does one task at a time, so it would try to bootstrap
   # while no Kubelets are running.
@@ -119,15 +129,9 @@ resource "null_resource" "bootkube-start" {
     timeout = "15m"
   }
 
-  provisioner "file" {
-    source      = var.asset_dir
-    destination = "$HOME/assets"
-  }
-
   provisioner "remote-exec" {
     inline = [
-      "sudo mv $HOME/assets /opt/bootkube",
-      "sudo systemctl start bootkube",
+      "sudo systemctl start bootstrap",
     ]
   }
 }