Change AWS Fedora module to fedora-atomic

aws/fedora-atomic/kubernetes/README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features <a href="https://www.cncf.io/certification/software-conformance/"><img align="right" src="https://storage.googleapis.com/poseidon/certified-kubernetes.png"></a>
 
-* Kubernetes v1.9.6 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.10.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/)

aws/fedora-atomic/kubernetes/ami.tf

@@ -14,6 +14,6 @@ data "aws_ami" "fedora" {
 
   filter {
     name   = "name"
-    values = ["Fedora-Cloud-Base-27*-gp2-0"]
+    values = ["Fedora-Atomic-27*-gp2-0"]
   }
 }

aws/fedora-atomic/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5f3546b66ffb9946b36e612537bb6a1830ae7746"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=61fb176647e15d4d0e72fdccb34d27e47430407c"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
@@ -11,4 +11,7 @@ module "bootkube" {
   pod_cidr              = "${var.pod_cidr}"
   service_cidr          = "${var.service_cidr}"
   cluster_domain_suffix = "${var.cluster_domain_suffix}"
+
+  # Fedora
+  trusted_certs_dir = "/etc/pki/tls/certs"
 }

aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl

@@ -1,15 +1,4 @@
 #cloud-config
-yum_repos:
-  kubernetes:
-    name: kubernetes
-    baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
-    enabled: true
-    gpgcheck: true
-    gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-packages:
-  - [docker, 1.13.1]
-  - [kubelet, 1.10.0]
-  - nfs-utils
 write_files:
   - path: /etc/systemd/system/etcd-member.service
     content: |
@@ -25,7 +14,7 @@ write_files:
       ExecStartPre=/bin/mkdir -p /var/lib/etcd
       ExecStart=/usr/bin/docker run --rm --name etcd-member \
         --net=host \
-        -v /usr/share/ca-certificates:/usr/share/ca-certificates:ro,z \
+        -v /etc/pki/tls/certs:/usr/share/ca-certificates:ro,z \
         -v /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
         -v /var/lib/etcd:/var/lib/etcd:Z \
         --env-file=/etc/etcd/etcd.conf \
@@ -55,12 +44,13 @@ write_files:
       ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
       ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
       ETCD_PEER_CLIENT_CERT_AUTH=true
-  - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf
+  - path: /etc/systemd/system/kubelet.service
     content: |
       [Unit]
       Description=Kubelet
       Wants=rpc-statd.service
       [Service]
+      WorkingDirectory=/etc/kubernetes
       ExecStartPre=/bin/mkdir -p /opt/cni/bin
       ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
       ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -69,8 +59,11 @@ write_files:
       ExecStartPre=/bin/mkdir -p /var/lib/cni
       ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
       ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+      # Atomic's system containers and RPMs are old and unfriendly. Use this.
+      ExecStartPre=/usr/bin/curl -L https://dl.k8s.io/v1.10.0/kubernetes-node-linux-amd64.tar.gz -o kubernetes-node-linux-amd64.tar.gz
+      ExecStartPre=/usr/bin/tar xzf kubernetes-node-linux-amd64.tar.gz -C /usr/local/bin --strip-components=3 kubernetes/node/bin/kubelet 
       ExecStart=
-      ExecStart=/usr/bin/kubelet \
+      ExecStart=/usr/local/bin/kubelet \
         --allow-privileged \
         --anonymous-auth=false \
         --cgroup-driver=systemd \
@@ -91,43 +84,59 @@ write_files:
       RestartSec=10
       [Install]
       WantedBy=multi-user.target 
+  - path: /etc/systemd/system/kubelet.path
+    content: |
+      [Unit]
+      Description=Watch for kubeconfig
+      [Path]
+      PathExists=/etc/kubernetes/kubeconfig
+      [Install]
+      WantedBy=multi-user.target
+  - path: /etc/systemd/system/bootkube.service
+    content: |
+      [Unit]
+      Description=Bootstrap a Kubernetes cluster
+      ConditionPathExists=!/var/bootkube/init_bootkube.done
+      [Service]
+      Type=oneshot
+      RemainAfterExit=true
+      WorkingDirectory=/var/bootkube
+      ExecStartPre=/bin/mkdir -p /var/bootkube
+      ExecStart=/usr/local/bin/bootkube-start
+      ExecStartPost=/bin/touch /var/bootkube/init_bootkube.done
+      [Install]
+      WantedBy=multi-user.target
+  - path: /etc/kubernetes/.keep
   - path: /etc/kubernetes/kubeconfig
     permissions: '0644'
     content: |
       ${kubeconfig}
   - path: /etc/selinux/config
+    owner: root:root
+    permissions: '0644'
     content: |
       SELINUX=permissive
-  - path: /etc/systemd/system/bootkube.service
+      SELINUXTYPE=targeted
-    content: |
+  - path: /var/bootkube/.keep
-      [Unit]
+  - path: /usr/local/bin/bootkube-start
-      Description=Bootstrap a Kubernetes cluster
+    permissions: '0755'
-      ConditionPathExists=!/opt/bootkube/init_bootkube.done
-      [Service]
-      Type=oneshot
-      RemainAfterExit=true
-      WorkingDirectory=/opt/bootkube
-      ExecStart=/opt/bootkube/bootkube-start
-      ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done
-      [Install]
-      WantedBy=multi-user.target
-  - path: /opt/bootkube/bootkube-start
-    permissions: '0544'
     content: |
       #!/bin/bash -e
       # Wrapper for bootkube start
-      [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
+      [ -n "$(ls /var/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /var/bootkube/assets/manifests-*/* /var/bootkube/assets/manifests && rm -rf /var/bootkube/assets/manifests-*
       /usr/bin/docker run --rm --name bootkube \
+        --net=host \
         --volume /etc/kubernetes:/etc/kubernetes:Z \
-        --volume /opt/bootkube/assets:/assets:Z \
+        --volume /var/bootkube/assets:/assets:Z \
         --entrypoint=/bootkube \
         quay.io/coreos/bootkube:v0.11.0 start --asset-dir=/assets
+bootcmd:
+  - [setenforce, Permissive]
 runcmd:
   - [systemctl, daemon-reload]
-  - [systemctl, enable, docker.service]
-  - [systemctl, start, --no-block, docker.service]
   - [systemctl, enable, etcd-member.service]
   - [systemctl, start, --no-block, etcd-member.service]
+  - [systemctl, disable, firewalld, --now]
   - [systemctl, enable, kubelet.service]
   - [systemctl, start, --no-block, kubelet.service]
 users:

aws/fedora-atomic/kubernetes/ssh.tf

@@ -82,7 +82,7 @@ resource "null_resource" "bootkube-start" {
   provisioner "remote-exec" {
     inline = [
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done",
-      "sudo mv $HOME/assets /opt/bootkube",
+      "sudo mv $HOME/assets /var/bootkube",
       "sudo systemctl start bootkube",
     ]
   }

aws/fedora-atomic/kubernetes/workers/ami.tf

@@ -14,6 +14,6 @@ data "aws_ami" "fedora" {
 
   filter {
     name   = "name"
-    values = ["Fedora-Cloud-Base-27*-gp2-0"]
+    values = ["Fedora-Atomic-27*-gp2-0"]
   }
 }

aws/fedora-atomic/kubernetes/workers/cloudinit/worker.yaml.tmpl

@@ -1,22 +1,12 @@
 #cloud-config
-yum_repos:
-  kubernetes:
-    name: kubernetes
-    baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
-    enabled: true
-    gpgcheck: true
-    gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-packages:
-  - [docker, 1.13.1]
-  - [kubelet, 1.10.0]
-  - nfs-utils
 write_files:
-  - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf
+  - path: /etc/systemd/system/kubelet.service
     content: |
       [Unit]
       Description=Kubelet
       Wants=rpc-statd.service
       [Service]
+      WorkingDirectory=/etc/kubernetes
       ExecStartPre=/bin/mkdir -p /opt/cni/bin
       ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
       ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -25,8 +15,11 @@ write_files:
       ExecStartPre=/bin/mkdir -p /var/lib/cni
       ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
       ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+      # Atomic's system containers and RPMs are old and unfriendly. Use this.
+      ExecStartPre=/usr/bin/curl -L https://dl.k8s.io/v1.10.0/kubernetes-node-linux-amd64.tar.gz -o kubernetes-node-linux-amd64.tar.gz
+      ExecStartPre=/usr/bin/tar xzf kubernetes-node-linux-amd64.tar.gz -C /usr/local/bin --strip-components=3 kubernetes/node/bin/kubelet 
       ExecStart=
-      ExecStart=/usr/bin/kubelet \
+      ExecStart=/usr/local/bin/kubelet \
         --allow-privileged \
         --anonymous-auth=false \
         --cgroup-driver=systemd \
@@ -45,19 +38,31 @@ write_files:
       RestartSec=10
       [Install]
       WantedBy=multi-user.target 
+  - path: /etc/systemd/system/kubelet.path
+    content: |
+      [Unit]
+      Description=Watch for kubeconfig
+      [Path]
+      PathExists=/etc/kubernetes/kubeconfig
+      [Install]
+      WantedBy=multi-user.target
+  - path: /etc/kubernetes/.keep
   - path: /etc/kubernetes/kubeconfig
     permissions: '0644'
     content: |
       ${kubeconfig}
   - path: /etc/selinux/config
+    owner: root:root
+    permissions: '0644'
     content: |
       SELINUX=permissive
+      SELINUXTYPE=targeted
+bootcmd:
+  - [setenforce, Permissive]
 runcmd:
   - [systemctl, daemon-reload]
-  - [systemctl, enable, docker.service]
+  - [systemctl, disable, firewalld, --now]
-  - [systemctl, start, --no-block, docker.service]
+  - [systemctl, enable, kubelet.service, --now]
-  - [systemctl, enable, kubelet.service]
-  - [systemctl, start, --no-block, kubelet.service]
 users:
   - default
   - name: fedora