Change DO Fedora module to fedora-atomic

digital-ocean/fedora-atomic/kubernetes/README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features <a href="https://www.cncf.io/certification/software-conformance/"><img align="right" src="https://storage.googleapis.com/poseidon/certified-kubernetes.png"></a>
 
-* Kubernetes v1.9.6 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.10.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/)

digital-ocean/fedora-atomic/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5f3546b66ffb9946b36e612537bb6a1830ae7746"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=61fb176647e15d4d0e72fdccb34d27e47430407c"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
@@ -11,4 +11,7 @@ module "bootkube" {
   pod_cidr              = "${var.pod_cidr}"
   service_cidr          = "${var.service_cidr}"
   cluster_domain_suffix = "${var.cluster_domain_suffix}"
+
+  # Fedora
+  trusted_certs_dir = "/etc/pki/tls/certs"
 }

digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl

@@ -1,29 +1,5 @@
 #cloud-config
-yum_repos:
-  kubernetes:
-    name: kubernetes
-    baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
-    enabled: true
-    gpgcheck: true
-    gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-packages:
-  - [docker, 1.13.1]
-  - [kubelet, 1.10.0]
-  - nfs-utils
 write_files:
-  - path: /etc/systemd/system/cloud-metadata.service
-    content: |
-      [Unit]
-      Description=Digital Ocean metadata agent
-      [Service]
-      Type=oneshot
-      Environment=OUTPUT=/run/metadata/digitalocean
-      ExecStart=/usr/bin/mkdir -p /run/metadata
-      ExecStart=/usr/bin/bash -c 'echo "DIGITALOCEAN_IPV4_PUBLIC_0=$(curl\
-        --url http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address\
-        --retry 10)\nDIGITALOCEAN_IPV4_PRIVATE_0=$(curl\
-        --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\
-        --retry 10)" > $${OUTPUT}'
   - path: /etc/systemd/system/etcd-member.service
     content: |
       [Unit]
@@ -38,7 +14,7 @@ write_files:
       ExecStartPre=/bin/mkdir -p /var/lib/etcd
       ExecStart=/usr/bin/docker run --rm --name etcd-member \
         --net=host \
-        -v /usr/share/ca-certificates:/usr/share/ca-certificates:ro,z \
+        -v /etc/pki/tls/certs:/usr/share/ca-certificates:ro,z \
         -v /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
         -v /var/lib/etcd:/var/lib/etcd:Z \
         --env-file=/etc/etcd/etcd.conf \
@@ -68,7 +44,20 @@ write_files:
       ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
       ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
       ETCD_PEER_CLIENT_CERT_AUTH=true
-  - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf
+  - path: /etc/systemd/system/cloud-metadata.service
+    content: |
+      [Unit]
+      Description=Digital Ocean metadata agent
+      [Service]
+      Type=oneshot
+      Environment=OUTPUT=/run/metadata/digitalocean
+      ExecStart=/usr/bin/mkdir -p /run/metadata
+      ExecStart=/usr/bin/bash -c 'echo "DIGITALOCEAN_IPV4_PUBLIC_0=$(curl\
+        --url http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address\
+        --retry 10)\nDIGITALOCEAN_IPV4_PRIVATE_0=$(curl\
+        --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\
+        --retry 10)" > $${OUTPUT}'
+  - path: /etc/systemd/system/kubelet.service
     content: |
       [Unit]
       Description=Kubelet
@@ -76,6 +65,7 @@ write_files:
       After=cloud-metadata.service
       Wants=rpc-statd.service
       [Service]
+      WorkingDirectory=/etc/kubernetes
       EnvironmentFile=/run/metadata/digitalocean
       ExecStartPre=/bin/mkdir -p /opt/cni/bin
       ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -85,8 +75,11 @@ write_files:
       ExecStartPre=/bin/mkdir -p /var/lib/cni
       ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
       ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+      # Atomic's system containers and RPMs are old and unfriendly. Use this.
+      ExecStartPre=/usr/bin/curl -L https://dl.k8s.io/v1.10.0/kubernetes-node-linux-amd64.tar.gz -o kubernetes-node-linux-amd64.tar.gz
+      ExecStartPre=/usr/bin/tar xzf kubernetes-node-linux-amd64.tar.gz -C /usr/local/bin --strip-components=3 kubernetes/node/bin/kubelet 
       ExecStart=
-      ExecStart=/usr/bin/kubelet \
+      ExecStart=/usr/local/bin/kubelet \
         --allow-privileged \
         --anonymous-auth=false \
         --cgroup-driver=systemd \
@@ -116,43 +109,50 @@ write_files:
       PathExists=/etc/kubernetes/kubeconfig
       [Install]
       WantedBy=multi-user.target
-  - path: /etc/kubernetes/.keep
-  - path: /etc/selinux/config
-    content: |
-      SELINUX=permissive
   - path: /etc/systemd/system/bootkube.service
     content: |
       [Unit]
       Description=Bootstrap a Kubernetes cluster
-      ConditionPathExists=!/opt/bootkube/init_bootkube.done
+      ConditionPathExists=!/var/bootkube/init_bootkube.done
       [Service]
       Type=oneshot
       RemainAfterExit=true
-      WorkingDirectory=/opt/bootkube
-      ExecStart=/opt/bootkube/bootkube-start
-      ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done
+      WorkingDirectory=/var/bootkube
+      ExecStartPre=/bin/mkdir -p /var/bootkube
+      ExecStart=/usr/local/bin/bootkube-start
+      ExecStartPost=/bin/touch /var/bootkube/init_bootkube.done
       [Install]
       WantedBy=multi-user.target
-  - path: /opt/bootkube/bootkube-start
-    permissions: '0544'
+  - path: /etc/kubernetes/.keep
+  - path: /var/bootkube/.keep
+  - path: /etc/selinux/config
+    owner: root:root
+    permissions: '0644'
+    content: |
+      SELINUX=permissive
+      SELINUXTYPE=targeted
+  - path: /usr/local/bin/bootkube-start
+    permissions: '0755'
     content: |
       #!/bin/bash -e
       # Wrapper for bootkube start
-      [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
+      [ -n "$(ls /var/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /var/bootkube/assets/manifests-*/* /var/bootkube/assets/manifests && rm -rf /var/bootkube/assets/manifests-*
       /usr/bin/docker run --rm --name bootkube \
+        --net=host \
         --volume /etc/kubernetes:/etc/kubernetes:Z \
-        --volume /opt/bootkube/assets:/assets:Z \
+        --volume /var/bootkube/assets:/assets:Z \
         --entrypoint=/bootkube \
         quay.io/coreos/bootkube:v0.11.0 start --asset-dir=/assets
+bootcmd:
+  - [setenforce, Permissive]
 runcmd:
   - [systemctl, daemon-reload]
-  - [systemctl, enable, docker.service]
-  - [systemctl, start, --no-block, docker.service]
   - [systemctl, enable, etcd-member.service]
   - [systemctl, start, --no-block, etcd-member.service]
   - [systemctl, enable, cloud-metadata.service]
   - [systemctl, enable, kubelet.path]
   - [systemctl, start, --no-block, kubelet.path]
+  - [systemctl, disable, firewalld, --now]
 users:
   - default
   - name: fedora

digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl

@@ -1,15 +1,4 @@
 #cloud-config
-yum_repos:
-  kubernetes:
-    name: kubernetes
-    baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
-    enabled: true
-    gpgcheck: true
-    gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-packages:
-  - [docker, 1.13.1]
-  - [kubelet, 1.10.0]
-  - nfs-utils
 write_files:
   - path: /etc/systemd/system/cloud-metadata.service
     content: |
@@ -24,7 +13,7 @@ write_files:
         --retry 10)\nDIGITALOCEAN_IPV4_PRIVATE_0=$(curl\
         --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\
         --retry 10)" > $${OUTPUT}'
-  - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf
+  - path: /etc/systemd/system/kubelet.service
     content: |
       [Unit]
       Description=Kubelet
@@ -32,6 +21,7 @@ write_files:
       After=cloud-metadata.service
       Wants=rpc-statd.service
       [Service]
+      WorkingDirectory=/etc/kubernetes
       EnvironmentFile=/run/metadata/digitalocean
       ExecStartPre=/bin/mkdir -p /opt/cni/bin
       ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
@@ -41,8 +31,11 @@ write_files:
       ExecStartPre=/bin/mkdir -p /var/lib/cni
       ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
       ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+      # Atomic's system containers and RPMs are old and unfriendly. Use this.
+      ExecStartPre=/usr/bin/curl -L https://dl.k8s.io/v1.10.0/kubernetes-node-linux-amd64.tar.gz -o kubernetes-node-linux-amd64.tar.gz
+      ExecStartPre=/usr/bin/tar xzf kubernetes-node-linux-amd64.tar.gz -C /usr/local/bin --strip-components=3 kubernetes/node/bin/kubelet 
       ExecStart=
-      ExecStart=/usr/bin/kubelet \
+      ExecStart=/usr/local/bin/kubelet \
         --allow-privileged \
         --anonymous-auth=false \
         --cgroup-driver=systemd \
@@ -72,15 +65,19 @@ write_files:
       WantedBy=multi-user.target
   - path: /etc/kubernetes/.keep
   - path: /etc/selinux/config
+    owner: root:root
+    permissions: '0644'
     content: |
       SELINUX=permissive
+      SELINUXTYPE=targeted
+bootcmd:
+  - [setenforce, Permissive]
 runcmd:
   - [systemctl, daemon-reload]
-  - [systemctl, enable, docker.service]
-  - [systemctl, start, --no-block, docker.service]
-  - [systemctl, enable, cloud-metadata.service]
   - [systemctl, enable, kubelet.path]
+  - [systemctl, enable, cloud-metadata.service]
   - [systemctl, start, --no-block, kubelet.path]
+  - [systemctl, disable, firewalld, --now]
 users:
   - default
   - name: fedora

digital-ocean/fedora-atomic/kubernetes/ssh.tf

@@ -110,7 +110,7 @@ resource "null_resource" "bootkube-start" {
   provisioner "remote-exec" {
     inline = [
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done",
-      "sudo mv $HOME/assets /opt/bootkube",
+      "sudo mv $HOME/assets /var/bootkube",
       "sudo systemctl start bootkube",
     ]
   }

digital-ocean/fedora-atomic/kubernetes/variables.tf

@@ -43,8 +43,8 @@ variable "worker_type" {
 
 variable "image" {
   type        = "string"
-  default     = "fedora-27-x64"
-  description = "OS image from which to initialize the disk (e.g. fedora-27-x64)"
+  default     = "fedora-27-x64-atomic"
+  description = "OS image from which to initialize the disk (e.g. fedora-27-x64-atomic)"
 }
 
 # configuration