digital-ocean: Distribute kubeconfig via Terraform null_resource

* Keep kubeconfig out of DigitalOcean metadata user-data

digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -5,6 +5,15 @@ systemd:
       enable: true
     - name: locksmithd.service
       mask: true
+    - name: kubelet.path
+      enable: true
+      contents: |
+        [Unit]
+        Description=Watch for kubeconfig
+        [Path]
+        PathExists=/etc/kubernetes/kubeconfig
+        [Install]
+        WantedBy=multi-user.target
     - name: wait-for-dns.service
       enable: true
       contents: |
@@ -19,7 +28,6 @@ systemd:
         [Install]
         RequiredBy=kubelet.service
     - name: kubelet.service
-      enable: true
       contents: |
         [Unit]
         Description=Kubelet via Hyperkube ACI
@@ -78,27 +86,6 @@ systemd:
         WantedBy=multi-user.target
 storage:
   files:
-    - path: /etc/kubernetes/kubeconfig
-      filesystem: root
-      mode: 0644
-      contents:
-        inline: |
-          apiVersion: v1
-          kind: Config
-          clusters:
-          - name: local
-            cluster:
-              server: ${kubeconfig_server}
-              certificate-authority-data: ${kubeconfig_ca_cert}
-          users:
-          - name: kubelet
-            user:
-              client-certificate-data: ${kubeconfig_kubelet_cert}
-              client-key-data: ${kubeconfig_kubelet_key}
-          contexts:
-          - context:
-              cluster: local
-              user: kubelet
     - path: /etc/kubernetes/kubelet.env
       filesystem: root
       mode: 0644

digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -5,6 +5,15 @@ systemd:
       enable: true
     - name: locksmithd.service
       mask: true
+    - name: kubelet.path
+      enable: true
+      contents: |
+        [Unit]
+        Description=Watch for kubeconfig
+        [Path]
+        PathExists=/etc/kubernetes/kubeconfig
+        [Install]
+        WantedBy=multi-user.target
     - name: wait-for-dns.service
       enable: true
       contents: |
@@ -19,7 +28,6 @@ systemd:
         [Install]
         RequiredBy=kubelet.service
     - name: kubelet.service
-      enable: true
       contents: |
         [Unit]
         Description=Kubelet via Hyperkube ACI
@@ -76,27 +84,6 @@ systemd:
         WantedBy=multi-user.target
 storage:
   files:
-    - path: /etc/kubernetes/kubeconfig
-      filesystem: root
-      mode: 0644
-      contents:
-        inline: |
-          apiVersion: v1
-          kind: Config
-          clusters:
-          - name: local
-            cluster:
-              server: ${kubeconfig_server}
-              certificate-authority-data: ${kubeconfig_ca_cert}
-          users:
-          - name: kubelet
-            user:
-              client-certificate-data: ${kubeconfig_kubelet_cert}
-              client-key-data: ${kubeconfig_kubelet_key}
-          contexts:
-          - context:
-              cluster: local
-              user: kubelet
     - path: /etc/kubernetes/kubelet.env
       filesystem: root
       mode: 0644

digital-ocean/container-linux/kubernetes/controllers.tf

@@ -48,10 +48,6 @@ data "template_file" "controller_config" {
   vars = {
     k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
     k8s_etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
-    kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
-    kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
-    kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"
-    kubeconfig_server       = "${module.bootkube.server}"
   }
 }
 

digital-ocean/container-linux/kubernetes/ssh.tf

@@ -1,7 +1,30 @@
+# Secure copy kubeconfig to all nodes. Activates kubelet.service
+resource "null_resource" "copy-secrets" {
+  count = "${var.controller_count + var.worker_count}"
+
+  connection {
+    type    = "ssh"
+    host    = "${element(concat(digitalocean_droplet.controllers.*.ipv4_address, digitalocean_droplet.workers.*.ipv4_address), count.index)}"
+    user    = "core"
+    timeout = "15m"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.kubeconfig}"
+    destination = "$HOME/kubeconfig"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
+    ]
+  }
+}
+  
 # Secure copy bootkube assets to ONE controller and start bootkube to perform
 # one-time self-hosted cluster bootstrapping.
 resource "null_resource" "bootkube-start" {
-  depends_on = ["module.bootkube", "digitalocean_droplet.controllers"]
+  depends_on = ["module.bootkube", "null_resource.copy-secrets", "digitalocean_droplet.controllers"]
 
   connection {
     type    = "ssh"

digital-ocean/container-linux/kubernetes/workers.tf

@@ -45,10 +45,6 @@ data "template_file" "worker_config" {
   vars = {
     k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
     k8s_etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
-    kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
-    kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
-    kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"
-    kubeconfig_server       = "${module.bootkube.server}"
   }
 }