From 2ff6d602d8614a1c7ac1a487d4f33515f49e80c2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Wed, 13 Sep 2017 20:08:28 -0700 Subject: [PATCH] digital-ocean: Distribute kubeconfig via Terraform null_resource * Keep kubeconfig out of DigitalOcean metadata user-data --- .../kubernetes/cl/controller.yaml.tmpl | 31 ++++++------------- .../kubernetes/cl/worker.yaml.tmpl | 31 ++++++------------- .../container-linux/kubernetes/controllers.tf | 4 --- .../container-linux/kubernetes/ssh.tf | 25 ++++++++++++++- .../container-linux/kubernetes/workers.tf | 4 --- 5 files changed, 42 insertions(+), 53 deletions(-) diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 2dae3bdf..2fedaff2 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -5,6 +5,15 @@ systemd: enable: true - name: locksmithd.service mask: true + - name: kubelet.path + enable: true + contents: | + [Unit] + Description=Watch for kubeconfig + [Path] + PathExists=/etc/kubernetes/kubeconfig + [Install] + WantedBy=multi-user.target - name: wait-for-dns.service enable: true contents: | @@ -19,7 +28,6 @@ systemd: [Install] RequiredBy=kubelet.service - name: kubelet.service - enable: true contents: | [Unit] Description=Kubelet via Hyperkube ACI @@ -78,27 +86,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubeconfig - filesystem: root - mode: 0644 - contents: - inline: | - apiVersion: v1 - kind: Config - clusters: - - name: local - cluster: - server: ${kubeconfig_server} - certificate-authority-data: ${kubeconfig_ca_cert} - users: - - name: kubelet - user: - client-certificate-data: ${kubeconfig_kubelet_cert} - client-key-data: ${kubeconfig_kubelet_key} - contexts: - - context: - cluster: local - user: kubelet - path: /etc/kubernetes/kubelet.env filesystem: root mode: 0644 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl index e3ed68bb..ff22082b 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -5,6 +5,15 @@ systemd: enable: true - name: locksmithd.service mask: true + - name: kubelet.path + enable: true + contents: | + [Unit] + Description=Watch for kubeconfig + [Path] + PathExists=/etc/kubernetes/kubeconfig + [Install] + WantedBy=multi-user.target - name: wait-for-dns.service enable: true contents: | @@ -19,7 +28,6 @@ systemd: [Install] RequiredBy=kubelet.service - name: kubelet.service - enable: true contents: | [Unit] Description=Kubelet via Hyperkube ACI @@ -76,27 +84,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubeconfig - filesystem: root - mode: 0644 - contents: - inline: | - apiVersion: v1 - kind: Config - clusters: - - name: local - cluster: - server: ${kubeconfig_server} - certificate-authority-data: ${kubeconfig_ca_cert} - users: - - name: kubelet - user: - client-certificate-data: ${kubeconfig_kubelet_cert} - client-key-data: ${kubeconfig_kubelet_key} - contexts: - - context: - cluster: local - user: kubelet - path: /etc/kubernetes/kubelet.env filesystem: root mode: 0644 diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index d1d3fbc2..c0279e03 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -48,10 +48,6 @@ data "template_file" "controller_config" { vars = { k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" - kubeconfig_ca_cert = "${module.bootkube.ca_cert}" - kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" - kubeconfig_kubelet_key = "${module.bootkube.kubelet_key}" - kubeconfig_server = "${module.bootkube.server}" } } diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index d2b9564c..6e270916 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -1,7 +1,30 @@ +# Secure copy kubeconfig to all nodes. Activates kubelet.service +resource "null_resource" "copy-secrets" { + count = "${var.controller_count + var.worker_count}" + + connection { + type = "ssh" + host = "${element(concat(digitalocean_droplet.controllers.*.ipv4_address, digitalocean_droplet.workers.*.ipv4_address), count.index)}" + user = "core" + timeout = "15m" + } + + provisioner "file" { + content = "${module.bootkube.kubeconfig}" + destination = "$HOME/kubeconfig" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig", + ] + } +} + # Secure copy bootkube assets to ONE controller and start bootkube to perform # one-time self-hosted cluster bootstrapping. resource "null_resource" "bootkube-start" { - depends_on = ["module.bootkube", "digitalocean_droplet.controllers"] + depends_on = ["module.bootkube", "null_resource.copy-secrets", "digitalocean_droplet.controllers"] connection { type = "ssh" diff --git a/digital-ocean/container-linux/kubernetes/workers.tf b/digital-ocean/container-linux/kubernetes/workers.tf index 976d489c..1f33a418 100644 --- a/digital-ocean/container-linux/kubernetes/workers.tf +++ b/digital-ocean/container-linux/kubernetes/workers.tf @@ -45,10 +45,6 @@ data "template_file" "worker_config" { vars = { k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" - kubeconfig_ca_cert = "${module.bootkube.ca_cert}" - kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" - kubeconfig_kubelet_key = "${module.bootkube.kubelet_key}" - kubeconfig_server = "${module.bootkube.server}" } }