From 7acd4931f6129d1192e883faf5f0f3bc5f7c2b22 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 26 Mar 2018 00:01:47 -0700 Subject: [PATCH] Remove redundant kubeconfig copy on AWS and GCP * AWS and Google Cloud make use of auto-scaling groups and managed instance groups, respectively. As such, the kubeconfig is already held in cloud user-data * Controller instances are provisioned with a kubeconfig from user-data. Its redundant to use a Terraform remote file copy step for the kubeconfig. --- aws/container-linux/kubernetes/controllers.tf | 6 +++--- aws/container-linux/kubernetes/ssh.tf | 19 +++++++++--------- .../container-linux/kubernetes/ssh.tf | 2 +- .../kubernetes/controllers/controllers.tf | 4 ++-- .../container-linux/kubernetes/ssh.tf | 20 +++++++++---------- .../container-linux/kubernetes/variables.tf | 8 ++++---- 6 files changed, 28 insertions(+), 31 deletions(-) diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 6f52b295..56f42800 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -56,10 +56,10 @@ data "template_file" "controller_config" { # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}" - k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" kubeconfig = "${indent(10, module.bootkube.kubeconfig)}" + ssh_authorized_key = "${var.ssh_authorized_key}" + k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" + cluster_domain_suffix = "${var.cluster_domain_suffix}" } } diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 4b58c297..111bbeaa 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -1,5 +1,5 @@ -# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service -resource "null_resource" "copy-secrets" { +# Secure copy etcd TLS assets to controllers. +resource "null_resource" "copy-controller-secrets" { count = "${var.controller_count}" connection { @@ -9,11 +9,6 @@ resource "null_resource" "copy-secrets" { timeout = "15m" } - provisioner "file" { - content = "${module.bootkube.kubeconfig}" - destination = "$HOME/kubeconfig" - } - provisioner "file" { content = "${module.bootkube.etcd_ca_cert}" destination = "$HOME/etcd-client-ca.crt" @@ -61,7 +56,6 @@ resource "null_resource" "copy-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig", ] } } @@ -69,7 +63,12 @@ resource "null_resource" "copy-secrets" { # Secure copy bootkube assets to ONE controller and start bootkube to perform # one-time self-hosted cluster bootstrapping. resource "null_resource" "bootkube-start" { - depends_on = ["module.bootkube", "null_resource.copy-secrets", "aws_route53_record.apiserver"] + depends_on = [ + "module.bootkube", + "module.workers", + "aws_route53_record.apiserver", + "null_resource.copy-controller-secrets", + ] connection { type = "ssh" @@ -85,7 +84,7 @@ resource "null_resource" "bootkube-start" { provisioner "remote-exec" { inline = [ - "sudo mv /home/core/assets /opt/bootkube", + "sudo mv $HOME/assets /opt/bootkube", "sudo systemctl start bootkube", ] } diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 4bb81e2e..0dc9c396 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -81,7 +81,7 @@ resource "null_resource" "copy-worker-secrets" { content = "${module.bootkube.kubeconfig}" destination = "$HOME/kubeconfig" } - + provisioner "remote-exec" { inline = [ "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", diff --git a/google-cloud/container-linux/kubernetes/controllers/controllers.tf b/google-cloud/container-linux/kubernetes/controllers/controllers.tf index df6d683d..d1821d76 100644 --- a/google-cloud/container-linux/kubernetes/controllers/controllers.tf +++ b/google-cloud/container-linux/kubernetes/controllers/controllers.tf @@ -65,10 +65,10 @@ data "template_file" "controller_config" { # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}" + kubeconfig = "${indent(10, var.kubeconfig)}" + ssh_authorized_key = "${var.ssh_authorized_key}" k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" cluster_domain_suffix = "${var.cluster_domain_suffix}" - ssh_authorized_key = "${var.ssh_authorized_key}" - kubeconfig = "${indent(10, var.kubeconfig)}" } } diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index a3179516..c1991bfd 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -1,6 +1,5 @@ -# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service -resource "null_resource" "copy-secrets" { - depends_on = ["module.bootkube"] +# Secure copy etcd TLS assets to controllers. +resource "null_resource" "copy-controller-secrets" { count = "${var.controller_count}" connection { @@ -10,11 +9,6 @@ resource "null_resource" "copy-secrets" { timeout = "15m" } - provisioner "file" { - content = "${module.bootkube.kubeconfig}" - destination = "$HOME/kubeconfig" - } - provisioner "file" { content = "${module.bootkube.etcd_ca_cert}" destination = "$HOME/etcd-client-ca.crt" @@ -62,7 +56,6 @@ resource "null_resource" "copy-secrets" { "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", "sudo chown -R etcd:etcd /etc/ssl/etcd", "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig", ] } } @@ -70,7 +63,12 @@ resource "null_resource" "copy-secrets" { # Secure copy bootkube assets to ONE controller and start bootkube to perform # one-time self-hosted cluster bootstrapping. resource "null_resource" "bootkube-start" { - depends_on = ["module.controllers", "module.bootkube", "module.workers", "null_resource.copy-secrets"] + depends_on = [ + "module.bootkube", + "module.controllers", + "module.workers", + "null_resource.copy-controller-secrets", + ] connection { type = "ssh" @@ -86,7 +84,7 @@ resource "null_resource" "bootkube-start" { provisioner "remote-exec" { inline = [ - "sudo mv /home/core/assets /opt/bootkube", + "sudo mv $HOME/assets /opt/bootkube", "sudo systemctl start bootkube", ] } diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf index ab589c5a..abcaec84 100644 --- a/google-cloud/container-linux/kubernetes/variables.tf +++ b/google-cloud/container-linux/kubernetes/variables.tf @@ -35,14 +35,14 @@ variable "worker_count" { } variable controller_type { - type = "string" - default = "n1-standard-1" + type = "string" + default = "n1-standard-1" description = "Machine type for controllers (see `gcloud compute machine-types list`)" } variable worker_type { - type = "string" - default = "n1-standard-1" + type = "string" + default = "n1-standard-1" description = "Machine type for controllers (see `gcloud compute machine-types list`)" }