Run etcd cluster on-host, across controllers on AWS

* Change controllers ASG to heterogeneous EC2 instances
* Create DNS records for each controller's private IP for etcd
* Change etcd to run on-host, across controllers (etcd-member.service)
* Reduce time to bootstrap a cluster
* Deprecate self-hosted-etcd on the AWS platform

aws/container-linux/kubernetes/bootkube.tf

@@ -2,13 +2,12 @@
 module "bootkube" {
   source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.8.1"
 
-  cluster_name                  = "${var.cluster_name}"
-  api_servers                   = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
-  etcd_servers                  = ["http://127.0.0.1:2379"]
-  asset_dir                     = "${var.asset_dir}"
-  networking                    = "${var.networking}"
-  network_mtu                   = "${var.network_mtu}"
-  pod_cidr                      = "${var.pod_cidr}"
-  service_cidr                  = "${var.service_cidr}"
-  experimental_self_hosted_etcd = "true"
+  cluster_name = "${var.cluster_name}"
+  api_servers  = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
+  etcd_servers = ["${aws_route53_record.etcds.*.fqdn}"]
+  asset_dir    = "${var.asset_dir}"
+  networking   = "${var.networking}"
+  network_mtu  = "${var.network_mtu}"
+  pod_cidr     = "${var.pod_cidr}"
+  service_cidr = "${var.service_cidr}"
 }

aws/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -1,6 +1,29 @@
 ---
 systemd:
   units:
+    - name: etcd-member.service
+      enable: true
+      dropins:
+        - name: 40-etcd-cluster.conf
+          contents: |
+            [Service]
+            Environment="ETCD_IMAGE_TAG=v3.2.0"
+            Environment="ETCD_NAME=${etcd_name}"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
+            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
+            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
+            Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
+            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
+            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
+            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
+            Environment="ETCD_CLIENT_CERT_AUTH=true"
+            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
+            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
+            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
+            Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
     - name: docker.service
       enable: true
     - name: locksmithd.service

aws/container-linux/kubernetes/controllers.tf

@@ -1,39 +1,30 @@
-# Controllers AutoScaling Group
-resource "aws_autoscaling_group" "controllers" {
-  name           = "${var.cluster_name}-controller"
-  load_balancers = ["${aws_elb.controllers.id}"]
+# Discrete DNS records for each controller's private IPv4 for etcd usage
+resource "aws_route53_record" "etcds" {
+  count = "${var.controller_count}"
 
-  # count
-  desired_capacity = "${var.controller_count}"
-  min_size         = "${var.controller_count}"
-  max_size         = "${var.controller_count}"
+  # DNS Zone where record should be created
+  zone_id = "${var.dns_zone_id}"
 
-  # network
-  vpc_zone_identifier = ["${aws_subnet.public.*.id}"]
+  name = "${format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)}"
+  type = "A"
+  ttl  = 300
 
-  # template
-  launch_configuration = "${aws_launch_configuration.controller.name}"
-
-  lifecycle {
-    # override the default destroy and replace update behavior
-    create_before_destroy = true
-    ignore_changes        = ["image_id"]
-  }
-
-  tags = [{
-    key                 = "Name"
-    value               = "${var.cluster_name}-controller"
-    propagate_at_launch = true
-  }]
+  # private IPv4 address for etcd
+  records = ["${element(aws_instance.controllers.*.private_ip, count.index)}"]
 }
 
-# Controller template
-resource "aws_launch_configuration" "controller" {
-  name_prefix   = "${var.cluster_name}-controller-template-"
-  image_id      = "${data.aws_ami.coreos.image_id}"
+# Controller instances
+resource "aws_instance" "controllers" {
+  count = "${var.controller_count}"
+
+  tags = {
+    Name = "${var.cluster_name}-controller-${count.index}"
+  }
+
   instance_type = "${var.controller_type}"
 
-  user_data = "${data.ct_config.controller_ign.rendered}"
+  ami       = "${data.aws_ami.coreos.image_id}"
+  user_data = "${element(data.ct_config.controller_ign.*.rendered, count.index)}"
 
   # storage
   root_block_device {
@@ -43,21 +34,25 @@ resource "aws_launch_configuration" "controller" {
 
   # network
   associate_public_ip_address = true
-  security_groups             = ["${aws_security_group.controller.id}"]
-
-  lifecycle {
-    // Override the default destroy and replace update behavior
-    create_before_destroy = true
-  }
+  subnet_id                   = "${element(aws_subnet.public.*.id, count.index)}"
+  vpc_security_group_ids      = ["${aws_security_group.controller.id}"]
 }
 
 # Controller Container Linux Config
 data "template_file" "controller_config" {
+  count = "${var.controller_count}"
+
   template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
 
   vars = {
+    # Cannot use cyclic dependencies on controllers or their DNS records
+    etcd_name   = "etcd${count.index}"
+    etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
+
+    # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
+    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
+
     k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
-    k8s_etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
     ssh_authorized_key      = "${var.ssh_authorized_key}"
     kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
     kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
@@ -66,8 +61,20 @@ data "template_file" "controller_config" {
   }
 }
 
+# Horrible hack to generate a Terraform list of a desired length without dependencies.
+# Ideal ${repeat("etcd", 3) -> ["etcd", "etcd", "etcd"]}
+resource null_resource "repeat" {
+  count = "${var.controller_count}"
+
+  triggers {
+    name   = "etcd${count.index}"
+    domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
+  }
+}
+
 data "ct_config" "controller_ign" {
-  content      = "${data.template_file.controller_config.rendered}"
+  count        = "${var.controller_count}"
+  content      = "${element(data.template_file.controller_config.*.rendered, count.index)}"
   pretty_print = false
 }
 

aws/container-linux/kubernetes/elb.tf

@@ -19,13 +19,6 @@ resource "aws_elb" "controllers" {
   subnets         = ["${aws_subnet.public.*.id}"]
   security_groups = ["${aws_security_group.controller.id}"]
 
-  listener {
-    lb_port           = 22
-    lb_protocol       = "tcp"
-    instance_port     = 22
-    instance_protocol = "tcp"
-  }
-
   listener {
     lb_port           = 443
     lb_protocol       = "tcp"
@@ -33,6 +26,8 @@ resource "aws_elb" "controllers" {
     instance_protocol = "tcp"
   }
 
+  instances = ["${aws_instance.controllers.*.id}"]
+
   # Kubelet HTTP health check
   health_check {
     target              = "HTTP:10255/healthz"
@@ -42,7 +37,7 @@ resource "aws_elb" "controllers" {
     interval            = 6
   }
 
-  idle_timeout                = 1800
+  idle_timeout                = 3600
   connection_draining         = true
   connection_draining_timeout = 300
 }

aws/container-linux/kubernetes/ssh.tf

@@ -1,12 +1,79 @@
+# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service
+resource "null_resource" "copy-secrets" {
+  count = "${var.controller_count}"
+
+  connection {
+    type    = "ssh"
+    host    = "${element(aws_instance.controllers.*.public_ip, count.index)}"
+    user    = "core"
+    timeout = "15m"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.kubeconfig}"
+    destination = "$HOME/kubeconfig"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_ca_cert}"
+    destination = "$HOME/etcd-client-ca.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_client_cert}"
+    destination = "$HOME/etcd-client.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_client_key}"
+    destination = "$HOME/etcd-client.key"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_server_cert}"
+    destination = "$HOME/etcd-server.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_server_key}"
+    destination = "$HOME/etcd-server.key"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_peer_cert}"
+    destination = "$HOME/etcd-peer.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_peer_key}"
+    destination = "$HOME/etcd-peer.key"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo mkdir -p /etc/ssl/etcd/etcd",
+      "sudo mv etcd-client* /etc/ssl/etcd/",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
+      "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
+      "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
+      "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
+      "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
+      "sudo chown -R etcd:etcd /etc/ssl/etcd",
+      "sudo chmod -R 500 /etc/ssl/etcd",
+      "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
+    ]
+  }
+}
+
 # Secure copy bootkube assets to ONE controller and start bootkube to perform
 # one-time self-hosted cluster bootstrapping.
 resource "null_resource" "bootkube-start" {
-  depends_on = ["module.bootkube", "aws_autoscaling_group.controllers"]
+  depends_on = ["module.bootkube", "null_resource.copy-secrets", "aws_route53_record.controllers"]
 
-  # TODO: SSH to a controller's IP instead of waiting on DNS resolution
   connection {
     type    = "ssh"
-    host    = "${aws_route53_record.controllers.fqdn}"
+    host    = "${aws_instance.controllers.0.public_ip}"
     user    = "core"
     timeout = "15m"
   }