digital-ocean: Run etcd cluster on-host, across controllers

* Run etcd peers with TLS across controller nodes
* Deprecate self-hosted-etcd on the Digital Ocean platform
* Distribute etcd TLS certificates as part of initial provisioning
* Check the status of etcd by running `systemctl status etcd-member`

CHANGES.md

@@ -2,6 +2,14 @@
 
 Notable changes between versions.
 
+## Latest
+
+#### Digital Ocean
+
+* Run etcd cluster across controller nodes (etcd-member.service)
+* Reduce time to bootstrap a cluster
+* Remove support for self-hosted etcd
+
 ## v1.7.7
 
 * Kubernetes v1.7.7

digital-ocean/container-linux/kubernetes/bootkube.tf

@@ -4,11 +4,10 @@ module "bootkube" {
 
   cluster_name = "${var.cluster_name}"
   api_servers  = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
-  etcd_servers                  = ["http://127.0.0.1:2379"]
+  etcd_servers = "${digitalocean_record.etcds.*.fqdn}"
   asset_dir    = "${var.asset_dir}"
   networking   = "${var.networking}"
   network_mtu  = 1440
   pod_cidr     = "${var.pod_cidr}"
   service_cidr = "${var.service_cidr}"
-  experimental_self_hosted_etcd = "true"
 }

digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -1,6 +1,29 @@
 ---
 systemd:
   units:
+    - name: etcd-member.service
+      enable: true
+      dropins:
+        - name: 40-etcd-cluster.conf
+          contents: |
+            [Service]
+            Environment="ETCD_IMAGE_TAG=v3.2.0"
+            Environment="ETCD_NAME=${etcd_name}"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
+            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
+            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
+            Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
+            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
+            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
+            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
+            Environment="ETCD_CLIENT_CERT_AUTH=true"
+            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
+            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
+            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
+            Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
     - name: docker.service
       enable: true
     - name: locksmithd.service

digital-ocean/container-linux/kubernetes/controllers.tf

@@ -14,6 +14,21 @@ resource "digitalocean_record" "controllers" {
   value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}"
 }
 
+resource "digitalocean_record" "etcds" {
+  count = "${var.controller_count}"
+
+  # DNS zone where record should be created
+  domain = "${var.dns_zone}"
+
+  # DNS record (will be prepended to domain)
+  name = "${var.cluster_name}-etcd${count.index}"
+  type = "A"
+  ttl  = 300
+
+  # IPv4 addresses of controllers
+  value = "${element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index)}"
+}
+
 # Controller droplet instances
 resource "digitalocean_droplet" "controllers" {
   count = "${var.controller_count}"
@@ -28,7 +43,7 @@ resource "digitalocean_droplet" "controllers" {
   ipv6               = true
   private_networking = true
 
-  user_data = "${data.ct_config.controller_ign.rendered}"
+  user_data = "${element(data.ct_config.controller_ign.*.rendered, count.index)}"
   ssh_keys  = "${var.ssh_fingerprints}"
 
   tags = [
@@ -43,15 +58,33 @@ resource "digitalocean_tag" "controllers" {
 
 # Controller Container Linux Config
 data "template_file" "controller_config" {
+  count = "${var.controller_count}"
+
   template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
 
   vars = {
+    # Cannot use cyclic dependencies on controllers or their DNS records
+    etcd_name            = "etcd${count.index}"
+    etcd_domain          = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
+    # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
+    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
     k8s_dns_service_ip   = "${cidrhost(var.service_cidr, 10)}"
-    k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
+  }
+}
+
+# Horrible hack to generate a Terraform list of a desired length without dependencies.
+# Ideal ${repeat("etcd", 3) -> ["etcd", "etcd", "etcd"]}
+resource null_resource "repeat" {
+  count = "${var.controller_count}"
+
+  triggers {
+    name   = "etcd${count.index}"
+    domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
   }
 }
 
 data "ct_config" "controller_ign" {
-  content      = "${data.template_file.controller_config.rendered}"
+  count        = "${var.controller_count}"
+  content      = "${element(data.template_file.controller_config.*.rendered, count.index)}"
   pretty_print = false
 }

digital-ocean/container-linux/kubernetes/ssh.tf

@@ -14,8 +14,53 @@ resource "null_resource" "copy-secrets" {
     destination = "$HOME/kubeconfig"
   }
 
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_ca_cert}"
+    destination = "$HOME/etcd-client-ca.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_client_cert}"
+    destination = "$HOME/etcd-client.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_client_key}"
+    destination = "$HOME/etcd-client.key"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_server_cert}"
+    destination = "$HOME/etcd-server.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_server_key}"
+    destination = "$HOME/etcd-server.key"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_peer_cert}"
+    destination = "$HOME/etcd-peer.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_peer_key}"
+    destination = "$HOME/etcd-peer.key"
+  }
+
   provisioner "remote-exec" {
     inline = [
+      "sudo mkdir -p /etc/ssl/etcd/etcd",
+      "sudo mv etcd-client* /etc/ssl/etcd/",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
+      "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
+      "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
+      "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
+      "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
+      "sudo chown -R etcd:etcd /etc/ssl/etcd",
+      "sudo chmod -R 500 /etc/ssl/etcd",
       "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
     ]
   }

docs/topics/performance.md

@@ -8,7 +8,7 @@ Provisioning times vary based on the platform. Sampling the time to create (appl
 |---------------|-------|---------|
 | AWS           | 20 min | 8 min 10 sec |
 | Bare-Metal    | 10-14 min | NA  |
-| Digital Ocean | 5 min 5 sec | 20 sec |
+| Digital Ocean | 3 min 30 sec | 20 sec |
 | Google Cloud  | 6 min 10 sec | 4 min 30 sec |
 
 Notes: