diff --git a/CHANGES.md b/CHANGES.md index 1b96e567..91e7eb84 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,14 @@ Notable changes between versions. +## Latest + +#### Digital Ocean + +* Run etcd cluster across controller nodes (etcd-member.service) +* Reduce time to bootstrap a cluster +* Remove support for self-hosted etcd + ## v1.7.7 * Kubernetes v1.7.7 diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 5b376f57..167f5a70 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -2,13 +2,12 @@ module "bootkube" { source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.7.0" - cluster_name = "${var.cluster_name}" - api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] - etcd_servers = ["http://127.0.0.1:2379"] - asset_dir = "${var.asset_dir}" - networking = "${var.networking}" - network_mtu = 1440 - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - experimental_self_hosted_etcd = "true" + cluster_name = "${var.cluster_name}" + api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] + etcd_servers = "${digitalocean_record.etcds.*.fqdn}" + asset_dir = "${var.asset_dir}" + networking = "${var.networking}" + network_mtu = 1440 + pod_cidr = "${var.pod_cidr}" + service_cidr = "${var.service_cidr}" } diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index 8fa36ba8..6796d43e 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -1,6 +1,29 @@ --- systemd: units: + - name: etcd-member.service + enable: true + dropins: + - name: 40-etcd-cluster.conf + contents: | + [Service] + Environment="ETCD_IMAGE_TAG=v3.2.0" + Environment="ETCD_NAME=${etcd_name}" + Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379" + Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380" + Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379" + Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380" + Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}" + Environment="ETCD_STRICT_RECONFIG_CHECK=true" + Environment="ETCD_SSL_DIR=/etc/ssl/etcd" + Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt" + Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt" + Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key" + Environment="ETCD_CLIENT_CERT_AUTH=true" + Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt" + Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt" + Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key" + Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" - name: docker.service enable: true - name: locksmithd.service diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf index 4a381f6e..ca9b52ab 100644 --- a/digital-ocean/container-linux/kubernetes/controllers.tf +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -14,6 +14,21 @@ resource "digitalocean_record" "controllers" { value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}" } +resource "digitalocean_record" "etcds" { + count = "${var.controller_count}" + + # DNS zone where record should be created + domain = "${var.dns_zone}" + + # DNS record (will be prepended to domain) + name = "${var.cluster_name}-etcd${count.index}" + type = "A" + ttl = 300 + + # IPv4 addresses of controllers + value = "${element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index)}" +} + # Controller droplet instances resource "digitalocean_droplet" "controllers" { count = "${var.controller_count}" @@ -28,7 +43,7 @@ resource "digitalocean_droplet" "controllers" { ipv6 = true private_networking = true - user_data = "${data.ct_config.controller_ign.rendered}" + user_data = "${element(data.ct_config.controller_ign.*.rendered, count.index)}" ssh_keys = "${var.ssh_fingerprints}" tags = [ @@ -43,15 +58,33 @@ resource "digitalocean_tag" "controllers" { # Controller Container Linux Config data "template_file" "controller_config" { + count = "${var.controller_count}" + template = "${file("${path.module}/cl/controller.yaml.tmpl")}" vars = { - k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" + # Cannot use cyclic dependencies on controllers or their DNS records + etcd_name = "etcd${count.index}" + etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" + # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,... + etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}" + k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" + } +} + +# Horrible hack to generate a Terraform list of a desired length without dependencies. +# Ideal ${repeat("etcd", 3) -> ["etcd", "etcd", "etcd"]} +resource null_resource "repeat" { + count = "${var.controller_count}" + + triggers { + name = "etcd${count.index}" + domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}" } } data "ct_config" "controller_ign" { - content = "${data.template_file.controller_config.rendered}" + count = "${var.controller_count}" + content = "${element(data.template_file.controller_config.*.rendered, count.index)}" pretty_print = false } diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 166230b5..17e37b39 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -14,8 +14,53 @@ resource "null_resource" "copy-secrets" { destination = "$HOME/kubeconfig" } + provisioner "file" { + content = "${module.bootkube.etcd_ca_cert}" + destination = "$HOME/etcd-client-ca.crt" + } + + provisioner "file" { + content = "${module.bootkube.etcd_client_cert}" + destination = "$HOME/etcd-client.crt" + } + + provisioner "file" { + content = "${module.bootkube.etcd_client_key}" + destination = "$HOME/etcd-client.key" + } + + provisioner "file" { + content = "${module.bootkube.etcd_server_cert}" + destination = "$HOME/etcd-server.crt" + } + + provisioner "file" { + content = "${module.bootkube.etcd_server_key}" + destination = "$HOME/etcd-server.key" + } + + provisioner "file" { + content = "${module.bootkube.etcd_peer_cert}" + destination = "$HOME/etcd-peer.crt" + } + + provisioner "file" { + content = "${module.bootkube.etcd_peer_key}" + destination = "$HOME/etcd-peer.key" + } + provisioner "remote-exec" { inline = [ + "sudo mkdir -p /etc/ssl/etcd/etcd", + "sudo mv etcd-client* /etc/ssl/etcd/", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", + "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", + "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", + "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", + "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", + "sudo chown -R etcd:etcd /etc/ssl/etcd", + "sudo chmod -R 500 /etc/ssl/etcd", "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig", ] } diff --git a/docs/topics/performance.md b/docs/topics/performance.md index 27969f1c..c4e58653 100644 --- a/docs/topics/performance.md +++ b/docs/topics/performance.md @@ -8,7 +8,7 @@ Provisioning times vary based on the platform. Sampling the time to create (appl |---------------|-------|---------| | AWS | 20 min | 8 min 10 sec | | Bare-Metal | 10-14 min | NA | -| Digital Ocean | 5 min 5 sec | 20 sec | +| Digital Ocean | 3 min 30 sec | 20 sec | | Google Cloud | 6 min 10 sec | 4 min 30 sec | Notes: