digital-ocean: Run etcd cluster on-host, across controllers

* Run etcd peers with TLS across controller nodes
* Deprecate self-hosted-etcd on the Digital Ocean platform
* Distribute etcd TLS certificates as part of initial provisioning
* Check the status of etcd by running `systemctl status etcd-member`
This commit is contained in:
Dalton Hubble 2017-10-04 00:07:48 -07:00
parent da63c89d71
commit 308c7dfb6e
6 changed files with 122 additions and 14 deletions

View File

@ -2,6 +2,14 @@
Notable changes between versions. Notable changes between versions.
## Latest
#### Digital Ocean
* Run etcd cluster across controller nodes (etcd-member.service)
* Reduce time to bootstrap a cluster
* Remove support for self-hosted etcd
## v1.7.7 ## v1.7.7
* Kubernetes v1.7.7 * Kubernetes v1.7.7

View File

@ -4,11 +4,10 @@ module "bootkube" {
cluster_name = "${var.cluster_name}" cluster_name = "${var.cluster_name}"
api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
etcd_servers = ["http://127.0.0.1:2379"] etcd_servers = "${digitalocean_record.etcds.*.fqdn}"
asset_dir = "${var.asset_dir}" asset_dir = "${var.asset_dir}"
networking = "${var.networking}" networking = "${var.networking}"
network_mtu = 1440 network_mtu = 1440
pod_cidr = "${var.pod_cidr}" pod_cidr = "${var.pod_cidr}"
service_cidr = "${var.service_cidr}" service_cidr = "${var.service_cidr}"
experimental_self_hosted_etcd = "true"
} }

View File

@ -1,6 +1,29 @@
--- ---
systemd: systemd:
units: units:
- name: etcd-member.service
enable: true
dropins:
- name: 40-etcd-cluster.conf
contents: |
[Service]
Environment="ETCD_IMAGE_TAG=v3.2.0"
Environment="ETCD_NAME=${etcd_name}"
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
Environment="ETCD_STRICT_RECONFIG_CHECK=true"
Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
Environment="ETCD_CLIENT_CERT_AUTH=true"
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
- name: docker.service - name: docker.service
enable: true enable: true
- name: locksmithd.service - name: locksmithd.service

View File

@ -14,6 +14,21 @@ resource "digitalocean_record" "controllers" {
value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}" value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}"
} }
resource "digitalocean_record" "etcds" {
count = "${var.controller_count}"
# DNS zone where record should be created
domain = "${var.dns_zone}"
# DNS record (will be prepended to domain)
name = "${var.cluster_name}-etcd${count.index}"
type = "A"
ttl = 300
# IPv4 addresses of controllers
value = "${element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index)}"
}
# Controller droplet instances # Controller droplet instances
resource "digitalocean_droplet" "controllers" { resource "digitalocean_droplet" "controllers" {
count = "${var.controller_count}" count = "${var.controller_count}"
@ -28,7 +43,7 @@ resource "digitalocean_droplet" "controllers" {
ipv6 = true ipv6 = true
private_networking = true private_networking = true
user_data = "${data.ct_config.controller_ign.rendered}" user_data = "${element(data.ct_config.controller_ign.*.rendered, count.index)}"
ssh_keys = "${var.ssh_fingerprints}" ssh_keys = "${var.ssh_fingerprints}"
tags = [ tags = [
@ -43,15 +58,33 @@ resource "digitalocean_tag" "controllers" {
# Controller Container Linux Config # Controller Container Linux Config
data "template_file" "controller_config" { data "template_file" "controller_config" {
count = "${var.controller_count}"
template = "${file("${path.module}/cl/controller.yaml.tmpl")}" template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
vars = { vars = {
# Cannot use cyclic dependencies on controllers or their DNS records
etcd_name = "etcd${count.index}"
etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
# etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" }
}
# Horrible hack to generate a Terraform list of a desired length without dependencies.
# Ideal ${repeat("etcd", 3) -> ["etcd", "etcd", "etcd"]}
resource null_resource "repeat" {
count = "${var.controller_count}"
triggers {
name = "etcd${count.index}"
domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
} }
} }
data "ct_config" "controller_ign" { data "ct_config" "controller_ign" {
content = "${data.template_file.controller_config.rendered}" count = "${var.controller_count}"
content = "${element(data.template_file.controller_config.*.rendered, count.index)}"
pretty_print = false pretty_print = false
} }

View File

@ -14,8 +14,53 @@ resource "null_resource" "copy-secrets" {
destination = "$HOME/kubeconfig" destination = "$HOME/kubeconfig"
} }
provisioner "file" {
content = "${module.bootkube.etcd_ca_cert}"
destination = "$HOME/etcd-client-ca.crt"
}
provisioner "file" {
content = "${module.bootkube.etcd_client_cert}"
destination = "$HOME/etcd-client.crt"
}
provisioner "file" {
content = "${module.bootkube.etcd_client_key}"
destination = "$HOME/etcd-client.key"
}
provisioner "file" {
content = "${module.bootkube.etcd_server_cert}"
destination = "$HOME/etcd-server.crt"
}
provisioner "file" {
content = "${module.bootkube.etcd_server_key}"
destination = "$HOME/etcd-server.key"
}
provisioner "file" {
content = "${module.bootkube.etcd_peer_cert}"
destination = "$HOME/etcd-peer.crt"
}
provisioner "file" {
content = "${module.bootkube.etcd_peer_key}"
destination = "$HOME/etcd-peer.key"
}
provisioner "remote-exec" { provisioner "remote-exec" {
inline = [ inline = [
"sudo mkdir -p /etc/ssl/etcd/etcd",
"sudo mv etcd-client* /etc/ssl/etcd/",
"sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
"sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
"sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
"sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
"sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
"sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
"sudo chown -R etcd:etcd /etc/ssl/etcd",
"sudo chmod -R 500 /etc/ssl/etcd",
"sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig", "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
] ]
} }

View File

@ -8,7 +8,7 @@ Provisioning times vary based on the platform. Sampling the time to create (appl
|---------------|-------|---------| |---------------|-------|---------|
| AWS | 20 min | 8 min 10 sec | | AWS | 20 min | 8 min 10 sec |
| Bare-Metal | 10-14 min | NA | | Bare-Metal | 10-14 min | NA |
| Digital Ocean | 5 min 5 sec | 20 sec | | Digital Ocean | 3 min 30 sec | 20 sec |
| Google Cloud | 6 min 10 sec | 4 min 30 sec | | Google Cloud | 6 min 10 sec | 4 min 30 sec |
Notes: Notes: