digital-ocean: Run etcd cluster on-host, across controllers
* Run etcd peers with TLS across controller nodes * Deprecate self-hosted-etcd on the Digital Ocean platform * Distribute etcd TLS certificates as part of initial provisioning * Check the status of etcd by running `systemctl status etcd-member`
This commit is contained in:
parent
da63c89d71
commit
308c7dfb6e
|
@ -2,6 +2,14 @@
|
||||||
|
|
||||||
Notable changes between versions.
|
Notable changes between versions.
|
||||||
|
|
||||||
|
## Latest
|
||||||
|
|
||||||
|
#### Digital Ocean
|
||||||
|
|
||||||
|
* Run etcd cluster across controller nodes (etcd-member.service)
|
||||||
|
* Reduce time to bootstrap a cluster
|
||||||
|
* Remove support for self-hosted etcd
|
||||||
|
|
||||||
## v1.7.7
|
## v1.7.7
|
||||||
|
|
||||||
* Kubernetes v1.7.7
|
* Kubernetes v1.7.7
|
||||||
|
|
|
@ -4,11 +4,10 @@ module "bootkube" {
|
||||||
|
|
||||||
cluster_name = "${var.cluster_name}"
|
cluster_name = "${var.cluster_name}"
|
||||||
api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
|
api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
|
||||||
etcd_servers = ["http://127.0.0.1:2379"]
|
etcd_servers = "${digitalocean_record.etcds.*.fqdn}"
|
||||||
asset_dir = "${var.asset_dir}"
|
asset_dir = "${var.asset_dir}"
|
||||||
networking = "${var.networking}"
|
networking = "${var.networking}"
|
||||||
network_mtu = 1440
|
network_mtu = 1440
|
||||||
pod_cidr = "${var.pod_cidr}"
|
pod_cidr = "${var.pod_cidr}"
|
||||||
service_cidr = "${var.service_cidr}"
|
service_cidr = "${var.service_cidr}"
|
||||||
experimental_self_hosted_etcd = "true"
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,29 @@
|
||||||
---
|
---
|
||||||
systemd:
|
systemd:
|
||||||
units:
|
units:
|
||||||
|
- name: etcd-member.service
|
||||||
|
enable: true
|
||||||
|
dropins:
|
||||||
|
- name: 40-etcd-cluster.conf
|
||||||
|
contents: |
|
||||||
|
[Service]
|
||||||
|
Environment="ETCD_IMAGE_TAG=v3.2.0"
|
||||||
|
Environment="ETCD_NAME=${etcd_name}"
|
||||||
|
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
|
||||||
|
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
|
||||||
|
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
|
||||||
|
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
|
||||||
|
Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
|
||||||
|
Environment="ETCD_STRICT_RECONFIG_CHECK=true"
|
||||||
|
Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
|
||||||
|
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
|
||||||
|
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
|
||||||
|
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
|
||||||
|
Environment="ETCD_CLIENT_CERT_AUTH=true"
|
||||||
|
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
|
||||||
|
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
|
||||||
|
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
|
||||||
|
Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
|
||||||
- name: docker.service
|
- name: docker.service
|
||||||
enable: true
|
enable: true
|
||||||
- name: locksmithd.service
|
- name: locksmithd.service
|
||||||
|
|
|
@ -14,6 +14,21 @@ resource "digitalocean_record" "controllers" {
|
||||||
value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}"
|
value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "digitalocean_record" "etcds" {
|
||||||
|
count = "${var.controller_count}"
|
||||||
|
|
||||||
|
# DNS zone where record should be created
|
||||||
|
domain = "${var.dns_zone}"
|
||||||
|
|
||||||
|
# DNS record (will be prepended to domain)
|
||||||
|
name = "${var.cluster_name}-etcd${count.index}"
|
||||||
|
type = "A"
|
||||||
|
ttl = 300
|
||||||
|
|
||||||
|
# IPv4 addresses of controllers
|
||||||
|
value = "${element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index)}"
|
||||||
|
}
|
||||||
|
|
||||||
# Controller droplet instances
|
# Controller droplet instances
|
||||||
resource "digitalocean_droplet" "controllers" {
|
resource "digitalocean_droplet" "controllers" {
|
||||||
count = "${var.controller_count}"
|
count = "${var.controller_count}"
|
||||||
|
@ -28,7 +43,7 @@ resource "digitalocean_droplet" "controllers" {
|
||||||
ipv6 = true
|
ipv6 = true
|
||||||
private_networking = true
|
private_networking = true
|
||||||
|
|
||||||
user_data = "${data.ct_config.controller_ign.rendered}"
|
user_data = "${element(data.ct_config.controller_ign.*.rendered, count.index)}"
|
||||||
ssh_keys = "${var.ssh_fingerprints}"
|
ssh_keys = "${var.ssh_fingerprints}"
|
||||||
|
|
||||||
tags = [
|
tags = [
|
||||||
|
@ -43,15 +58,33 @@ resource "digitalocean_tag" "controllers" {
|
||||||
|
|
||||||
# Controller Container Linux Config
|
# Controller Container Linux Config
|
||||||
data "template_file" "controller_config" {
|
data "template_file" "controller_config" {
|
||||||
|
count = "${var.controller_count}"
|
||||||
|
|
||||||
template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
|
template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
|
||||||
|
|
||||||
vars = {
|
vars = {
|
||||||
|
# Cannot use cyclic dependencies on controllers or their DNS records
|
||||||
|
etcd_name = "etcd${count.index}"
|
||||||
|
etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
|
||||||
|
# etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
|
||||||
|
etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
|
||||||
k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
|
k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
|
||||||
k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Horrible hack to generate a Terraform list of a desired length without dependencies.
|
||||||
|
# Ideal ${repeat("etcd", 3) -> ["etcd", "etcd", "etcd"]}
|
||||||
|
resource null_resource "repeat" {
|
||||||
|
count = "${var.controller_count}"
|
||||||
|
|
||||||
|
triggers {
|
||||||
|
name = "etcd${count.index}"
|
||||||
|
domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
data "ct_config" "controller_ign" {
|
data "ct_config" "controller_ign" {
|
||||||
content = "${data.template_file.controller_config.rendered}"
|
count = "${var.controller_count}"
|
||||||
|
content = "${element(data.template_file.controller_config.*.rendered, count.index)}"
|
||||||
pretty_print = false
|
pretty_print = false
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,8 +14,53 @@ resource "null_resource" "copy-secrets" {
|
||||||
destination = "$HOME/kubeconfig"
|
destination = "$HOME/kubeconfig"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
provisioner "file" {
|
||||||
|
content = "${module.bootkube.etcd_ca_cert}"
|
||||||
|
destination = "$HOME/etcd-client-ca.crt"
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "file" {
|
||||||
|
content = "${module.bootkube.etcd_client_cert}"
|
||||||
|
destination = "$HOME/etcd-client.crt"
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "file" {
|
||||||
|
content = "${module.bootkube.etcd_client_key}"
|
||||||
|
destination = "$HOME/etcd-client.key"
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "file" {
|
||||||
|
content = "${module.bootkube.etcd_server_cert}"
|
||||||
|
destination = "$HOME/etcd-server.crt"
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "file" {
|
||||||
|
content = "${module.bootkube.etcd_server_key}"
|
||||||
|
destination = "$HOME/etcd-server.key"
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "file" {
|
||||||
|
content = "${module.bootkube.etcd_peer_cert}"
|
||||||
|
destination = "$HOME/etcd-peer.crt"
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "file" {
|
||||||
|
content = "${module.bootkube.etcd_peer_key}"
|
||||||
|
destination = "$HOME/etcd-peer.key"
|
||||||
|
}
|
||||||
|
|
||||||
provisioner "remote-exec" {
|
provisioner "remote-exec" {
|
||||||
inline = [
|
inline = [
|
||||||
|
"sudo mkdir -p /etc/ssl/etcd/etcd",
|
||||||
|
"sudo mv etcd-client* /etc/ssl/etcd/",
|
||||||
|
"sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
|
||||||
|
"sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
|
||||||
|
"sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
|
||||||
|
"sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
|
||||||
|
"sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
|
||||||
|
"sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
|
||||||
|
"sudo chown -R etcd:etcd /etc/ssl/etcd",
|
||||||
|
"sudo chmod -R 500 /etc/ssl/etcd",
|
||||||
"sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
|
"sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,7 +8,7 @@ Provisioning times vary based on the platform. Sampling the time to create (appl
|
||||||
|---------------|-------|---------|
|
|---------------|-------|---------|
|
||||||
| AWS | 20 min | 8 min 10 sec |
|
| AWS | 20 min | 8 min 10 sec |
|
||||||
| Bare-Metal | 10-14 min | NA |
|
| Bare-Metal | 10-14 min | NA |
|
||||||
| Digital Ocean | 5 min 5 sec | 20 sec |
|
| Digital Ocean | 3 min 30 sec | 20 sec |
|
||||||
| Google Cloud | 6 min 10 sec | 4 min 30 sec |
|
| Google Cloud | 6 min 10 sec | 4 min 30 sec |
|
||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
|
|
Loading…
Reference in New Issue