From 6070ffb449b552197cb6a4db950f0fb0c99a2aa6 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 29 Jul 2017 11:35:53 -0700 Subject: [PATCH] Add dghubble/pegasus Digital Ocean Kubernetes Terraform module --- README.md | 1 + .../container-linux/kubernetes/bootkube.tf | 12 ++ .../kubernetes/cl/controller.yaml.tmpl | 138 ++++++++++++++++++ .../kubernetes/cl/worker.yaml.tmpl | 122 ++++++++++++++++ .../container-linux/kubernetes/controllers.tf | 58 ++++++++ .../container-linux/kubernetes/outputs.tf | 23 +++ .../container-linux/kubernetes/ssh.tf | 25 ++++ .../container-linux/kubernetes/variables.tf | 76 ++++++++++ .../container-linux/kubernetes/workers.tf | 58 ++++++++ .../container-linux/kubernetes/variables.tf | 2 +- 10 files changed, 514 insertions(+), 1 deletion(-) create mode 100644 digital-ocean/container-linux/kubernetes/bootkube.tf create mode 100644 digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl create mode 100644 digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl create mode 100644 digital-ocean/container-linux/kubernetes/controllers.tf create mode 100644 digital-ocean/container-linux/kubernetes/outputs.tf create mode 100644 digital-ocean/container-linux/kubernetes/ssh.tf create mode 100644 digital-ocean/container-linux/kubernetes/variables.tf create mode 100644 digital-ocean/container-linux/kubernetes/workers.tf diff --git a/README.md b/README.md index 8458edf8..cc79e59c 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,4 @@ Currently, `purenetes` is the author's personal distribution of Kubernetes. * Bare-Metal * Google Cloud +* Digital Ocean diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf new file mode 100644 index 00000000..fd959a0c --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -0,0 +1,12 @@ +# Self-hosted Kubernetes assets (kubeconfig, manifests) +module "bootkube" { + source = "git::https://github.com/dghubble/bootkube-terraform.git?ref=v0.6.0" + + cluster_name = "${var.cluster_name}" + api_servers = ["${var.k8s_domain_name}"] + etcd_servers = ["http://127.0.0.1:2379"] + asset_dir = "${var.asset_dir}" + pod_cidr = "${var.pod_cidr}" + service_cidr = "${var.service_cidr}" + experimental_self_hosted_etcd = "true" +} diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl new file mode 100644 index 00000000..5cf34618 --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -0,0 +1,138 @@ +--- +systemd: + units: + - name: docker.service + enable: true + - name: locksmithd.service + mask: true + - name: wait-for-dns.service + enable: true + contents: | + [Unit] + Description=Wait for DNS entries + Wants=systemd-resolved.service + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + enable: true + contents: | + [Unit] + Description=Kubelet via Hyperkube ACI + [Service] + EnvironmentFile=/etc/kubernetes/kubelet.env + Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ + --volume=resolv,kind=host,source=/etc/resolv.conf \ + --mount volume=resolv,target=/etc/resolv.conf \ + --volume var-lib-cni,kind=host,source=/var/lib/cni \ + --mount volume=var-lib-cni,target=/var/lib/cni \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log" + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets + ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /var/lib/cni + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid + ExecStart=/usr/lib/coreos/kubelet-wrapper \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --require-kubeconfig \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --anonymous-auth=false \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --network-plugin=cni \ + --lock-file=/var/run/lock/kubelet.lock \ + --exit-on-lock-contention \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged \ + --node-labels=node-role.kubernetes.io/master \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --cluster_dns=${k8s_dns_service_ip} \ + --cluster_domain=cluster.local + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid + Restart=always + RestartSec=10 + [Install] + WantedBy=multi-user.target + - name: bootkube.service + contents: | + [Unit] + Description=Bootstrap a Kubernetes cluster + ConditionPathExists=!/opt/bootkube/init_bootkube.done + [Service] + Type=oneshot + RemainAfterExit=true + WorkingDirectory=/opt/bootkube + ExecStart=/opt/bootkube/bootkube-start + ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + [Install] + WantedBy=multi-user.target +storage: + files: + - path: /etc/kubernetes/kubeconfig + filesystem: root + mode: 0644 + contents: + inline: | + apiVersion: v1 + kind: Config + clusters: + - name: local + cluster: + server: ${kubeconfig_server} + certificate-authority-data: ${kubeconfig_ca_cert} + users: + - name: kubelet + user: + client-certificate-data: ${kubeconfig_kubelet_cert} + client-key-data: ${kubeconfig_kubelet_key} + contexts: + - context: + cluster: local + user: kubelet + - path: /etc/kubernetes/kubelet.env + filesystem: root + mode: 0644 + contents: + inline: | + KUBELET_IMAGE_URL=quay.io/coreos/hyperkube + KUBELET_IMAGE_TAG=v1.7.1_coreos.0 + - path: /etc/sysctl.d/max-user-watches.conf + filesystem: root + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /opt/bootkube/bootkube-start + filesystem: root + mode: 0544 + user: + id: 500 + group: + id: 500 + contents: + inline: | + #!/bin/bash + # Wrapper for bootkube start + set -e + # Move experimental manifests + [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests + [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests + BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}" + BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.6.0}" + BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}" + exec /usr/bin/rkt run \ + --trust-keys-from-https \ + --volume assets,kind=host,source=$${BOOTKUBE_ASSETS} \ + --mount volume=assets,target=/assets \ + --volume bootstrap,kind=host,source=/etc/kubernetes \ + --mount volume=bootstrap,target=/etc/kubernetes \ + $${RKT_OPTS} \ + $${BOOTKUBE_ACI}:$${BOOTKUBE_VERSION} \ + --net=host \ + --dns=host \ + --exec=/bootkube -- start --asset-dir=/assets "$@" diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl new file mode 100644 index 00000000..4c967baa --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -0,0 +1,122 @@ +--- +systemd: + units: + - name: docker.service + enable: true + - name: locksmithd.service + mask: true + - name: wait-for-dns.service + enable: true + contents: | + [Unit] + Description=Wait for DNS entries + Wants=systemd-resolved.service + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + enable: true + contents: | + [Unit] + Description=Kubelet via Hyperkube ACI + [Service] + EnvironmentFile=/etc/kubernetes/kubelet.env + Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ + --volume=resolv,kind=host,source=/etc/resolv.conf \ + --mount volume=resolv,target=/etc/resolv.conf \ + --volume var-lib-cni,kind=host,source=/var/lib/cni \ + --mount volume=var-lib-cni,target=/var/lib/cni \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log" + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets + ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /var/lib/cni + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid + ExecStart=/usr/lib/coreos/kubelet-wrapper \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --require-kubeconfig \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --anonymous-auth=false \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --network-plugin=cni \ + --lock-file=/var/run/lock/kubelet.lock \ + --exit-on-lock-contention \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged \ + --node-labels=node-role.kubernetes.io/node \ + --cluster_dns=${k8s_dns_service_ip} \ + --cluster_domain=cluster.local + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid + Restart=always + RestartSec=5 + [Install] + WantedBy=multi-user.target + - name: delete-node.service + enable: true + contents: | + [Unit] + Description=Waiting to delete Kubernetes node on shutdown + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/true + ExecStop=/etc/kubernetes/delete-node + [Install] + WantedBy=multi-user.target +storage: + files: + - path: /etc/kubernetes/kubeconfig + filesystem: root + mode: 0644 + contents: + inline: | + apiVersion: v1 + kind: Config + clusters: + - name: local + cluster: + server: ${kubeconfig_server} + certificate-authority-data: ${kubeconfig_ca_cert} + users: + - name: kubelet + user: + client-certificate-data: ${kubeconfig_kubelet_cert} + client-key-data: ${kubeconfig_kubelet_key} + contexts: + - context: + cluster: local + user: kubelet + - path: /etc/kubernetes/kubelet.env + filesystem: root + mode: 0644 + contents: + inline: | + KUBELET_IMAGE_URL=quay.io/coreos/hyperkube + KUBELET_IMAGE_TAG=v1.7.1_coreos.0 + - path: /etc/sysctl.d/max-user-watches.conf + filesystem: root + contents: + inline: | + fs.inotify.max_user_watches=16184 + - path: /etc/kubernetes/delete-node + filesystem: root + mode: 0744 + contents: + inline: | + #!/bin/bash + set -e + exec /usr/bin/rkt run \ + --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes \ + --mount volume=config,target=/etc/kubernetes \ + quay.io/coreos/hyperkube:v1.7.1_coreos.0 \ + --net=host \ + --dns=host \ + --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/digital-ocean/container-linux/kubernetes/controllers.tf b/digital-ocean/container-linux/kubernetes/controllers.tf new file mode 100644 index 00000000..bbd06b61 --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/controllers.tf @@ -0,0 +1,58 @@ +# Controller DNS records +resource "digitalocean_record" "controllers" { + count = "${var.controller_count}" + + # DNS zone where record should be created + domain = "${var.dns_zone}" + + name = "${var.cluster_name}" + type = "A" + ttl = 300 + value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}" +} + +# Controller droplet instances +resource "digitalocean_droplet" "controllers" { + count = "${var.controller_count}" + + name = "${var.cluster_name}-controller-${count.index}" + region = "${var.region}" + + image = "${var.image}" + size = "${var.controller_type}" + + # network + ipv6 = true + private_networking = true + + user_data = "${data.ct_config.controller_ign.rendered}" + ssh_keys = "${var.ssh_fingerprints}" + + tags = [ + "${digitalocean_tag.controllers.id}" + ] +} + +// Tag to label controllers +resource "digitalocean_tag" "controllers" { + name = "${var.cluster_name}-controller" +} + +# Controller Container Linux Config +data "template_file" "controller_config" { + template = "${file("${path.module}/cl/controller.yaml.tmpl")}" + + vars = { + k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" + k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" + kubeconfig_ca_cert = "${module.bootkube.ca_cert}" + kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}" + kubeconfig_kubelet_key = "${module.bootkube.kubelet_key}" + kubeconfig_server = "${module.bootkube.server}" + } +} + +data "ct_config" "controller_ign" { + content = "${data.template_file.controller_config.rendered}" + pretty_print = false +} diff --git a/digital-ocean/container-linux/kubernetes/outputs.tf b/digital-ocean/container-linux/kubernetes/outputs.tf new file mode 100644 index 00000000..d49b752f --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/outputs.tf @@ -0,0 +1,23 @@ +output "controllers_dns" { + value = "${digitalocean_record.controllers.fqdn}" +} + +output "workers_dns" { + value = "${digitalocean_record.workers.fqdn}" +} + +output "controllers_ipv4" { + value = ["${digitalocean_droplet.controllers.*.ipv4_address}"] +} + +output "controllers_ipv6" { + value = ["${digitalocean_droplet.controllers.*.ipv6_address}"] +} + +output "workers_ipv4" { + value = ["${digitalocean_droplet.workers.*.ipv4_address}"] +} + +output "workers_ipv6" { + value = ["${digitalocean_droplet.workers.*.ipv6_address}"] +} diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf new file mode 100644 index 00000000..fac3cd90 --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -0,0 +1,25 @@ +# Secure copy bootkube assets to ONE controller and start bootkube to perform +# one-time self-hosted cluster bootstrapping. +resource "null_resource" "bootkube-start" { + depends_on = ["module.bootkube"] + + connection { + type = "ssh" + host = "${digitalocean_droplet.controllers.0.ipv4_address}" + user = "core" + timeout = "15m" + } + + provisioner "file" { + source = "${var.asset_dir}" + destination = "$HOME/assets" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv /home/core/assets /opt/bootkube", + "sudo systemctl start bootkube", + ] + } +} + diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf new file mode 100644 index 00000000..520c820f --- /dev/null +++ b/digital-ocean/container-linux/kubernetes/variables.tf @@ -0,0 +1,76 @@ +variable "cluster_name" { + type = "string" + description = "Unique cluster name" +} + +variable "region" { + type = "string" + description = "Digital Ocean region (e.g. nyc1, sfo2, fra1, tor1)" +} + +variable "dns_zone" { + type = "string" + description = "Digital Ocean domain name (i.e. DNS zone with NS records) (e.g. digital-ocean.dghubble.io)" +} + +variable "image" { + type = "string" + description = "OS image from which to initialize the disk (e.g. coreos-stable)" +} + +variable "controller_type" { + type = "string" + default = "1gb" + description = "Digital Ocean droplet type or size (e.g. 2gb, 4gb, 8gb). Do not choose a value below 2gb." +} + +variable "controller_count" { + type = "string" + default = "1" + description = "Number of controllers" +} + +variable "worker_type" { + type = "string" + default = "512mb" + description = "Digital Ocean droplet type or size (e.g. 512mb, 1gb, 2gb, 4gb)" +} + +variable "worker_count" { + type = "string" + default = "1" + description = "Number of workers" +} + +variable "ssh_fingerprints" { + type = "list" + description = "SSH public key fingerprints. Use ssh-add -l -E md5." +} + +# bootkube assets + +variable "k8s_domain_name" { + type = "string" + description = "Controller DNS name which resolves to the controller instance. Kubectl and workers use TLS client credentials to communicate via this endpoint." +} + +variable "asset_dir" { + description = "Path to a directory where generated assets should be placed (contains secrets)" + type = "string" +} + +variable "pod_cidr" { + description = "CIDR IP range to assign Kubernetes pods" + type = "string" + default = "10.2.0.0/16" +} + +variable "service_cidr" { + description = <