Add dghubble/pegasus Digital Ocean Kubernetes Terraform module

2017-07-29 11:35:53 -07:00 · 2017-07-29 11:35:53 -07:00 · 6070ffb449
parent 2d33b9abe2
commit 6070ffb449
10 changed files with 514 additions and 1 deletions
--- a/README.md
+++ b/README.md
@ -20,3 +20,4 @@ Currently, `purenetes` is the author's personal distribution of Kubernetes.

 * Bare-Metal
 * Google Cloud
+* Digital Ocean
--- a/digital-ocean/container-linux/kubernetes/bootkube.tf
+++ b/digital-ocean/container-linux/kubernetes/bootkube.tf
@ -0,0 +1,12 @@
+# Self-hosted Kubernetes assets (kubeconfig, manifests)
+module "bootkube" {
+  source = "git::https://github.com/dghubble/bootkube-terraform.git?ref=v0.6.0"
+
+  cluster_name                  = "${var.cluster_name}"
+  api_servers                   = ["${var.k8s_domain_name}"]
+  etcd_servers                  = ["http://127.0.0.1:2379"]
+  asset_dir                     = "${var.asset_dir}"
+  pod_cidr                      = "${var.pod_cidr}"
+  service_cidr                  = "${var.service_cidr}"
+  experimental_self_hosted_etcd = "true"
+}
--- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl
+++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl
@ -0,0 +1,138 @@
+---
+systemd:
+  units:
+    - name: docker.service
+      enable: true
+    - name: locksmithd.service
+      mask: true
+    - name: wait-for-dns.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+    - name: kubelet.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Kubelet via Hyperkube ACI
+        [Service]
+        EnvironmentFile=/etc/kubernetes/kubelet.env
+        Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \
+          --volume=resolv,kind=host,source=/etc/resolv.conf \
+          --mount volume=resolv,target=/etc/resolv.conf \
+          --volume var-lib-cni,kind=host,source=/var/lib/cni \
+          --mount volume=var-lib-cni,target=/var/lib/cni \
+          --volume var-log,kind=host,source=/var/log \
+          --mount volume=var-log,target=/var/log"
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
+        ExecStartPre=/bin/mkdir -p /var/lib/cni
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
+        ExecStart=/usr/lib/coreos/kubelet-wrapper \
+          --kubeconfig=/etc/kubernetes/kubeconfig \
+          --require-kubeconfig \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --anonymous-auth=false \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --network-plugin=cni \
+          --lock-file=/var/run/lock/kubelet.lock \
+          --exit-on-lock-contention \
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --allow-privileged \
+          --node-labels=node-role.kubernetes.io/master \
+          --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
+          --cluster_dns=${k8s_dns_service_ip} \
+          --cluster_domain=cluster.local
+        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
+        Restart=always
+        RestartSec=10
+        [Install]
+        WantedBy=multi-user.target
+    - name: bootkube.service
+      contents: |
+        [Unit]
+        Description=Bootstrap a Kubernetes cluster
+        ConditionPathExists=!/opt/bootkube/init_bootkube.done
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        WorkingDirectory=/opt/bootkube
+        ExecStart=/opt/bootkube/bootkube-start
+        ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done
+        [Install]
+        WantedBy=multi-user.target
+storage:
+  files:
+    - path: /etc/kubernetes/kubeconfig
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: v1
+          kind: Config
+          clusters:
+          - name: local
+            cluster:
+              server: ${kubeconfig_server}
+              certificate-authority-data: ${kubeconfig_ca_cert}
+          users:
+          - name: kubelet
+            user:
+              client-certificate-data: ${kubeconfig_kubelet_cert}
+              client-key-data: ${kubeconfig_kubelet_key}
+          contexts:
+          - context:
+              cluster: local
+              user: kubelet
+    - path: /etc/kubernetes/kubelet.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
+          KUBELET_IMAGE_TAG=v1.7.1_coreos.0
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+    - path: /opt/bootkube/bootkube-start
+      filesystem: root
+      mode: 0544
+      user:
+        id: 500
+      group:
+        id: 500
+      contents:
+        inline: |
+          #!/bin/bash
+          # Wrapper for bootkube start
+          set -e
+          # Move experimental manifests
+          [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
+          [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
+          BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
+          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.6.0}"
+          BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
+          exec /usr/bin/rkt run \
+            --trust-keys-from-https \
+            --volume assets,kind=host,source=$${BOOTKUBE_ASSETS} \
+            --mount volume=assets,target=/assets \
+            --volume bootstrap,kind=host,source=/etc/kubernetes \
+            --mount volume=bootstrap,target=/etc/kubernetes \
+            $${RKT_OPTS} \
+            $${BOOTKUBE_ACI}:$${BOOTKUBE_VERSION} \
+            --net=host \
+            --dns=host \
+            --exec=/bootkube -- start --asset-dir=/assets "$@"
--- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl
+++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl
@ -0,0 +1,122 @@
+---
+systemd:
+  units:
+    - name: docker.service
+      enable: true
+    - name: locksmithd.service
+      mask: true
+    - name: wait-for-dns.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+    - name: kubelet.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Kubelet via Hyperkube ACI
+        [Service]
+        EnvironmentFile=/etc/kubernetes/kubelet.env
+        Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \
+          --volume=resolv,kind=host,source=/etc/resolv.conf \
+          --mount volume=resolv,target=/etc/resolv.conf \
+          --volume var-lib-cni,kind=host,source=/var/lib/cni \
+          --mount volume=var-lib-cni,target=/var/lib/cni \
+          --volume var-log,kind=host,source=/var/log \
+          --mount volume=var-log,target=/var/log"
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
+        ExecStartPre=/bin/mkdir -p /var/lib/cni
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
+        ExecStart=/usr/lib/coreos/kubelet-wrapper \
+          --kubeconfig=/etc/kubernetes/kubeconfig \
+          --require-kubeconfig \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --anonymous-auth=false \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --network-plugin=cni \
+          --lock-file=/var/run/lock/kubelet.lock \
+          --exit-on-lock-contention \
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --allow-privileged \
+          --node-labels=node-role.kubernetes.io/node \
+          --cluster_dns=${k8s_dns_service_ip} \
+          --cluster_domain=cluster.local
+        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
+        Restart=always
+        RestartSec=5
+        [Install]
+        WantedBy=multi-user.target
+    - name: delete-node.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Waiting to delete Kubernetes node on shutdown
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/true
+        ExecStop=/etc/kubernetes/delete-node
+        [Install]
+        WantedBy=multi-user.target
+storage:
+  files:
+    - path: /etc/kubernetes/kubeconfig
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: v1
+          kind: Config
+          clusters:
+          - name: local
+            cluster:
+              server: ${kubeconfig_server}
+              certificate-authority-data: ${kubeconfig_ca_cert}
+          users:
+          - name: kubelet
+            user:
+              client-certificate-data: ${kubeconfig_kubelet_cert}
+              client-key-data: ${kubeconfig_kubelet_key}
+          contexts:
+          - context:
+              cluster: local
+              user: kubelet
+    - path: /etc/kubernetes/kubelet.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
+          KUBELET_IMAGE_TAG=v1.7.1_coreos.0
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+    - path: /etc/kubernetes/delete-node
+      filesystem: root
+      mode: 0744
+      contents:
+        inline: |
+          #!/bin/bash
+          set -e
+          exec /usr/bin/rkt run \
+            --trust-keys-from-https \
+            --volume config,kind=host,source=/etc/kubernetes \
+            --mount volume=config,target=/etc/kubernetes \
+            quay.io/coreos/hyperkube:v1.7.1_coreos.0 \
+            --net=host \
+            --dns=host \
+            --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
--- a/digital-ocean/container-linux/kubernetes/controllers.tf
+++ b/digital-ocean/container-linux/kubernetes/controllers.tf
@ -0,0 +1,58 @@
+# Controller DNS records
+resource "digitalocean_record" "controllers" {
+  count = "${var.controller_count}"
+
+  # DNS zone where record should be created
+  domain = "${var.dns_zone}"
+  
+  name = "${var.cluster_name}"
+  type = "A"
+  ttl = 300
+  value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}"
+}
+
+# Controller droplet instances
+resource "digitalocean_droplet" "controllers" {
+  count = "${var.controller_count}"
+
+  name = "${var.cluster_name}-controller-${count.index}"
+  region = "${var.region}"
+
+  image = "${var.image}"
+  size = "${var.controller_type}"
+  
+  # network
+  ipv6 = true
+  private_networking = true
+  
+  user_data = "${data.ct_config.controller_ign.rendered}"
+  ssh_keys = "${var.ssh_fingerprints}"
+
+  tags = [
+    "${digitalocean_tag.controllers.id}"
+  ]
+}
+
+// Tag to label controllers
+resource "digitalocean_tag" "controllers" {
+  name = "${var.cluster_name}-controller"
+}
+
+# Controller Container Linux Config
+data "template_file" "controller_config" {
+  template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
+
+  vars = {
+    k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
+    k8s_etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
+    kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
+    kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
+    kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"
+    kubeconfig_server       = "${module.bootkube.server}"
+  }
+}
+
+data "ct_config" "controller_ign" {
+  content      = "${data.template_file.controller_config.rendered}"
+  pretty_print = false
+}
--- a/digital-ocean/container-linux/kubernetes/outputs.tf
+++ b/digital-ocean/container-linux/kubernetes/outputs.tf
@ -0,0 +1,23 @@
+output "controllers_dns" {
+  value = "${digitalocean_record.controllers.fqdn}"
+}
+
+output "workers_dns" {
+  value = "${digitalocean_record.workers.fqdn}"
+}
+
+output "controllers_ipv4" {
+  value = ["${digitalocean_droplet.controllers.*.ipv4_address}"]
+}
+
+output "controllers_ipv6" {
+  value = ["${digitalocean_droplet.controllers.*.ipv6_address}"]
+}
+
+output "workers_ipv4" {
+  value = ["${digitalocean_droplet.workers.*.ipv4_address}"]
+}
+
+output "workers_ipv6" {
+  value = ["${digitalocean_droplet.workers.*.ipv6_address}"]
+}
--- a/digital-ocean/container-linux/kubernetes/ssh.tf
+++ b/digital-ocean/container-linux/kubernetes/ssh.tf
@ -0,0 +1,25 @@
+# Secure copy bootkube assets to ONE controller and start bootkube to perform
+# one-time self-hosted cluster bootstrapping.
+resource "null_resource" "bootkube-start" {
+  depends_on = ["module.bootkube"]
+
+  connection {
+    type    = "ssh"
+    host    = "${digitalocean_droplet.controllers.0.ipv4_address}"
+    user    = "core"
+    timeout = "15m"
+  }
+
+  provisioner "file" {
+    source      = "${var.asset_dir}"
+    destination = "$HOME/assets"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo mv /home/core/assets /opt/bootkube",
+      "sudo systemctl start bootkube",
+    ]
+  }
+}
+
--- a/digital-ocean/container-linux/kubernetes/variables.tf
+++ b/digital-ocean/container-linux/kubernetes/variables.tf
@ -0,0 +1,76 @@
+variable "cluster_name" {
+  type        = "string"
+  description = "Unique cluster name"
+}
+
+variable "region" {
+  type        = "string"
+  description = "Digital Ocean region (e.g. nyc1, sfo2, fra1, tor1)"
+}
+
+variable "dns_zone" {
+  type = "string"
+  description = "Digital Ocean domain name (i.e. DNS zone with NS records) (e.g. digital-ocean.dghubble.io)"
+}
+
+variable "image" {
+  type        = "string"
+  description = "OS image from which to initialize the disk (e.g. coreos-stable)"
+}
+
+variable "controller_type" {
+  type = "string"
+  default = "1gb"
+  description = "Digital Ocean droplet type or size (e.g. 2gb, 4gb, 8gb). Do not choose a value below 2gb."
+}
+
+variable "controller_count" {
+  type        = "string"
+  default     = "1"
+  description = "Number of controllers"
+}
+
+variable "worker_type" {
+  type = "string"
+  default = "512mb"
+  description = "Digital Ocean droplet type or size (e.g. 512mb, 1gb, 2gb, 4gb)"
+}
+
+variable "worker_count" {
+  type        = "string"
+  default     = "1"
+  description = "Number of workers"
+}
+
+variable "ssh_fingerprints" {
+  type        = "list"
+  description = "SSH public key fingerprints. Use ssh-add -l -E md5."
+}
+
+# bootkube assets
+
+variable "k8s_domain_name" {
+  type        = "string"
+  description = "Controller DNS name which resolves to the controller instance. Kubectl and workers use TLS client credentials to communicate via this endpoint."
+}
+
+variable "asset_dir" {
+  description = "Path to a directory where generated assets should be placed (contains secrets)"
+  type        = "string"
+}
+
+variable "pod_cidr" {
+  description = "CIDR IP range to assign Kubernetes pods"
+  type        = "string"
+  default     = "10.2.0.0/16"
+}
+
+variable "service_cidr" {
+  description = <<EOD
+CIDR IP range to assign Kubernetes services.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns, the 15th IP will be reserved for self-hosted etcd, and the 200th IP will be reserved for bootstrap self-hosted etcd.
+EOD
+
+  type    = "string"
+  default = "10.3.0.0/16"
+}
--- a/digital-ocean/container-linux/kubernetes/workers.tf
+++ b/digital-ocean/container-linux/kubernetes/workers.tf
@ -0,0 +1,58 @@
+# Worker DNS records
+resource "digitalocean_record" "workers" {
+  count = "${var.worker_count}"
+
+  # DNS zone where record should be created
+  domain = "${var.dns_zone}"
+
+  name = "${var.cluster_name}-workers"
+  type = "A"
+  ttl = 300
+  value = "${element(digitalocean_droplet.workers.*.ipv4_address, count.index)}"
+}
+
+# Worker droplet instances
+resource "digitalocean_droplet" "workers" {
+  count = "${var.worker_count}"
+
+  name = "${var.cluster_name}-worker-${count.index}"
+  region = "${var.region}"
+
+  image = "${var.image}"
+  size = "${var.worker_type}"
+    
+  # network
+  ipv6 = true
+  private_networking = true
+  
+  user_data = "${data.ct_config.worker_ign.rendered}"
+  ssh_keys = "${var.ssh_fingerprints}"
+
+  tags = [
+    "${digitalocean_tag.workers.id}"
+  ]
+}
+
+// Tag to label workers
+resource "digitalocean_tag" "workers" {
+  name = "${var.cluster_name}-worker"
+}
+
+# Worker Container Linux Config
+data "template_file" "worker_config" {
+  template = "${file("${path.module}/cl/worker.yaml.tmpl")}"
+
+  vars = {
+    k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
+    k8s_etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
+    kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
+    kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
+    kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"
+    kubeconfig_server       = "${module.bootkube.server}"
+  }
+}
+
+data "ct_config" "worker_ign" {
+  content      = "${data.template_file.worker_config.rendered}"
+  pretty_print = false
+}
--- a/google-cloud/container-linux/kubernetes/variables.tf
+++ b/google-cloud/container-linux/kubernetes/variables.tf
@ -42,7 +42,7 @@ variable "os_image" {
 variable "controller_count" {
  type        = "string"
  default     = "1"
-  description = "Number of workers"
+  description = "Number of controllers"
 }

 variable "worker_count" {