Add Terraform module for bare-metal PXE workers

metal-worker-pxe/cl/bootkube-worker.yaml.tmpl

@@ -0,0 +1,111 @@
+---
+systemd:
+  units:
+    - name: docker.service
+      enable: true
+    - name: locksmithd.service
+      mask: true
+    - name: kubelet.path
+      enable: true
+      contents: |
+        [Unit]
+        Description=Watch for kubeconfig
+        [Path]
+        PathExists=/etc/kubernetes/kubeconfig
+        [Install]
+        WantedBy=multi-user.target
+    - name: wait-for-dns.service
+      enable: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+    - name: kubelet.service
+      contents: |
+        [Unit]
+        Description=Kubelet via Hyperkube ACI
+        [Service]
+        EnvironmentFile=/etc/kubernetes/kubelet.env
+        Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \
+          --volume=resolv,kind=host,source=/etc/resolv.conf \
+          --mount volume=resolv,target=/etc/resolv.conf \
+          --volume var-lib-cni,kind=host,source=/var/lib/cni \
+          --mount volume=var-lib-cni,target=/var/lib/cni \
+          --volume var-log,kind=host,source=/var/log \
+          --mount volume=var-log,target=/var/log"
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
+        ExecStartPre=/bin/mkdir -p /var/lib/cni
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
+        ExecStart=/usr/lib/coreos/kubelet-wrapper \
+          --kubeconfig=/etc/kubernetes/kubeconfig \
+          --require-kubeconfig \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --anonymous-auth=false \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --network-plugin=cni \
+          --lock-file=/var/run/lock/kubelet.lock \
+          --exit-on-lock-contention \
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --allow-privileged \
+          --hostname-override={{.domain_name}} \
+          --node-labels=node-role.kubernetes.io/node \
+          --cluster_dns={{.k8s_dns_service_ip}} \
+          --cluster_domain=cluster.local
+        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
+        Restart=always
+        RestartSec=5
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  {{ if index . "pxe" }}
+  disks:
+    - device: /dev/sda
+      wipe_table: true
+      partitions:
+        - label: ROOT
+  filesystems:
+    - name: root
+      mount:
+        device: "/dev/sda1"
+        format: "ext4"
+        create:
+          force: true
+          options:
+            - "-LROOT"
+  {{end}}
+  files:
+    - path: /etc/kubernetes/kubelet.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
+          KUBELET_IMAGE_TAG=v1.6.6_coreos.1
+    - path: /etc/hostname
+      filesystem: root
+      mode: 0644
+      contents:
+        inline:
+          {{.domain_name}}
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - {{.ssh_authorized_key}}

metal-worker-pxe/groups.tf

@@ -0,0 +1,21 @@
+resource "matchbox_group" "workers" {
+  count   = "${length(var.worker_names)}"
+  name    = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}"
+  profile = "${matchbox_profile.bootkube-worker-pxe.name}"
+
+  selector {
+    mac = "${element(var.worker_macs, count.index)}"
+  }
+
+  metadata {
+    pxe            = "true"
+    domain_name    = "${element(var.worker_domains, count.index)}"
+    etcd_endpoints = "${join(",", formatlist("%s:2379", var.controller_domains))}"
+
+    # TODO
+    etcd_on_host        = "true"
+    k8s_etcd_service_ip = "10.3.0.15"
+    k8s_dns_service_ip  = "${var.kube_dns_service_ip}"
+    ssh_authorized_key  = "${var.ssh_authorized_key}"
+  }
+}

metal-worker-pxe/profiles.tf

@@ -0,0 +1,20 @@
+// Container Linux Install profile (from release.core-os.net)
+resource "matchbox_profile" "bootkube-worker-pxe" {
+  name   = "bootkube-worker-pxe"
+  kernel = "http://${var.container_linux_channel}.release.core-os.net/amd64-usr/${var.container_linux_version}/coreos_production_pxe.vmlinuz"
+
+  initrd = [
+    "http://${var.container_linux_channel}.release.core-os.net/amd64-usr/${var.container_linux_version}/coreos_production_pxe_image.cpio.gz",
+  ]
+
+  args = [
+    "root=/dev/sda1",
+    "coreos.config.url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
+    "coreos.first_boot=yes",
+    "console=tty0",
+    "console=ttyS0",
+    "kvm-intel.nested=1",
+  ]
+
+  container_linux_config = "${file("${path.module}/cl/bootkube-worker.yaml.tmpl")}"
+}

metal-worker-pxe/ssh.tf

@@ -0,0 +1,22 @@
+# Secure copy kubeconfig to all nodes to activate kubelet.service
+resource "null_resource" "copy-kubeconfig" {
+  count = "${length(var.worker_names)}"
+
+  connection {
+    type    = "ssh"
+    host    = "${element(var.worker_domains, count.index)}"
+    user    = "core"
+    timeout = "60m"
+  }
+
+  provisioner "file" {
+    content     = "${var.kubeconfig}"
+    destination = "$HOME/kubeconfig"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
+    ]
+  }
+}

metal-worker-pxe/variables.tf

@@ -0,0 +1,55 @@
+variable "cluster_name" {
+  description = "Cluster name"
+  type        = "string"
+}
+
+variable "matchbox_http_endpoint" {
+  type        = "string"
+  description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)"
+}
+
+variable "container_linux_channel" {
+  type        = "string"
+  description = "Container Linux channel corresponding to the container_linux_version"
+}
+
+variable "container_linux_version" {
+  type        = "string"
+  description = "Container Linux version of the kernel/initrd to PXE or the image to install"
+}
+
+variable "ssh_authorized_key" {
+  type        = "string"
+  description = "SSH public key to set as an authorized key"
+}
+
+# machines
+# Terraform's crude "type system" does properly support lists of maps so we do this.
+
+variable "controller_domains" {
+  type = "list"
+}
+
+variable "worker_names" {
+  type = "list"
+}
+
+variable "worker_macs" {
+  type = "list"
+}
+
+variable "worker_domains" {
+  type = "list"
+}
+
+# bootkube
+
+variable "kubeconfig" {
+  type = "string"
+}
+
+variable "kube_dns_service_ip" {
+  description = "Kubernetes service IP for kube-dns (must be within server_cidr)"
+  type        = "string"
+  default     = "10.3.0.10"
+}