From 4175b3ba66e1035612aa5be80ba12d5d9fa610b5 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 16 Jul 2017 22:45:34 -0700 Subject: [PATCH] Add Terraform module for bare-metal PXE workers --- metal-worker-pxe/cl/bootkube-worker.yaml.tmpl | 111 ++++++++++++++++++ metal-worker-pxe/groups.tf | 21 ++++ metal-worker-pxe/profiles.tf | 20 ++++ metal-worker-pxe/ssh.tf | 22 ++++ metal-worker-pxe/variables.tf | 55 +++++++++ 5 files changed, 229 insertions(+) create mode 100644 metal-worker-pxe/cl/bootkube-worker.yaml.tmpl create mode 100644 metal-worker-pxe/groups.tf create mode 100644 metal-worker-pxe/profiles.tf create mode 100644 metal-worker-pxe/ssh.tf create mode 100644 metal-worker-pxe/variables.tf diff --git a/metal-worker-pxe/cl/bootkube-worker.yaml.tmpl b/metal-worker-pxe/cl/bootkube-worker.yaml.tmpl new file mode 100644 index 00000000..761ad472 --- /dev/null +++ b/metal-worker-pxe/cl/bootkube-worker.yaml.tmpl @@ -0,0 +1,111 @@ +--- +systemd: + units: + - name: docker.service + enable: true + - name: locksmithd.service + mask: true + - name: kubelet.path + enable: true + contents: | + [Unit] + Description=Watch for kubeconfig + [Path] + PathExists=/etc/kubernetes/kubeconfig + [Install] + WantedBy=multi-user.target + - name: wait-for-dns.service + enable: true + contents: | + [Unit] + Description=Wait for DNS entries + Wants=systemd-resolved.service + Before=kubelet.service + [Service] + Type=oneshot + RemainAfterExit=true + ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' + [Install] + RequiredBy=kubelet.service + - name: kubelet.service + contents: | + [Unit] + Description=Kubelet via Hyperkube ACI + [Service] + EnvironmentFile=/etc/kubernetes/kubelet.env + Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ + --volume=resolv,kind=host,source=/etc/resolv.conf \ + --mount volume=resolv,target=/etc/resolv.conf \ + --volume var-lib-cni,kind=host,source=/var/lib/cni \ + --mount volume=var-lib-cni,target=/var/lib/cni \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log" + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets + ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests + ExecStartPre=/bin/mkdir -p /var/lib/cni + ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid + ExecStart=/usr/lib/coreos/kubelet-wrapper \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --require-kubeconfig \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --anonymous-auth=false \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --network-plugin=cni \ + --lock-file=/var/run/lock/kubelet.lock \ + --exit-on-lock-contention \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --allow-privileged \ + --hostname-override={{.domain_name}} \ + --node-labels=node-role.kubernetes.io/node \ + --cluster_dns={{.k8s_dns_service_ip}} \ + --cluster_domain=cluster.local + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid + Restart=always + RestartSec=5 + [Install] + WantedBy=multi-user.target + +storage: + {{ if index . "pxe" }} + disks: + - device: /dev/sda + wipe_table: true + partitions: + - label: ROOT + filesystems: + - name: root + mount: + device: "/dev/sda1" + format: "ext4" + create: + force: true + options: + - "-LROOT" + {{end}} + files: + - path: /etc/kubernetes/kubelet.env + filesystem: root + mode: 0644 + contents: + inline: | + KUBELET_IMAGE_URL=quay.io/coreos/hyperkube + KUBELET_IMAGE_TAG=v1.6.6_coreos.1 + - path: /etc/hostname + filesystem: root + mode: 0644 + contents: + inline: + {{.domain_name}} + - path: /etc/sysctl.d/max-user-watches.conf + filesystem: root + contents: + inline: | + fs.inotify.max_user_watches=16184 +passwd: + users: + - name: core + ssh_authorized_keys: + - {{.ssh_authorized_key}} diff --git a/metal-worker-pxe/groups.tf b/metal-worker-pxe/groups.tf new file mode 100644 index 00000000..8fc41bb8 --- /dev/null +++ b/metal-worker-pxe/groups.tf @@ -0,0 +1,21 @@ +resource "matchbox_group" "workers" { + count = "${length(var.worker_names)}" + name = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}" + profile = "${matchbox_profile.bootkube-worker-pxe.name}" + + selector { + mac = "${element(var.worker_macs, count.index)}" + } + + metadata { + pxe = "true" + domain_name = "${element(var.worker_domains, count.index)}" + etcd_endpoints = "${join(",", formatlist("%s:2379", var.controller_domains))}" + + # TODO + etcd_on_host = "true" + k8s_etcd_service_ip = "10.3.0.15" + k8s_dns_service_ip = "${var.kube_dns_service_ip}" + ssh_authorized_key = "${var.ssh_authorized_key}" + } +} diff --git a/metal-worker-pxe/profiles.tf b/metal-worker-pxe/profiles.tf new file mode 100644 index 00000000..3b6c9ef9 --- /dev/null +++ b/metal-worker-pxe/profiles.tf @@ -0,0 +1,20 @@ +// Container Linux Install profile (from release.core-os.net) +resource "matchbox_profile" "bootkube-worker-pxe" { + name = "bootkube-worker-pxe" + kernel = "http://${var.container_linux_channel}.release.core-os.net/amd64-usr/${var.container_linux_version}/coreos_production_pxe.vmlinuz" + + initrd = [ + "http://${var.container_linux_channel}.release.core-os.net/amd64-usr/${var.container_linux_version}/coreos_production_pxe_image.cpio.gz", + ] + + args = [ + "root=/dev/sda1", + "coreos.config.url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", + "coreos.first_boot=yes", + "console=tty0", + "console=ttyS0", + "kvm-intel.nested=1", + ] + + container_linux_config = "${file("${path.module}/cl/bootkube-worker.yaml.tmpl")}" +} diff --git a/metal-worker-pxe/ssh.tf b/metal-worker-pxe/ssh.tf new file mode 100644 index 00000000..77263b65 --- /dev/null +++ b/metal-worker-pxe/ssh.tf @@ -0,0 +1,22 @@ +# Secure copy kubeconfig to all nodes to activate kubelet.service +resource "null_resource" "copy-kubeconfig" { + count = "${length(var.worker_names)}" + + connection { + type = "ssh" + host = "${element(var.worker_domains, count.index)}" + user = "core" + timeout = "60m" + } + + provisioner "file" { + content = "${var.kubeconfig}" + destination = "$HOME/kubeconfig" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig", + ] + } +} diff --git a/metal-worker-pxe/variables.tf b/metal-worker-pxe/variables.tf new file mode 100644 index 00000000..d8f6cc68 --- /dev/null +++ b/metal-worker-pxe/variables.tf @@ -0,0 +1,55 @@ +variable "cluster_name" { + description = "Cluster name" + type = "string" +} + +variable "matchbox_http_endpoint" { + type = "string" + description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)" +} + +variable "container_linux_channel" { + type = "string" + description = "Container Linux channel corresponding to the container_linux_version" +} + +variable "container_linux_version" { + type = "string" + description = "Container Linux version of the kernel/initrd to PXE or the image to install" +} + +variable "ssh_authorized_key" { + type = "string" + description = "SSH public key to set as an authorized key" +} + +# machines +# Terraform's crude "type system" does properly support lists of maps so we do this. + +variable "controller_domains" { + type = "list" +} + +variable "worker_names" { + type = "list" +} + +variable "worker_macs" { + type = "list" +} + +variable "worker_domains" { + type = "list" +} + +# bootkube + +variable "kubeconfig" { + type = "string" +} + +variable "kube_dns_service_ip" { + description = "Kubernetes service IP for kube-dns (must be within server_cidr)" + type = "string" + default = "10.3.0.10" +}