bare-metal: Ues Terraform templating for Container Linux configs

* Template bare-metal Container Linux configs with Terraform's
(limited) template_file module. This allows rendering problems
to be identified during `terraform plan` and is favored over
using the Matchbox templating feature when the configs are
served to PXE booting nodes.
* Writes a Matchbox profile for each machine, which will be served
as-is. The effect is the same, each node gets provisioned with its
own Container Linux config.

bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -9,12 +9,12 @@ systemd:
           contents: |
             [Service]
             Environment="ETCD_IMAGE_TAG=v3.2.0"
-            Environment="ETCD_NAME={{.etcd_name}}"
+            Environment="ETCD_NAME=${etcd_name}"
-            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://{{.domain_name}}:2379"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379"
-            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://{{.domain_name}}:2380"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380"
             Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
             Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
-            Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}"
+            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
             Environment="ETCD_STRICT_RECONFIG_CHECK=true"
             Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
             Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
@@ -79,11 +79,11 @@ systemd:
           --allow-privileged \
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns={{.k8s_dns_service_ip}} \
+          --cluster_dns=${k8s_dns_service_ip} \
           --cluster_domain=cluster.local \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
-          --hostname-override={{.domain_name}} \
+          --hostname-override=${domain_name} \
           --kubeconfig=/etc/kubernetes/kubeconfig \
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
@@ -137,7 +137,7 @@ storage:
       mode: 0644
       contents:
         inline:
-          {{.domain_name}}
+          ${domain_name}
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -159,17 +159,17 @@ storage:
           [ -d /opt/bootkube/assets/manifests-* ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
           [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
           [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
-          BOOTKUBE_ACI="${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
+          BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
-          BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.6.2}"
+          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.6.2}"
-          BOOTKUBE_ASSETS="${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
+          BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
           exec /usr/bin/rkt run \
             --trust-keys-from-https \
             --volume assets,kind=host,source=$BOOTKUBE_ASSETS \
             --mount volume=assets,target=/assets \
             --volume bootstrap,kind=host,source=/etc/kubernetes \
             --mount volume=bootstrap,target=/etc/kubernetes \
-            $RKT_OPTS \
+            $$RKT_OPTS \
-            ${BOOTKUBE_ACI}:${BOOTKUBE_VERSION} \
+            $${BOOTKUBE_ACI}:$${BOOTKUBE_VERSION} \
             --net=host \
             --dns=host \
             --exec=/bootkube -- start --asset-dir=/assets "$@"
@@ -177,5 +177,4 @@ passwd:
   users:
     - name: core
       ssh_authorized_keys:
-        - {{.ssh_authorized_key}}
+        - ${ssh_authorized_key}
-

bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -54,11 +54,11 @@ systemd:
           --allow-privileged \
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
-          --cluster_dns={{.k8s_dns_service_ip}} \
+          --cluster_dns=${k8s_dns_service_ip} \
           --cluster_domain=cluster.local \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
-          --hostname-override={{.domain_name}} \
+          --hostname-override=${domain_name} \
           --kubeconfig=/etc/kubernetes/kubeconfig \
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
@@ -101,7 +101,7 @@ storage:
       mode: 0644
       contents:
         inline:
-          {{.domain_name}}
+          ${domain_name}
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -111,5 +111,5 @@ passwd:
   users:
     - name: core
       ssh_authorized_keys:
-        - {{.ssh_authorized_key}}
+        - ${ssh_authorized_key}
 

bare-metal/container-linux/kubernetes/groups.tf

@@ -17,7 +17,7 @@ resource "matchbox_group" "container-linux-install" {
 resource "matchbox_group" "controller" {
   count   = "${length(var.controller_names)}"
   name    = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}"
-  profile = "${matchbox_profile.controller.name}"
+  profile = "${element(matchbox_profile.controllers.*.name, count.index)}"
 
   selector {
     mac = "${element(var.controller_macs, count.index)}"
@@ -25,29 +25,17 @@ resource "matchbox_group" "controller" {
   }
 
   metadata {
-    domain_name          = "${element(var.controller_domains, count.index)}"
-    etcd_name            = "${element(var.controller_names, count.index)}"
-    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}"
     etcd_on_host = "${var.experimental_self_hosted_etcd ? "false" : "true"}"
-    k8s_dns_service_ip   = "${module.bootkube.kube_dns_service_ip}"
-    ssh_authorized_key   = "${var.ssh_authorized_key}"
   }
 }
 
 resource "matchbox_group" "worker" {
   count   = "${length(var.worker_names)}"
   name    = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}"
-  profile = "${matchbox_profile.worker.name}"
+  profile = "${element(matchbox_profile.workers.*.name, count.index)}"
 
   selector {
     mac = "${element(var.worker_macs, count.index)}"
     os  = "installed"
   }
-
-  metadata {
-    domain_name        = "${element(var.worker_domains, count.index)}"
-    etcd_on_host       = "${var.experimental_self_hosted_etcd ? "false" : "true"}"
-    k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
-    ssh_authorized_key = "${var.ssh_authorized_key}"
-  }
 }

bare-metal/container-linux/kubernetes/profiles.tf

@@ -67,14 +67,42 @@ data "template_file" "cached-container-linux-install-config" {
   }
 }
 
-// Kubernetes Controller profile
+// Kubernetes Controller profiles
-resource "matchbox_profile" "controller" {
+resource "matchbox_profile" "controllers" {
-  name                   = "controller"
+  count                  = "${length(var.controller_names)}"
-  container_linux_config = "${file("${path.module}/cl/controller.yaml.tmpl")}"
+  name                   = "${format("%s-controller-%s", var.cluster_name, element(var.controller_names, count.index))}"
+  container_linux_config = "${element(data.template_file.controller-configs.*.rendered, count.index)}"
 }
 
-// Kubernetes Worker profile
+data "template_file" "controller-configs" {
-resource "matchbox_profile" "worker" {
+  count = "${length(var.controller_names)}"
-  name                   = "worker"
+
-  container_linux_config = "${file("${path.module}/cl/worker.yaml.tmpl")}"
+  template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
+
+  vars {
+    domain_name          = "${element(var.controller_domains, count.index)}"
+    etcd_name            = "${element(var.controller_names, count.index)}"
+    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}"
+    k8s_dns_service_ip   = "${module.bootkube.kube_dns_service_ip}"
+    ssh_authorized_key   = "${var.ssh_authorized_key}"
+  }
+}
+
+// Kubernetes Worker profiles
+resource "matchbox_profile" "workers" {
+  count                  = "${length(var.worker_names)}"
+  name                   = "${format("%s-worker-%s", var.cluster_name, element(var.worker_names, count.index))}"
+  container_linux_config = "${element(data.template_file.worker-configs.*.rendered, count.index)}"
+}
+
+data "template_file" "worker-configs" {
+  count = "${length(var.worker_names)}"
+
+  template = "${file("${path.module}/cl/worker.yaml.tmpl")}"
+
+  vars {
+    domain_name        = "${element(var.worker_domains, count.index)}"
+    k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
+    ssh_authorized_key = "${var.ssh_authorized_key}"
+  }
 }