From bca96bb124b2e74e2fa8e957eac0b6e5848c92bb Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 23 Sep 2017 11:49:12 -0700 Subject: [PATCH] bare-metal: Ues Terraform templating for Container Linux configs * Template bare-metal Container Linux configs with Terraform's (limited) template_file module. This allows rendering problems to be identified during `terraform plan` and is favored over using the Matchbox templating feature when the configs are served to PXE booting nodes. * Writes a Matchbox profile for each machine, which will be served as-is. The effect is the same, each node gets provisioned with its own Container Linux config. --- .../kubernetes/cl/controller.yaml.tmpl | 27 ++++++------ .../kubernetes/cl/worker.yaml.tmpl | 8 ++-- .../container-linux/kubernetes/groups.tf | 18 ++------ .../container-linux/kubernetes/profiles.tf | 44 +++++++++++++++---- 4 files changed, 56 insertions(+), 41 deletions(-) diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 9e41eb25..950c729b 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -9,12 +9,12 @@ systemd: contents: | [Service] Environment="ETCD_IMAGE_TAG=v3.2.0" - Environment="ETCD_NAME={{.etcd_name}}" - Environment="ETCD_ADVERTISE_CLIENT_URLS=https://{{.domain_name}}:2379" - Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://{{.domain_name}}:2380" + Environment="ETCD_NAME=${etcd_name}" + Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379" + Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380" Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379" Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380" - Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}" + Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}" Environment="ETCD_STRICT_RECONFIG_CHECK=true" Environment="ETCD_SSL_DIR=/etc/ssl/etcd" Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt" @@ -79,11 +79,11 @@ systemd: --allow-privileged \ --anonymous-auth=false \ --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns={{.k8s_dns_service_ip}} \ + --cluster_dns=${k8s_dns_service_ip} \ --cluster_domain=cluster.local \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ - --hostname-override={{.domain_name}} \ + --hostname-override=${domain_name} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ @@ -137,7 +137,7 @@ storage: mode: 0644 contents: inline: - {{.domain_name}} + ${domain_name} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -159,17 +159,17 @@ storage: [ -d /opt/bootkube/assets/manifests-* ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests - BOOTKUBE_ACI="${BOOTKUBE_ACI:-quay.io/coreos/bootkube}" - BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.6.2}" - BOOTKUBE_ASSETS="${BOOTKUBE_ASSETS:-/opt/bootkube/assets}" + BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}" + BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.6.2}" + BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}" exec /usr/bin/rkt run \ --trust-keys-from-https \ --volume assets,kind=host,source=$BOOTKUBE_ASSETS \ --mount volume=assets,target=/assets \ --volume bootstrap,kind=host,source=/etc/kubernetes \ --mount volume=bootstrap,target=/etc/kubernetes \ - $RKT_OPTS \ - ${BOOTKUBE_ACI}:${BOOTKUBE_VERSION} \ + $$RKT_OPTS \ + $${BOOTKUBE_ACI}:$${BOOTKUBE_VERSION} \ --net=host \ --dns=host \ --exec=/bootkube -- start --asset-dir=/assets "$@" @@ -177,5 +177,4 @@ passwd: users: - name: core ssh_authorized_keys: - - {{.ssh_authorized_key}} - + - ${ssh_authorized_key} diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 60207f3e..9d350ff6 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -54,11 +54,11 @@ systemd: --allow-privileged \ --anonymous-auth=false \ --client-ca-file=/etc/kubernetes/ca.crt \ - --cluster_dns={{.k8s_dns_service_ip}} \ + --cluster_dns=${k8s_dns_service_ip} \ --cluster_domain=cluster.local \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ - --hostname-override={{.domain_name}} \ + --hostname-override=${domain_name} \ --kubeconfig=/etc/kubernetes/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ @@ -101,7 +101,7 @@ storage: mode: 0644 contents: inline: - {{.domain_name}} + ${domain_name} - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: @@ -111,5 +111,5 @@ passwd: users: - name: core ssh_authorized_keys: - - {{.ssh_authorized_key}} + - ${ssh_authorized_key} diff --git a/bare-metal/container-linux/kubernetes/groups.tf b/bare-metal/container-linux/kubernetes/groups.tf index 98d9c60a..cffd74de 100644 --- a/bare-metal/container-linux/kubernetes/groups.tf +++ b/bare-metal/container-linux/kubernetes/groups.tf @@ -17,7 +17,7 @@ resource "matchbox_group" "container-linux-install" { resource "matchbox_group" "controller" { count = "${length(var.controller_names)}" name = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}" - profile = "${matchbox_profile.controller.name}" + profile = "${element(matchbox_profile.controllers.*.name, count.index)}" selector { mac = "${element(var.controller_macs, count.index)}" @@ -25,29 +25,17 @@ resource "matchbox_group" "controller" { } metadata { - domain_name = "${element(var.controller_domains, count.index)}" - etcd_name = "${element(var.controller_names, count.index)}" - etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}" - etcd_on_host = "${var.experimental_self_hosted_etcd ? "false" : "true"}" - k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - ssh_authorized_key = "${var.ssh_authorized_key}" + etcd_on_host = "${var.experimental_self_hosted_etcd ? "false" : "true"}" } } resource "matchbox_group" "worker" { count = "${length(var.worker_names)}" name = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}" - profile = "${matchbox_profile.worker.name}" + profile = "${element(matchbox_profile.workers.*.name, count.index)}" selector { mac = "${element(var.worker_macs, count.index)}" os = "installed" } - - metadata { - domain_name = "${element(var.worker_domains, count.index)}" - etcd_on_host = "${var.experimental_self_hosted_etcd ? "false" : "true"}" - k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - ssh_authorized_key = "${var.ssh_authorized_key}" - } } diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 430998a1..f64b33d8 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -67,14 +67,42 @@ data "template_file" "cached-container-linux-install-config" { } } -// Kubernetes Controller profile -resource "matchbox_profile" "controller" { - name = "controller" - container_linux_config = "${file("${path.module}/cl/controller.yaml.tmpl")}" +// Kubernetes Controller profiles +resource "matchbox_profile" "controllers" { + count = "${length(var.controller_names)}" + name = "${format("%s-controller-%s", var.cluster_name, element(var.controller_names, count.index))}" + container_linux_config = "${element(data.template_file.controller-configs.*.rendered, count.index)}" } -// Kubernetes Worker profile -resource "matchbox_profile" "worker" { - name = "worker" - container_linux_config = "${file("${path.module}/cl/worker.yaml.tmpl")}" +data "template_file" "controller-configs" { + count = "${length(var.controller_names)}" + + template = "${file("${path.module}/cl/controller.yaml.tmpl")}" + + vars { + domain_name = "${element(var.controller_domains, count.index)}" + etcd_name = "${element(var.controller_names, count.index)}" + etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}" + k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + ssh_authorized_key = "${var.ssh_authorized_key}" + } +} + +// Kubernetes Worker profiles +resource "matchbox_profile" "workers" { + count = "${length(var.worker_names)}" + name = "${format("%s-worker-%s", var.cluster_name, element(var.worker_names, count.index))}" + container_linux_config = "${element(data.template_file.worker-configs.*.rendered, count.index)}" +} + +data "template_file" "worker-configs" { + count = "${length(var.worker_names)}" + + template = "${file("${path.module}/cl/worker.yaml.tmpl")}" + + vars { + domain_name = "${element(var.worker_domains, count.index)}" + k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + ssh_authorized_key = "${var.ssh_authorized_key}" + } }