From ab7913a0613abdb03bde00bdca1ec385d144380c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 8 Mar 2020 20:39:18 -0700 Subject: [PATCH] Accept initial worker node labels and taints map on bare-metal * Add `worker_node_labels` map from node name to a list of initial node label strings * Add `worker_node_taints` map from node name to a list of initial node taint strings * Unlike cloud platforms, bare-metal node labels and taints are defined via a map from node name to list of labels/taints. Bare-metal clusters may have heterogeneous hardware so per node labels and taints are accepted * Only worker node names are allowed. Workloads are not scheduled on controller nodes so altering their labels/taints isn't suitable ``` module "mercury" { ... worker_node_labels = { "node2" = ["role=special"] } worker_node_taints = { "node2" = ["role=special:NoSchedule"] } } ``` Related: https://github.com/poseidon/typhoon/issues/429 --- CHANGES.md | 7 ++++++- bare-metal/container-linux/kubernetes/cl/worker.yaml | 6 ++++++ bare-metal/container-linux/kubernetes/profiles.tf | 2 ++ bare-metal/container-linux/kubernetes/variables.tf | 12 ++++++++++++ bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 6 ++++++ bare-metal/fedora-coreos/kubernetes/profiles.tf | 2 ++ bare-metal/fedora-coreos/kubernetes/variables.tf | 12 ++++++++++++ docs/cl/bare-metal.md | 2 ++ docs/fedora-coreos/bare-metal.md | 2 ++ 9 files changed, 50 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 265bd8af..5c22af18 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,11 +17,16 @@ Notable changes between versions. #### Azure * Upgrade to `terraform-provider-azurerm` [v2.0+](https://www.terraform.io/docs/providers/azurerm/guides/2.0-upgrade-guide.html) (action required) + * Change `worker_priority` from `Low` to `Spot` if used (action required) * Switch to Azure's new Linux VM and Linux VM Scale Set resources - * If set, change `worker_priority` from `Low` to `Spot` (action required) * Set controller's Azure disk caching to None * Associate subnets (in addition to NICs) with security groups (aesthetic) +#### Bare-Metal + +* Add `worker_node_labels` map variable for per-worker node labels ([#663](https://github.com/poseidon/typhoon/pull/663)) +* Add `worker_node_taints` map variable for per-worker node taints ([#663](https://github.com/poseidon/typhoon/pull/663)) + #### Google Cloud * Fix `worker_node_labels` on Fedora CoreOS ([#651](https://github.com/poseidon/typhoon/pull/651)) diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 8f6e984f..8ffb1d2e 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -93,6 +93,12 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{~ for label in compact(split(",", node_labels)) ~} + --node-labels=${label} \ + %{~ endfor ~} + %{~ for taint in compact(split(",", node_taints)) ~} + --register-with-taints=${taint} \ + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/bare-metal/container-linux/kubernetes/profiles.tf b/bare-metal/container-linux/kubernetes/profiles.tf index 6bbaa1d5..98efdcc3 100644 --- a/bare-metal/container-linux/kubernetes/profiles.tf +++ b/bare-metal/container-linux/kubernetes/profiles.tf @@ -188,6 +188,8 @@ data "template_file" "worker-configs" { cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key + node_labels = join(",", lookup(var.worker_node_labels, var.workers.*.name[count.index], [])) + node_taints = join(",", lookup(var.worker_node_taints, var.workers.*.name[count.index], [])) } } diff --git a/bare-metal/container-linux/kubernetes/variables.tf b/bare-metal/container-linux/kubernetes/variables.tf index 80dd70bc..27b8ac8e 100644 --- a/bare-metal/container-linux/kubernetes/variables.tf +++ b/bare-metal/container-linux/kubernetes/variables.tf @@ -55,6 +55,18 @@ variable "clc_snippets" { default = {} } +variable "worker_node_labels" { + type = map(list(string)) + description = "Map from worker names to lists of initial node labels" + default = {} +} + +variable "worker_node_taints" { + type = map(list(string)) + description = "Map from worker names to lists of initial node taints" + default = {} +} + # configuration variable "k8s_domain_name" { diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index c0e09b58..d1d5e8aa 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -68,6 +68,12 @@ systemd: --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ + %{~ for label in compact(split(",", node_labels)) ~} + --node-labels=${label} \ + %{~ endfor ~} + %{~ for taint in compact(split(",", node_taints)) ~} + --register-with-taints=${taint} \ + %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf index fefed885..8d75b830 100644 --- a/bare-metal/fedora-coreos/kubernetes/profiles.tf +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -96,6 +96,8 @@ data "template_file" "worker-configs" { cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip cluster_domain_suffix = var.cluster_domain_suffix ssh_authorized_key = var.ssh_authorized_key + node_labels = join(",", lookup(var.worker_node_labels, var.workers.*.name[count.index], [])) + node_taints = join(",", lookup(var.worker_node_taints, var.workers.*.name[count.index], [])) } } diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 8e956754..45fc0341 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -56,6 +56,18 @@ variable "snippets" { default = {} } +variable "worker_node_labels" { + type = map(list(string)) + description = "Map from worker names to lists of initial node labels" + default = {} +} + +variable "worker_node_taints" { + type = map(list(string)) + description = "Map from worker names to lists of initial node taints" + default = {} +} + # configuration variable "k8s_domain_name" { diff --git a/docs/cl/bare-metal.md b/docs/cl/bare-metal.md index 77587027..5cf4f777 100644 --- a/docs/cl/bare-metal.md +++ b/docs/cl/bare-metal.md @@ -361,4 +361,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | kernel_args | Additional kernel args to provide at PXE boot | [] | ["kvm-intel.nested=1"] | +| worker_node_labels | Map from worker name to list of initial node labels | {} | {"node2" = ["role=special"]} | +| worker_node_taints | Map from worker name to list of initial node taints | {} | {"node2" = ["role=special:NoSchedule"]} | diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index 42974f6f..c95fbf3d 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -347,4 +347,6 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | kernel_args | Additional kernel args to provide at PXE boot | [] | ["kvm-intel.nested=1"] | +| worker_node_labels | Map from worker name to list of initial node labels | {} | {"node2" = ["role=special"]} | +| worker_node_taints | Map from worker name to list of initial node taints | {} | {"node2" = ["role=special:NoSchedule"]} |