Allow custom initial node taints on worker pool nodes

* Add `node_taints` variable to worker modules to set custom
initial node taints on cloud platforms that support auto-scaling
worker pools of heterogeneous nodes (i.e. AWS, Azure, GCP)
* Worker pools could use custom `node_labels` to allowed workloads
to select among differentiated nodes, while custom `node_taints`
allows a worker pool's nodes to be tainted as special to prevent
scheduling, except by workloads that explicitly tolerate the
taint
* Expose `daemonset_tolerations` in AWS, Azure, and GCP kubernetes
cluster modules, to determine whether `kube-system` components
should tolerate the custom taint (advanced use covered in docs)

Rel: #550, #663
Closes #429

google-cloud/flatcar-linux/kubernetes/bootstrap.tf

@@ -12,6 +12,7 @@ module "bootstrap" {
   cluster_domain_suffix = var.cluster_domain_suffix
   enable_reporting      = var.enable_reporting
   enable_aggregation    = var.enable_aggregation
+  daemonset_tolerations = var.daemonset_tolerations
 
   // temporary
   external_apiserver_port = 443

google-cloud/flatcar-linux/kubernetes/variables.tf

@@ -130,3 +130,8 @@ variable "cluster_domain_suffix" {
   default     = "cluster.local"
 }
 
+variable "daemonset_tolerations" {
+  type        = list(string)
+  description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
+  default     = []
+}

google-cloud/flatcar-linux/kubernetes/workers/cl/worker.yaml

@@ -70,6 +70,9 @@ systemd:
           %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
           %{~ endfor ~}
+          %{~ for taint in split(",", node_taints) ~}
+          --register-with-taints=${taint} \
+          %{~ endfor ~}
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --rotate-certificates \

google-cloud/flatcar-linux/kubernetes/workers/variables.tf

@@ -84,6 +84,12 @@ variable "node_labels" {
   default     = []
 }
 
+variable "node_taints" {
+  type        = list(string)
+  description = "List of initial node taints"
+  default     = []
+}
+
 # unofficial, undocumented, unsupported, temporary
 
 variable "cluster_domain_suffix" {

google-cloud/flatcar-linux/kubernetes/workers/workers.tf

@@ -86,6 +86,7 @@ data "template_file" "worker-config" {
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
     node_labels            = join(",", var.node_labels)
+    node_taints            = join(",", var.node_taints)
   }
 }