Add an `install_container_networking` variable (default `true`)

* When `true`, the chosen container `networking` provider is installed during cluster bootstrap
* Set `false` to self-manage the container networking provider. This allows flannel, Calico, or Cilium
to be managed via Terraform (like any other Kubernetes resources). Nodes will be NotReady until you
apply the self-managed container networking provider. This may become the default in future.

CHANGES.md

@@ -9,7 +9,13 @@ Notable changes between versions.
 * Kubernetes [v1.29.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.29.md#v1292)
 * Update Cilium from v1.14.3 to [v1.15.1](https://github.com/cilium/cilium/releases/tag/v1.15.1)
 * Update flannel from v0.22.2 to [v0.24.2](https://github.com/flannel-io/flannel/releases/tag/v0.24.2)
-* Allow CNI `networking` to be set to "none" to skip bootstrapping flannel, Calico, or Cilium ([#1419](https://github.com/poseidon/typhoon/pull/1419))
+* Add an `install_container_networking` variable (default `true`)
+  * When `true`, the chosen container `networking` provider is installed during cluster bootstrap
+  * Set `false` to self-manage the container networking provider. This allows flannel, Calico, or Cilium
+  to be managed via Terraform (like any other Kubernetes resources). Nodes will be NotReady until you
+  apply the self-managed container networking provider. This may become the default in future.
+  * Continue to set `networking` to one of the three supported container networking providers. Most
+  require custom firewall / security policies be present across nodes so they have some infra tie-ins.
 
 ## v1.29.1
 

aws/fedora-coreos/kubernetes/bootstrap.tf

@@ -5,7 +5,7 @@ module "bootstrap" {
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers          = aws_route53_record.etcds.*.fqdn
-  networking            = var.networking
+  networking            = var.install_container_networking ? var.networking : "none"
   network_mtu           = var.network_mtu
   pod_cidr              = var.pod_cidr
   service_cidr          = var.service_cidr

aws/fedora-coreos/kubernetes/variables.tf

@@ -107,6 +107,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "network_mtu" {
   type        = number
   description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames."

aws/flatcar-linux/kubernetes/bootstrap.tf

@@ -5,7 +5,7 @@ module "bootstrap" {
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers          = aws_route53_record.etcds.*.fqdn
-  networking            = var.networking
+  networking            = var.install_container_networking ? var.networking : "none"
   network_mtu           = var.network_mtu
   pod_cidr              = var.pod_cidr
   service_cidr          = var.service_cidr

aws/flatcar-linux/kubernetes/variables.tf

@@ -107,6 +107,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "network_mtu" {
   type        = number
   description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames."

azure/fedora-coreos/kubernetes/bootstrap.tf

@@ -6,8 +6,7 @@ module "bootstrap" {
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone)
 
-  networking = var.networking
+  networking = var.install_container_networking ? var.networking : "none"
-
   # only effective with Calico networking
   # we should be able to use 1450 MTU, but in practice, 1410 was needed
   network_encapsulation = "vxlan"

azure/fedora-coreos/kubernetes/variables.tf

@@ -94,6 +94,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "host_cidr" {
   type        = string
   description = "CIDR IPv4 range to assign to instances"

azure/flatcar-linux/kubernetes/bootstrap.tf

@@ -6,8 +6,7 @@ module "bootstrap" {
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone)
 
-  networking = var.networking
+  networking = var.install_container_networking ? var.networking : "none"
-
   # only effective with Calico networking
   # we should be able to use 1450 MTU, but in practice, 1410 was needed
   network_encapsulation = "vxlan"

azure/flatcar-linux/kubernetes/variables.tf

@@ -100,6 +100,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "host_cidr" {
   type        = string
   description = "CIDR IPv4 range to assign to instances"

bare-metal/fedora-coreos/kubernetes/bootstrap.tf

@@ -5,7 +5,7 @@ module "bootstrap" {
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]
   etcd_servers                    = var.controllers.*.domain
-  networking                      = var.networking
+  networking                      = var.install_container_networking ? var.networking : "none"
   network_mtu                     = var.network_mtu
   network_ip_autodetection_method = var.network_ip_autodetection_method
   pod_cidr                        = var.pod_cidr

bare-metal/fedora-coreos/kubernetes/variables.tf

@@ -92,6 +92,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "network_mtu" {
   type        = number
   description = "CNI interface MTU (applies to calico only)"

bare-metal/flatcar-linux/kubernetes/bootstrap.tf

@@ -5,7 +5,7 @@ module "bootstrap" {
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]
   etcd_servers                    = var.controllers.*.domain
-  networking                      = var.networking
+  networking                      = var.install_container_networking ? var.networking : "none"
   network_mtu                     = var.network_mtu
   network_ip_autodetection_method = var.network_ip_autodetection_method
   pod_cidr                        = var.pod_cidr

bare-metal/flatcar-linux/kubernetes/variables.tf

@@ -91,6 +91,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "network_mtu" {
   type        = number
   description = "CNI interface MTU (applies to calico only)"

digital-ocean/fedora-coreos/kubernetes/bootstrap.tf

@@ -6,8 +6,7 @@ module "bootstrap" {
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers = digitalocean_record.etcds.*.fqdn
 
-  networking = var.networking
+  networking = var.install_container_networking ? var.networking : "none"
-
   # only effective with Calico networking
   network_encapsulation = "vxlan"
   network_mtu           = "1450"

digital-ocean/fedora-coreos/kubernetes/variables.tf

@@ -71,6 +71,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "pod_cidr" {
   type        = string
   description = "CIDR IPv4 range to assign Kubernetes pods"

digital-ocean/flatcar-linux/kubernetes/bootstrap.tf

@@ -6,8 +6,7 @@ module "bootstrap" {
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers = digitalocean_record.etcds.*.fqdn
 
-  networking = var.networking
+  networking = var.install_container_networking ? var.networking : "none"
-
   # only effective with Calico networking
   network_encapsulation = "vxlan"
   network_mtu           = "1450"

digital-ocean/flatcar-linux/kubernetes/variables.tf

@@ -71,6 +71,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "pod_cidr" {
   type        = string
   description = "CIDR IPv4 range to assign Kubernetes pods"

google-cloud/fedora-coreos/kubernetes/bootstrap.tf

@@ -5,7 +5,7 @@ module "bootstrap" {
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers          = [for fqdn in google_dns_record_set.etcds.*.name : trimsuffix(fqdn, ".")]
-  networking            = var.networking
+  networking            = var.install_container_networking ? var.networking : "none"
   network_mtu           = 1440
   pod_cidr              = var.pod_cidr
   service_cidr          = var.service_cidr

google-cloud/fedora-coreos/kubernetes/variables.tf

@@ -94,6 +94,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "pod_cidr" {
   type        = string
   description = "CIDR IPv4 range to assign Kubernetes pods"

google-cloud/flatcar-linux/kubernetes/bootstrap.tf

@@ -5,7 +5,7 @@ module "bootstrap" {
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]
   etcd_servers          = [for fqdn in google_dns_record_set.etcds.*.name : trimsuffix(fqdn, ".")]
-  networking            = var.networking
+  networking            = var.install_container_networking ? var.networking : "none"
   network_mtu           = 1440
   pod_cidr              = var.pod_cidr
   service_cidr          = var.service_cidr

google-cloud/flatcar-linux/kubernetes/variables.tf

@@ -94,6 +94,12 @@ variable "networking" {
   default     = "cilium"
 }
 
+variable "install_container_networking" {
+  type        = bool
+  description = "Install the chosen networking provider during cluster bootstrap (use false to self-manage)"
+  default     = true
+}
+
 variable "pod_cidr" {
   type        = string
   description = "CIDR IPv4 range to assign Kubernetes pods"