Add firewall and security rules for Cilium/Hubble metrics

* Add firewall or security riles to allow node-to-node traffic
on ports 9962-9965 for Cilium and Hubble metrics. Cilium runs
with host network, so these require cloud firewall changes

CHANGES.md

@@ -4,6 +4,7 @@ Notable changes between versions.
 
 ## Latest
 
+* Add firewall rules and security group rules for Cilium and Hubble metrics ([#1449](https://github.com/poseidon/typhoon/pull/1449))
 * Update Cilium from v1.15.3 to [v1.15.4](https://github.com/cilium/cilium/releases/tag/v1.15.4)
 * Update flannel from v0.24.4 to [v0.25.1](https://github.com/flannel-io/flannel/releases/tag/v0.25.1)
 

aws/fedora-coreos/kubernetes/security.tf

@@ -92,6 +92,30 @@ resource "aws_security_group_rule" "controller-cilium-health-self" {
   self      = true
 }
 
+resource "aws_security_group_rule" "controller-cilium-metrics" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 9962
+  to_port                  = 9965
+  source_security_group_id = aws_security_group.worker.id
+}
+
+resource "aws_security_group_rule" "controller-cilium-metrics-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type      = "ingress"
+  protocol  = "tcp"
+  from_port = 9962
+  to_port   = 9965
+  self      = true
+}
+
 # IANA VXLAN default
 resource "aws_security_group_rule" "controller-vxlan" {
   count = var.networking == "flannel" ? 1 : 0
@@ -379,6 +403,30 @@ resource "aws_security_group_rule" "worker-cilium-health-self" {
   self      = true
 }
 
+resource "aws_security_group_rule" "worker-cilium-metrics" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 9962
+  to_port                  = 9965
+  source_security_group_id = aws_security_group.controller.id
+}
+
+resource "aws_security_group_rule" "worker-cilium-metrics-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type      = "ingress"
+  protocol  = "tcp"
+  from_port = 9962
+  to_port   = 9965
+  self      = true
+}
+
 # IANA VXLAN default
 resource "aws_security_group_rule" "worker-vxlan" {
   count = var.networking == "flannel" ? 1 : 0

aws/flatcar-linux/kubernetes/security.tf

@@ -92,6 +92,30 @@ resource "aws_security_group_rule" "controller-cilium-health-self" {
   self      = true
 }
 
+resource "aws_security_group_rule" "controller-cilium-metrics" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 9962
+  to_port                  = 9965
+  source_security_group_id = aws_security_group.worker.id
+}
+
+resource "aws_security_group_rule" "controller-cilium-metrics-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.controller.id
+
+  type      = "ingress"
+  protocol  = "tcp"
+  from_port = 9962
+  to_port   = 9965
+  self      = true
+}
+
 # IANA VXLAN default
 resource "aws_security_group_rule" "controller-vxlan" {
   count = var.networking == "flannel" ? 1 : 0
@@ -379,6 +403,30 @@ resource "aws_security_group_rule" "worker-cilium-health-self" {
   self      = true
 }
 
+resource "aws_security_group_rule" "worker-cilium-metrics" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 9962
+  to_port                  = 9965
+  source_security_group_id = aws_security_group.controller.id
+}
+
+resource "aws_security_group_rule" "worker-cilium-metrics-self" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  security_group_id = aws_security_group.worker.id
+
+  type      = "ingress"
+  protocol  = "tcp"
+  from_port = 9962
+  to_port   = 9965
+  self      = true
+}
+
 # IANA VXLAN default
 resource "aws_security_group_rule" "worker-vxlan" {
   count = var.networking == "flannel" ? 1 : 0

azure/fedora-coreos/kubernetes/security.tf

@@ -121,7 +121,7 @@ resource "azurerm_network_security_rule" "controller-cilium-health" {
 
   name                         = "allow-cilium-health"
   network_security_group_name  = azurerm_network_security_group.controller.name
-  priority                     = "2019"
+  priority                     = "2018"
   access                       = "Allow"
   direction                    = "Inbound"
   protocol                     = "Tcp"
@@ -131,6 +131,22 @@ resource "azurerm_network_security_rule" "controller-cilium-health" {
   destination_address_prefixes = azurerm_subnet.controller.address_prefixes
 }
 
+resource "azurerm_network_security_rule" "controller-cilium-metrics" {
+  resource_group_name = azurerm_resource_group.cluster.name
+  count               = var.networking == "cilium" ? 1 : 0
+
+  name                         = "allow-cilium-metrics"
+  network_security_group_name  = azurerm_network_security_group.controller.name
+  priority                     = "2019"
+  access                       = "Allow"
+  direction                    = "Inbound"
+  protocol                     = "Tcp"
+  source_port_range            = "*"
+  destination_port_range       = "9962-9965"
+  source_address_prefixes      = concat(azurerm_subnet.controller.address_prefixes, azurerm_subnet.worker.address_prefixes)
+  destination_address_prefixes = azurerm_subnet.controller.address_prefixes
+}
+
 resource "azurerm_network_security_rule" "controller-vxlan" {
   resource_group_name = azurerm_resource_group.cluster.name
 
@@ -303,7 +319,7 @@ resource "azurerm_network_security_rule" "worker-cilium-health" {
 
   name                         = "allow-cilium-health"
   network_security_group_name  = azurerm_network_security_group.worker.name
-  priority                     = "2014"
+  priority                     = "2013"
   access                       = "Allow"
   direction                    = "Inbound"
   protocol                     = "Tcp"
@@ -313,6 +329,22 @@ resource "azurerm_network_security_rule" "worker-cilium-health" {
   destination_address_prefixes = azurerm_subnet.worker.address_prefixes
 }
 
+resource "azurerm_network_security_rule" "worker-cilium-metrics" {
+  resource_group_name = azurerm_resource_group.cluster.name
+  count               = var.networking == "cilium" ? 1 : 0
+
+  name                         = "allow-cilium-metrics"
+  network_security_group_name  = azurerm_network_security_group.worker.name
+  priority                     = "2014"
+  access                       = "Allow"
+  direction                    = "Inbound"
+  protocol                     = "Tcp"
+  source_port_range            = "*"
+  destination_port_range       = "9962-9965"
+  source_address_prefixes      = concat(azurerm_subnet.controller.address_prefixes, azurerm_subnet.worker.address_prefixes)
+  destination_address_prefixes = azurerm_subnet.worker.address_prefixes
+}
+
 resource "azurerm_network_security_rule" "worker-vxlan" {
   resource_group_name = azurerm_resource_group.cluster.name
 

azure/flatcar-linux/kubernetes/security.tf

@@ -121,7 +121,7 @@ resource "azurerm_network_security_rule" "controller-cilium-health" {
 
   name                         = "allow-cilium-health"
   network_security_group_name  = azurerm_network_security_group.controller.name
-  priority                     = "2019"
+  priority                     = "2018"
   access                       = "Allow"
   direction                    = "Inbound"
   protocol                     = "Tcp"
@@ -131,6 +131,22 @@ resource "azurerm_network_security_rule" "controller-cilium-health" {
   destination_address_prefixes = azurerm_subnet.controller.address_prefixes
 }
 
+resource "azurerm_network_security_rule" "controller-cilium-metrics" {
+  resource_group_name = azurerm_resource_group.cluster.name
+  count               = var.networking == "cilium" ? 1 : 0
+
+  name                         = "allow-cilium-metrics"
+  network_security_group_name  = azurerm_network_security_group.controller.name
+  priority                     = "2019"
+  access                       = "Allow"
+  direction                    = "Inbound"
+  protocol                     = "Tcp"
+  source_port_range            = "*"
+  destination_port_range       = "9962-9965"
+  source_address_prefixes      = concat(azurerm_subnet.controller.address_prefixes, azurerm_subnet.worker.address_prefixes)
+  destination_address_prefixes = azurerm_subnet.controller.address_prefixes
+}
+
 resource "azurerm_network_security_rule" "controller-vxlan" {
   resource_group_name = azurerm_resource_group.cluster.name
 
@@ -303,7 +319,7 @@ resource "azurerm_network_security_rule" "worker-cilium-health" {
 
   name                         = "allow-cilium-health"
   network_security_group_name  = azurerm_network_security_group.worker.name
-  priority                     = "2014"
+  priority                     = "2013"
   access                       = "Allow"
   direction                    = "Inbound"
   protocol                     = "Tcp"
@@ -313,6 +329,22 @@ resource "azurerm_network_security_rule" "worker-cilium-health" {
   destination_address_prefixes = azurerm_subnet.worker.address_prefixes
 }
 
+resource "azurerm_network_security_rule" "worker-cilium-metrics" {
+  resource_group_name = azurerm_resource_group.cluster.name
+  count               = var.networking == "cilium" ? 1 : 0
+
+  name                         = "allow-cilium-metrics"
+  network_security_group_name  = azurerm_network_security_group.worker.name
+  priority                     = "2014"
+  access                       = "Allow"
+  direction                    = "Inbound"
+  protocol                     = "Tcp"
+  source_port_range            = "*"
+  destination_port_range       = "9962-9965"
+  source_address_prefixes      = concat(azurerm_subnet.controller.address_prefixes, azurerm_subnet.worker.address_prefixes)
+  destination_address_prefixes = azurerm_subnet.worker.address_prefixes
+}
+
 resource "azurerm_network_security_rule" "worker-vxlan" {
   resource_group_name = azurerm_resource_group.cluster.name
 

digital-ocean/fedora-coreos/kubernetes/network.tf

@@ -32,6 +32,13 @@ resource "digitalocean_firewall" "rules" {
     source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
   }
 
+  # Cilium metrics
+  inbound_rule {
+    protocol    = "tcp"
+    port_range  = "9962-9965"
+    source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
+  }
+
   # IANA vxlan (flannel, calico)
   inbound_rule {
     protocol    = "udp"

digital-ocean/flatcar-linux/kubernetes/network.tf

@@ -32,6 +32,13 @@ resource "digitalocean_firewall" "rules" {
     source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
   }
 
+  # Cilium metrics
+  inbound_rule {
+    protocol    = "tcp"
+    port_range  = "9962-9965"
+    source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name]
+  }
+
   # IANA vxlan (flannel, calico)
   inbound_rule {
     protocol    = "udp"

google-cloud/fedora-coreos/kubernetes/network.tf

@@ -112,13 +112,14 @@ resource "google_compute_firewall" "internal-vxlan" {
   target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 }
 
-# Cilium VXLAN
+# Cilium
-resource "google_compute_firewall" "internal-linux-vxlan" {
+resource "google_compute_firewall" "internal-cilium" {
   count = var.networking == "cilium" ? 1 : 0
 
-  name    = "${var.cluster_name}-linux-vxlan"
+  name    = "${var.cluster_name}-cilium"
   network = google_compute_network.network.name
 
+  # vxlan
   allow {
     protocol = "udp"
     ports    = [8472]
@@ -128,12 +129,17 @@ resource "google_compute_firewall" "internal-linux-vxlan" {
   allow {
     protocol = "icmp"
   }
-
   allow {
     protocol = "tcp"
     ports    = [4240]
   }
 
+  # metrics
+  allow {
+    protocol = "tcp"
+    ports    = [9962, 9963, 9964, 9965]
+  }
+
   source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
   target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 }

google-cloud/flatcar-linux/kubernetes/network.tf

@@ -112,13 +112,14 @@ resource "google_compute_firewall" "internal-vxlan" {
   target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 }
 
-# Cilium VXLAN
+# Cilium
-resource "google_compute_firewall" "internal-linux-vxlan" {
+resource "google_compute_firewall" "internal-cilium" {
   count = var.networking == "cilium" ? 1 : 0
 
-  name    = "${var.cluster_name}-linux-vxlan"
+  name    = "${var.cluster_name}-cilium"
   network = google_compute_network.network.name
 
+  # vxlan
   allow {
     protocol = "udp"
     ports    = [8472]
@@ -128,12 +129,17 @@ resource "google_compute_firewall" "internal-linux-vxlan" {
   allow {
     protocol = "icmp"
   }
-
   allow {
     protocol = "tcp"
     ports    = [4240]
   }
 
+  # metrics
+  allow {
+    protocol = "tcp"
+    ports    = [9962, 9963, 9964, 9965]
+  }
+
   source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
   target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 }