Remove an IPv4 address from Azure clusters

* Consolidate load balancer frontend IPs to just the minimal IPv4
and IPv6 addresses that are needed per load balancer. apiserver and
ingress use separate ports, so there is not a true need for a separate
public IPv4 address just for apiserver
* Some might prefer a separate IP just because it slightly hides the
apiserver, but these are public hosted endpoints that can be discovered
* Reduce the cost of an Azure cluster since IPv4 public IPs are billed
($3.60/mo/cluster)
This commit is contained in:
Dalton Hubble 2024-07-10 21:49:41 -07:00
parent 24b7f31c55
commit a4fab61066
No known key found for this signature in database
GPG Key ID: BD34C2E3EF32B7A0
7 changed files with 140 additions and 101 deletions

View File

@ -6,7 +6,6 @@ Notable changes between versions.
### Azure
* Rename `region` variable to `location` to align with Azure platform conventions
* Configure the virtual network and subnets with IPv6 private address space
* Change `host_cidr` variable (string) to a `network_cidr` object with `ipv4` and `ipv6` fields that list CIDR strings. Leave the variable unset to use the defaults. (**breaking**)
* Add support for dual-stack Kubernetes Ingress Load Balancing
@ -18,6 +17,8 @@ Notable changes between versions.
* Configure controller nodes to have a public IPv6 address
* Configure worker nodes to use outbound rules and the load balancer for SNAT
* Extend network security rules to allow IPv6 traffic, analogous to IPv4
* Rename `region` variable to `location` to align with Azure platform conventions ([#1469](https://github.com/poseidon/typhoon/pull/1469))
* Reduce the number of public IPv4 addresses needed for the Azure load balancer ([#1470](https://github.com/poseidon/typhoon/pull/1470))
```diff
module "cluster" {

View File

@ -8,16 +8,14 @@ locals {
# Discrete DNS records for each controller's private IPv4 for etcd usage
resource "azurerm_dns_a_record" "etcds" {
count = var.controller_count
resource_group_name = var.dns_zone_group
count = var.controller_count
# DNS Zone name where record should be created
zone_name = var.dns_zone
zone_name = var.dns_zone
resource_group_name = var.dns_zone_group
# DNS record
name = format("%s-etcd%d", var.cluster_name, count.index)
ttl = 300
# private IPv4 address for etcd
records = [azurerm_network_interface.controllers[count.index].private_ip_address]
}
@ -135,12 +133,20 @@ resource "azurerm_network_interface_security_group_association" "controllers" {
}
# Associate controller network interface with controller backend address pool
resource "azurerm_network_interface_backend_address_pool_association" "controllers" {
resource "azurerm_network_interface_backend_address_pool_association" "controllers-ipv4" {
count = var.controller_count
network_interface_id = azurerm_network_interface.controllers[count.index].id
ip_configuration_name = "ipv4"
backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id
backend_address_pool_id = azurerm_lb_backend_address_pool.controller-ipv4.id
}
resource "azurerm_network_interface_backend_address_pool_association" "controllers-ipv6" {
count = var.controller_count
network_interface_id = azurerm_network_interface.controllers[count.index].id
ip_configuration_name = "ipv6"
backend_address_pool_id = azurerm_lb_backend_address_pool.controller-ipv6.id
}
# Fedora CoreOS controllers

View File

@ -1,30 +1,30 @@
# DNS record for the apiserver load balancer
# DNS A record for the apiserver load balancer
resource "azurerm_dns_a_record" "apiserver" {
resource_group_name = var.dns_zone_group
# DNS Zone name where record should be created
zone_name = var.dns_zone
zone_name = var.dns_zone
resource_group_name = var.dns_zone_group
# DNS record
name = var.cluster_name
ttl = 300
# IPv4 address of apiserver load balancer
records = [azurerm_public_ip.apiserver-ipv4.ip_address]
records = [azurerm_public_ip.frontend-ipv4.ip_address]
}
# Static IPv4 address for the apiserver frontend
resource "azurerm_public_ip" "apiserver-ipv4" {
name = "${var.cluster_name}-apiserver-ipv4"
resource_group_name = azurerm_resource_group.cluster.name
location = var.location
sku = "Standard"
allocation_method = "Static"
# DNS AAAA record for the apiserver load balancer
resource "azurerm_dns_aaaa_record" "apiserver" {
# DNS Zone name where record should be created
zone_name = var.dns_zone
resource_group_name = var.dns_zone_group
# DNS record
name = var.cluster_name
ttl = 300
# IPv4 address of apiserver load balancer
records = [azurerm_public_ip.frontend-ipv6.ip_address]
}
# Static IPv4 address for the ingress frontend
resource "azurerm_public_ip" "ingress-ipv4" {
name = "${var.cluster_name}-ingress-ipv4"
# Static IPv4 address for the load balancer
resource "azurerm_public_ip" "frontend-ipv4" {
name = "${var.cluster_name}-frontend-ipv4"
resource_group_name = azurerm_resource_group.cluster.name
location = var.location
ip_version = "IPv4"
@ -32,9 +32,9 @@ resource "azurerm_public_ip" "ingress-ipv4" {
allocation_method = "Static"
}
# Static IPv6 address for the ingress frontend
resource "azurerm_public_ip" "ingress-ipv6" {
name = "${var.cluster_name}-ingress-ipv6"
# Static IPv6 address for the load balancer
resource "azurerm_public_ip" "frontend-ipv6" {
name = "${var.cluster_name}-frontend-ipv6"
resource_group_name = azurerm_resource_group.cluster.name
location = var.location
ip_version = "IPv6"
@ -50,38 +50,46 @@ resource "azurerm_lb" "cluster" {
sku = "Standard"
frontend_ip_configuration {
name = "apiserver"
public_ip_address_id = azurerm_public_ip.apiserver-ipv4.id
name = "frontend-ipv4"
public_ip_address_id = azurerm_public_ip.frontend-ipv4.id
}
frontend_ip_configuration {
name = "ingress-ipv4"
public_ip_address_id = azurerm_public_ip.ingress-ipv4.id
}
frontend_ip_configuration {
name = "ingress-ipv6"
public_ip_address_id = azurerm_public_ip.ingress-ipv6.id
name = "frontend-ipv6"
public_ip_address_id = azurerm_public_ip.frontend-ipv6.id
}
}
resource "azurerm_lb_rule" "apiserver" {
name = "apiserver"
resource "azurerm_lb_rule" "apiserver-ipv4" {
name = "apiserver-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "apiserver"
frontend_ip_configuration_name = "frontend-ipv4"
disable_outbound_snat = true
protocol = "Tcp"
frontend_port = 6443
backend_port = 6443
backend_address_pool_ids = [azurerm_lb_backend_address_pool.controller.id]
backend_address_pool_ids = [azurerm_lb_backend_address_pool.controller-ipv4.id]
probe_id = azurerm_lb_probe.apiserver.id
}
resource "azurerm_lb_rule" "apiserver-ipv6" {
name = "apiserver-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "frontend-ipv6"
disable_outbound_snat = true
protocol = "Tcp"
frontend_port = 6443
backend_port = 6443
backend_address_pool_ids = [azurerm_lb_backend_address_pool.controller-ipv6.id]
probe_id = azurerm_lb_probe.apiserver.id
}
resource "azurerm_lb_rule" "ingress-http-ipv4" {
name = "ingress-http-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv4"
frontend_ip_configuration_name = "frontend-ipv4"
disable_outbound_snat = true
protocol = "Tcp"
@ -94,7 +102,7 @@ resource "azurerm_lb_rule" "ingress-http-ipv4" {
resource "azurerm_lb_rule" "ingress-https-ipv4" {
name = "ingress-https-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv4"
frontend_ip_configuration_name = "frontend-ipv4"
disable_outbound_snat = true
protocol = "Tcp"
@ -107,7 +115,7 @@ resource "azurerm_lb_rule" "ingress-https-ipv4" {
resource "azurerm_lb_rule" "ingress-http-ipv6" {
name = "ingress-http-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv6"
frontend_ip_configuration_name = "frontend-ipv6"
disable_outbound_snat = true
protocol = "Tcp"
@ -120,7 +128,7 @@ resource "azurerm_lb_rule" "ingress-http-ipv6" {
resource "azurerm_lb_rule" "ingress-https-ipv6" {
name = "ingress-https-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv6"
frontend_ip_configuration_name = "frontend-ipv6"
disable_outbound_snat = true
protocol = "Tcp"
@ -133,8 +141,13 @@ resource "azurerm_lb_rule" "ingress-https-ipv6" {
# Backend Address Pools
# Address pool of controllers
resource "azurerm_lb_backend_address_pool" "controller" {
name = "controller"
resource "azurerm_lb_backend_address_pool" "controller-ipv4" {
name = "controller-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
}
resource "azurerm_lb_backend_address_pool" "controller-ipv6" {
name = "controller-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
}
@ -182,7 +195,7 @@ resource "azurerm_lb_outbound_rule" "outbound-ipv4" {
loadbalancer_id = azurerm_lb.cluster.id
backend_address_pool_id = azurerm_lb_backend_address_pool.worker-ipv4.id
frontend_ip_configuration {
name = "ingress-ipv4"
name = "frontend-ipv4"
}
}
@ -192,6 +205,6 @@ resource "azurerm_lb_outbound_rule" "outbound-ipv6" {
loadbalancer_id = azurerm_lb.cluster.id
backend_address_pool_id = azurerm_lb_backend_address_pool.worker-ipv6.id
frontend_ip_configuration {
name = "ingress-ipv6"
name = "frontend-ipv6"
}
}

View File

@ -6,12 +6,12 @@ output "kubeconfig-admin" {
# Outputs for Kubernetes Ingress
output "ingress_static_ipv4" {
value = azurerm_public_ip.ingress-ipv4.ip_address
value = azurerm_public_ip.frontend-ipv4.ip_address
description = "IPv4 address of the load balancer for distributing traffic to Ingress controllers"
}
output "ingress_static_ipv6" {
value = azurerm_public_ip.ingress-ipv6.ip_address
value = azurerm_public_ip.frontend-ipv6.ip_address
description = "IPv6 address of the load balancer for distributing traffic to Ingress controllers"
}

View File

@ -14,16 +14,14 @@ locals {
# Discrete DNS records for each controller's private IPv4 for etcd usage
resource "azurerm_dns_a_record" "etcds" {
count = var.controller_count
resource_group_name = var.dns_zone_group
count = var.controller_count
# DNS Zone name where record should be created
zone_name = var.dns_zone
zone_name = var.dns_zone
resource_group_name = var.dns_zone_group
# DNS record
name = format("%s-etcd%d", var.cluster_name, count.index)
ttl = 300
# private IPv4 address for etcd
records = [azurerm_network_interface.controllers[count.index].private_ip_address]
}
@ -156,13 +154,21 @@ resource "azurerm_network_interface_security_group_association" "controllers" {
network_security_group_id = azurerm_network_security_group.controller.id
}
# Associate controller network interface with controller backend address pool
resource "azurerm_network_interface_backend_address_pool_association" "controllers" {
# Associate controller network interface with controller backend address pools
resource "azurerm_network_interface_backend_address_pool_association" "controllers-ipv4" {
count = var.controller_count
network_interface_id = azurerm_network_interface.controllers[count.index].id
ip_configuration_name = "ipv4"
backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id
backend_address_pool_id = azurerm_lb_backend_address_pool.controller-ipv4.id
}
resource "azurerm_network_interface_backend_address_pool_association" "controllers-ipv6" {
count = var.controller_count
network_interface_id = azurerm_network_interface.controllers[count.index].id
ip_configuration_name = "ipv6"
backend_address_pool_id = azurerm_lb_backend_address_pool.controller-ipv6.id
}
# Flatcar Linux controllers

View File

@ -1,30 +1,30 @@
# DNS record for the apiserver load balancer
# DNS A record for the apiserver load balancer
resource "azurerm_dns_a_record" "apiserver" {
resource_group_name = var.dns_zone_group
# DNS Zone name where record should be created
zone_name = var.dns_zone
zone_name = var.dns_zone
resource_group_name = var.dns_zone_group
# DNS record
name = var.cluster_name
ttl = 300
# IPv4 address of apiserver load balancer
records = [azurerm_public_ip.apiserver-ipv4.ip_address]
records = [azurerm_public_ip.frontend-ipv4.ip_address]
}
# Static IPv4 address for the apiserver frontend
resource "azurerm_public_ip" "apiserver-ipv4" {
name = "${var.cluster_name}-apiserver-ipv4"
resource_group_name = azurerm_resource_group.cluster.name
location = var.location
sku = "Standard"
allocation_method = "Static"
# DNS AAAA record for the apiserver load balancer
resource "azurerm_dns_aaaa_record" "apiserver" {
# DNS Zone name where record should be created
zone_name = var.dns_zone
resource_group_name = var.dns_zone_group
# DNS record
name = var.cluster_name
ttl = 300
# IPv6 address of apiserver load balancer
records = [azurerm_public_ip.frontend-ipv6.ip_address]
}
# Static IPv4 address for the ingress frontend
resource "azurerm_public_ip" "ingress-ipv4" {
name = "${var.cluster_name}-ingress-ipv4"
# Static IPv4 address for the load balancer
resource "azurerm_public_ip" "frontend-ipv4" {
name = "${var.cluster_name}-frontend-ipv4"
resource_group_name = azurerm_resource_group.cluster.name
location = var.location
ip_version = "IPv4"
@ -32,8 +32,8 @@ resource "azurerm_public_ip" "ingress-ipv4" {
allocation_method = "Static"
}
# Static IPv6 address for the ingress frontend
resource "azurerm_public_ip" "ingress-ipv6" {
# Static IPv6 address for the load balancer
resource "azurerm_public_ip" "frontend-ipv6" {
name = "${var.cluster_name}-ingress-ipv6"
resource_group_name = azurerm_resource_group.cluster.name
location = var.location
@ -50,38 +50,46 @@ resource "azurerm_lb" "cluster" {
sku = "Standard"
frontend_ip_configuration {
name = "apiserver"
public_ip_address_id = azurerm_public_ip.apiserver-ipv4.id
name = "frontend-ipv4"
public_ip_address_id = azurerm_public_ip.frontend-ipv4.id
}
frontend_ip_configuration {
name = "ingress-ipv4"
public_ip_address_id = azurerm_public_ip.ingress-ipv4.id
}
frontend_ip_configuration {
name = "ingress-ipv6"
public_ip_address_id = azurerm_public_ip.ingress-ipv6.id
name = "frontend-ipv6"
public_ip_address_id = azurerm_public_ip.frontend-ipv6.id
}
}
resource "azurerm_lb_rule" "apiserver" {
name = "apiserver"
resource "azurerm_lb_rule" "apiserver-ipv4" {
name = "apiserver-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "apiserver"
frontend_ip_configuration_name = "frontend-ipv4"
disable_outbound_snat = true
protocol = "Tcp"
frontend_port = 6443
backend_port = 6443
backend_address_pool_ids = [azurerm_lb_backend_address_pool.controller.id]
backend_address_pool_ids = [azurerm_lb_backend_address_pool.controller-ipv4.id]
probe_id = azurerm_lb_probe.apiserver.id
}
resource "azurerm_lb_rule" "apiserver-ipv6" {
name = "apiserver-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "frontend-ipv6"
disable_outbound_snat = true
protocol = "Tcp"
frontend_port = 6443
backend_port = 6443
backend_address_pool_ids = [azurerm_lb_backend_address_pool.controller-ipv6.id]
probe_id = azurerm_lb_probe.apiserver.id
}
resource "azurerm_lb_rule" "ingress-http-ipv4" {
name = "ingress-http-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv4"
frontend_ip_configuration_name = "frontend-ipv4"
disable_outbound_snat = true
protocol = "Tcp"
@ -94,7 +102,7 @@ resource "azurerm_lb_rule" "ingress-http-ipv4" {
resource "azurerm_lb_rule" "ingress-https-ipv4" {
name = "ingress-https-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv4"
frontend_ip_configuration_name = "frontend-ipv4"
disable_outbound_snat = true
protocol = "Tcp"
@ -107,7 +115,7 @@ resource "azurerm_lb_rule" "ingress-https-ipv4" {
resource "azurerm_lb_rule" "ingress-http-ipv6" {
name = "ingress-http-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv6"
frontend_ip_configuration_name = "frontend-ipv6"
disable_outbound_snat = true
protocol = "Tcp"
@ -120,7 +128,7 @@ resource "azurerm_lb_rule" "ingress-http-ipv6" {
resource "azurerm_lb_rule" "ingress-https-ipv6" {
name = "ingress-https-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
frontend_ip_configuration_name = "ingress-ipv6"
frontend_ip_configuration_name = "frontend-ipv6"
disable_outbound_snat = true
protocol = "Tcp"
@ -133,8 +141,13 @@ resource "azurerm_lb_rule" "ingress-https-ipv6" {
# Backend Address Pools
# Address pool of controllers
resource "azurerm_lb_backend_address_pool" "controller" {
name = "controller"
resource "azurerm_lb_backend_address_pool" "controller-ipv4" {
name = "controller-ipv4"
loadbalancer_id = azurerm_lb.cluster.id
}
resource "azurerm_lb_backend_address_pool" "controller-ipv6" {
name = "controller-ipv6"
loadbalancer_id = azurerm_lb.cluster.id
}
@ -182,7 +195,7 @@ resource "azurerm_lb_outbound_rule" "outbound-ipv4" {
loadbalancer_id = azurerm_lb.cluster.id
backend_address_pool_id = azurerm_lb_backend_address_pool.worker-ipv4.id
frontend_ip_configuration {
name = "ingress-ipv4"
name = "frontend-ipv4"
}
}
@ -192,6 +205,6 @@ resource "azurerm_lb_outbound_rule" "outbound-ipv6" {
loadbalancer_id = azurerm_lb.cluster.id
backend_address_pool_id = azurerm_lb_backend_address_pool.worker-ipv6.id
frontend_ip_configuration {
name = "ingress-ipv6"
name = "frontend-ipv6"
}
}

View File

@ -6,12 +6,12 @@ output "kubeconfig-admin" {
# Outputs for Kubernetes Ingress
output "ingress_static_ipv4" {
value = azurerm_public_ip.ingress-ipv4.ip_address
value = azurerm_public_ip.frontend-ipv4.ip_address
description = "IPv4 address of the load balancer for distributing traffic to Ingress controllers"
}
output "ingress_static_ipv6" {
value = azurerm_public_ip.ingress-ipv6.ip_address
value = azurerm_public_ip.frontend-ipv6.ip_address
description = "IPv6 address of the load balancer for distributing traffic to Ingress controllers"
}