mirror of
https://github.com/puppetmaster/typhoon.git
synced 2024-12-26 06:19:33 +01:00
bbbaf949f9
* Add "lb" outbound rule for worker TCP _and_ UDP traffic * Fix Azure worker nodes clock synchronization being inactive due to timeouts reaching the CoreOS / Flatcar NTP pool * Fix Azure worker nodes not providing outbount UDP connectivity Background: Azure provides VMs outbound connectivity either by having a public IP or via an SNAT masquerade feature bundled with their virtual load balancing abstraction (in contrast with, say, a NAT gateway). Azure worker nodes have only a private IP, but are associated with the cluster load balancer's backend pool and ingress frontend IP. Outbound traffic uses SNAT with this frontend IP. A subtle detail with Azure SNAT seems to be that since both inbound lb_rule's are TCP only, outbound UDP traffic isn't SNAT'd (highlights the reasons Azure shouldn't have conflated inbound load balancing with outbound SNAT concepts). However, adding a separate outbound rule and disabling outbound SNAT on our ingress lb_rule's we can tell Azure to continue load balancing as before, and support outbound SNAT for worker traffic of both the TCP and UDP protocol. Fixes clock synchronization timeouts: ``` systemd-timesyncd[786]: Timed out waiting for reply from 45.79.36.123:123 (3.flatcar.pool.ntp.org) ``` Azure controller nodes have their own public IP, so controllers (and etcd) nodes have not had clock synchronization or outbound UDP issues
162 lines
4.5 KiB
HCL
162 lines
4.5 KiB
HCL
# DNS record for the apiserver load balancer
|
|
resource "azurerm_dns_a_record" "apiserver" {
|
|
resource_group_name = var.dns_zone_group
|
|
|
|
# DNS Zone name where record should be created
|
|
zone_name = var.dns_zone
|
|
|
|
# DNS record
|
|
name = var.cluster_name
|
|
ttl = 300
|
|
|
|
# IPv4 address of apiserver load balancer
|
|
records = [azurerm_public_ip.apiserver-ipv4.ip_address]
|
|
}
|
|
|
|
# Static IPv4 address for the apiserver frontend
|
|
resource "azurerm_public_ip" "apiserver-ipv4" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "${var.cluster_name}-apiserver-ipv4"
|
|
location = var.region
|
|
sku = "Standard"
|
|
allocation_method = "Static"
|
|
}
|
|
|
|
# Static IPv4 address for the ingress frontend
|
|
resource "azurerm_public_ip" "ingress-ipv4" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "${var.cluster_name}-ingress-ipv4"
|
|
location = var.region
|
|
sku = "Standard"
|
|
allocation_method = "Static"
|
|
}
|
|
|
|
# Network Load Balancer for apiservers and ingress
|
|
resource "azurerm_lb" "cluster" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = var.cluster_name
|
|
location = var.region
|
|
sku = "Standard"
|
|
|
|
frontend_ip_configuration {
|
|
name = "apiserver"
|
|
public_ip_address_id = azurerm_public_ip.apiserver-ipv4.id
|
|
}
|
|
|
|
frontend_ip_configuration {
|
|
name = "ingress"
|
|
public_ip_address_id = azurerm_public_ip.ingress-ipv4.id
|
|
}
|
|
}
|
|
|
|
resource "azurerm_lb_rule" "apiserver" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "apiserver"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
frontend_ip_configuration_name = "apiserver"
|
|
|
|
protocol = "Tcp"
|
|
frontend_port = 6443
|
|
backend_port = 6443
|
|
backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id
|
|
probe_id = azurerm_lb_probe.apiserver.id
|
|
}
|
|
|
|
resource "azurerm_lb_rule" "ingress-http" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "ingress-http"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
frontend_ip_configuration_name = "ingress"
|
|
disable_outbound_snat = true
|
|
|
|
protocol = "Tcp"
|
|
frontend_port = 80
|
|
backend_port = 80
|
|
backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
|
|
probe_id = azurerm_lb_probe.ingress.id
|
|
}
|
|
|
|
resource "azurerm_lb_rule" "ingress-https" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "ingress-https"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
frontend_ip_configuration_name = "ingress"
|
|
disable_outbound_snat = true
|
|
|
|
protocol = "Tcp"
|
|
frontend_port = 443
|
|
backend_port = 443
|
|
backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
|
|
probe_id = azurerm_lb_probe.ingress.id
|
|
}
|
|
|
|
# Worker outbound TCP/UDP SNAT
|
|
resource "azurerm_lb_outbound_rule" "worker-outbound" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "worker"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
frontend_ip_configuration {
|
|
name = "ingress"
|
|
}
|
|
|
|
protocol = "All"
|
|
backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
|
|
}
|
|
|
|
# Address pool of controllers
|
|
resource "azurerm_lb_backend_address_pool" "controller" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "controller"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
}
|
|
|
|
# Address pool of workers
|
|
resource "azurerm_lb_backend_address_pool" "worker" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "worker"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
}
|
|
|
|
# Health checks / probes
|
|
|
|
# TCP health check for apiserver
|
|
resource "azurerm_lb_probe" "apiserver" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "apiserver"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
protocol = "Tcp"
|
|
port = 6443
|
|
|
|
# unhealthy threshold
|
|
number_of_probes = 3
|
|
|
|
interval_in_seconds = 5
|
|
}
|
|
|
|
# HTTP health check for ingress
|
|
resource "azurerm_lb_probe" "ingress" {
|
|
resource_group_name = azurerm_resource_group.cluster.name
|
|
|
|
name = "ingress"
|
|
loadbalancer_id = azurerm_lb.cluster.id
|
|
protocol = "Http"
|
|
port = 10254
|
|
request_path = "/healthz"
|
|
|
|
# unhealthy threshold
|
|
number_of_probes = 3
|
|
|
|
interval_in_seconds = 5
|
|
}
|
|
|