From 4775e9d0f72a7edd7358163d8aa114e1242cc20c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 27 Oct 2019 00:49:46 -0700 Subject: [PATCH] Upgrade Calico v3.9.2 to v3.10.0 * Allow advertising Kubernetes service ClusterIPs to BGPPeer routers via a BGPConfiguration * Improve EdgeRouter docs about routes and BGP * https://docs.projectcalico.org/v3.10/release-notes/ * https://docs.projectcalico.org/v3.10/networking/advertise-service-ips --- CHANGES.md | 2 + aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/bootstrap.tf | 2 +- docs/topics/hardware.md | 174 +++++++++++------- .../container-linux/kubernetes/bootstrap.tf | 2 +- 9 files changed, 113 insertions(+), 77 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b3501c58..5756c660 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,8 @@ Notable changes between versions. ## Latest +* Upgrade Calico from v3.9.2 to [v3.10.0](https://docs.projectcalico.org/v3.10/release-notes/) + * Allow advertising service ClusterIPs to peer routers via a [BGPConfiguration](https://docs.projectcalico.org/v3.10/networking/advertise-service-ips) * Switch `kube-proxy` from iptables to ipvs mode ([#574](https://github.com/poseidon/typhoon/pull/574)) #### Addons diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 51dadec1..a68b40a1 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 1bc9d791..d87f5073 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 8ca2a295..b80fbca6 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index fa3a628d..b806a357 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index f51f9bf4..5ebfee5f 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 0764a4eb..c4a348b6 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/docs/topics/hardware.md b/docs/topics/hardware.md index 5751b18b..d95ad227 100644 --- a/docs/topics/hardware.md +++ b/docs/topics/hardware.md @@ -6,6 +6,39 @@ Typhoon ensures certain networking hardware integrates well with bare-metal Kube Ubiquiti EdgeRouters and EdgeOS work well with bare-metal Kubernetes clusters. Familiarity with EdgeRouter setup and CLI usage is required. +### DHCP + +Assign static IPs to clients with known MAC addresses. This is called a static mapping by EdgeOS. Configure the router with the commands based on region inventory. + +``` +configure +show service dhcp-server shared-network +set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME mac-address MACADDR +set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME ip-address 10.0.0.20 +``` + +### DNS + +Add DNS A records to static IPs as `dnsmasq` host-records. + +``` +configure +set service dns forwarding options host-record=node.example.com,10.0.0.20 +``` + +Forward `*.svc.cluster.local` queries to the CoreDNS Kubernetes service IP to allow clients to resolve Kubernetes services. + +``` +set service dns forwarding options server=/svc.cluster.local/10.3.0.10 +commit-confirm +``` + +Restart `dnsmasq`. + +``` +sudo /etc/init.d/dnsmasq restart +``` + ### PXE Ubiquiti EdgeRouters can provide a PXE-enabled network boot environment for client machines. @@ -65,55 +98,88 @@ set service dns forwarding options tftp-root=/config/tftpboot commit-confirm ``` -### DHCP +### Routing -Assign static IPs to clients with known MAC addresses. This is called a static mapping by EdgeOS. Configure the router with the commands based on region inventory. +#### Static Routes -``` -configure -show service dhcp-server shared-network -set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME mac-address MACADDR -set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME ip-address 10.0.0.20 -``` - -### DNS - -Assign DNS A records to nodes as options to `dnsmasq`. - -``` -configure -set service dns forwarding options host-record=node.example.com,10.0.0.20 -``` - -Restart `dnsmasq`. - -``` -sudo /etc/init.d/dnsmasq restart -``` - -Configure queries for `*.svc.cluster.local` to be forwarded to the Kubernetes `coredns` service IP to allow hosts to resolve cluster-local Kubernetes names. - -``` -configure -show service dns forwarding -set service dns forwarding options server=/svc.cluster.local/10.3.0.10 -commit-confirm -``` - -### Kubernetes Services - -Add static routes for the Kubernetes IPv4 service range to Kubernetes node(s) so hosts can route to Kubernetes services (default: 10.3.0.0/16). +Add static route(s) to Kubernetes node(s) that can route to Kubernetes service IPs (default: 10.3.0.0/16). Kubernetes service IPs will become routeable on the LAN. ``` configure show protocols static route set protocols static route 10.3.0.0/16 next-hop NODE_IP -... commit-confirm ``` !!! note - Adding multiple next-hop nodes provides equal-cost multi-path (ECMP) routing. EdgeOS v2.0+ is required. The kernel in prior versions used flow-hash to balanced packets, whereas with v2.0, round-robin sessions are used. + Adding multiple next-hop nodes provides equal-cost multi-path (ECMP) routing. EdgeOS v2.0+ is required. The kernel in prior versions used flow-hash to balanced packets, whereas with v2.0, round-robin sessions are used. + +#### BGP + +EdgeRouter can exchange routes with other autonomous systems, including a cluster's Calico AS. Peers will exchange `podCIDR` routes to make individual pods routeable on the LAN. + +Define the EdgeRouter AS (if undefined). + +``` +configure +show protocols bgp 1 +set protocols bgp 1 parameters router-id ROUTER_IP +``` + +Peer with node(s) in another AS (eg. Calico default 64512) + +``` +set protocols bgp 1 neighbor NODE1_IP remote-as 64512 +set protocols bgp 1 neighbor NODE2_IP remote-as 64512 +set protocols bgp 1 neighbor NODE3_IP remote-as 64512 +commit-confirm +``` + +Configure Calico node(s) as to peer with the EdgeRouter. + +``` +apiVersion: crd.projectcalico.org/v1 +kind: BGPPeer +metadata: + name: NODE_NAME-to-edgerouter +spec: + peerIP: ROUTER_IP + asNumber: 1 + node: NODE_NAME +``` + +Or, if every node is to be peered (i.e. full mesh), define a global BGPPeer. + +``` +apiVersion: crd.projectcalico.org/v1 +kind: BGPPeer +metadata: + name: global +spec: + peerIP: ROUTER_IP + asNumber: 1 +``` + +If Calico nodes should advertise Kubernetes Service IPs (i.e. ClusterIPs) as well, add a `BGPConfiguration`. + +``` +apiVersion: crd.projectcalico.org/v1 +kind: BGPConfiguration +metadata: + name: default +spec: + logSeverityScreen: Info + nodeToNodeMeshEnabled: true + serviceClusterIPs: + - cidr: 10.3.0.0/16 +``` + +Show a summary of peers and exchanged routes. + +``` +show ip bgp summary +show ip route bgp +``` ### Port Forwarding @@ -150,35 +216,3 @@ set service gui https-port 4443 commit-confirm ``` -### BGP - -Add the EdgeRouter as a global BGP peer for nodes in a Kubernetes cluster (requires Calico). Neighbors will exchange `podCIDR` routes and individual pods will become routable on the LAN. - -Configure node(s) as BGP neighbors. - -``` -show protocols bgp 1 -set protocols bgp 1 parameters router-id LAN_IP -set protocols bgp 1 neighbor NODE1_IP remote-as 64512 -set protocols bgp 1 neighbor NODE2_IP remote-as 64512 -set protocols bgp 1 neighbor NODE3_IP remote-as 64512 -``` - -View the neighbors and exchanged routes. - -``` -show ip bgp neighbors -show ip route bgp -``` - -Be sure to register the peer by creating a Calico `BGPPeer` CRD with `kubectl apply`. - -``` -apiVersion: crd.projectcalico.org/v1 -kind: BGPPeer -metadata: - name: NAME -spec: - peerIP: LAN_IP - asNumber: 64512 -``` diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 9e3fa00e..878bfa4a 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e09d6bef33693455ee77b9e6b6882dc7d35c523c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=3c7334ab55b4ebef4109072da99452d59ee6179a" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]