From 6c5a1964aadbb3d23a1c6c6b99cf8c44b763b21f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 Jun 2018 21:57:58 -0700 Subject: [PATCH] Change kube-apiserver port from 443 to 6443 * Adjust firewall rules, security groups, cloud load balancers, and generated kubeconfig's * Facilitates some future simplifications and cost reductions * Bare-Metal users who exposed kube-apiserver on a WAN via their router or load balancer will need to adjust its configuration. This is uncommon, most apiserver are on LAN and/or behind VPN so no routing infrastructure is configured with the port number --- CHANGES.md | 13 +++++++++++++ aws/container-linux/kubernetes/apiserver.tf | 8 ++++---- aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/container-linux/kubernetes/security.tf | 4 ++-- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/security.tf | 4 ++-- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/network.tf | 7 ++++++- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/network.tf | 7 ++++++- docs/addons/ingress.md | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 4 +++- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 5 ++++- 15 files changed, 47 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 417c967a..790a574f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,10 +6,23 @@ Notable changes between versions. * Update etcd from v3.3.6 to v3.3.8 ([#243](https://github.com/poseidon/typhoon/pull/243), [#247](https://github.com/poseidon/typhoon/pull/247)) +#### AWS + +* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Update NLB, security groups, and generated kubeconfig's + * Facilitates some future simplifications and cost reductions + #### Bare-Metal +* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Users who exposed kube-apiserver on a WAN via their router/load-balancer will need to adjust its configuration (e.g. DNAT 6443). Most apiservers are on a LAN (internal, VPN-only, etc) so if you didn't specially configure network gear for 443, no change is needed. (possible action required) * Fix possible deadlock when provisioning clusters larger than 10 nodes ([#244](https://github.com/poseidon/typhoon/pull/244)) +#### DigitalOcean + +* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Update firewall rules and generated kubeconfig's + #### Addons * Update CLUO from v0.6.0 to v0.7.0 ([#242](https://github.com/poseidon/typhoon/pull/242)) diff --git a/aws/container-linux/kubernetes/apiserver.tf b/aws/container-linux/kubernetes/apiserver.tf index 8cc5eed6..93cbcee4 100644 --- a/aws/container-linux/kubernetes/apiserver.tf +++ b/aws/container-linux/kubernetes/apiserver.tf @@ -28,7 +28,7 @@ resource "aws_lb" "apiserver" { resource "aws_lb_listener" "apiserver-https" { load_balancer_arn = "${aws_lb.apiserver.arn}" protocol = "TCP" - port = "443" + port = "6443" default_action { type = "forward" @@ -43,12 +43,12 @@ resource "aws_lb_target_group" "controllers" { target_type = "instance" protocol = "TCP" - port = 443 + port = 6443 # TCP health check for apiserver health_check { protocol = "TCP" - port = 443 + port = 6443 # NLBs required to use same healthy and unhealthy thresholds healthy_threshold = 3 @@ -65,5 +65,5 @@ resource "aws_lb_target_group_attachment" "controllers" { target_group_arn = "${aws_lb_target_group.controllers.arn}" target_id = "${element(aws_instance.controllers.*.id, count.index)}" - port = 443 + port = 6443 } diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 4da810cc..bb4694a6 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index 8534200b..8104080e 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" { type = "ingress" protocol = "tcp" - from_port = 443 - to_port = 443 + from_port = 6443 + to_port = 6443 cidr_blocks = ["0.0.0.0/0"] } diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 7f990934..a2c3aabc 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/security.tf b/aws/fedora-atomic/kubernetes/security.tf index 8534200b..8104080e 100644 --- a/aws/fedora-atomic/kubernetes/security.tf +++ b/aws/fedora-atomic/kubernetes/security.tf @@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" { type = "ingress" protocol = "tcp" - from_port = 443 - to_port = 443 + from_port = 6443 + to_port = 6443 cidr_blocks = ["0.0.0.0/0"] } diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 7662cb65..b5ed364d 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index df40c89f..9b71c657 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 1195cae0..39209899 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index fc257a03..312d7966 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" { tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - # allow ssh, http/https ingress, and peer-to-peer traffic + # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic inbound_rule = [ { protocol = "tcp" @@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" { port_range = "443" source_addresses = ["0.0.0.0/0", "::/0"] }, + { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, { protocol = "udp" port_range = "1-65535" diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index b9b57058..adbbb3a7 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/network.tf b/digital-ocean/fedora-atomic/kubernetes/network.tf index fc257a03..312d7966 100644 --- a/digital-ocean/fedora-atomic/kubernetes/network.tf +++ b/digital-ocean/fedora-atomic/kubernetes/network.tf @@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" { tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - # allow ssh, http/https ingress, and peer-to-peer traffic + # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic inbound_rule = [ { protocol = "tcp" @@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" { port_range = "443" source_addresses = ["0.0.0.0/0", "::/0"] }, + { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, { protocol = "udp" port_range = "1-65535" diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index 754bec80..e6c9e81e 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -4,7 +4,7 @@ Nginx Ingress controller pods accept and demultiplex HTTP, HTTPS, TCP, or UDP tr ## AWS -On AWS, an elastic load balancer distributes traffic across worker nodes (i.e. an auto-scaling group) running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic. +On AWS, a network load balancer (NLB) distributes traffic across a target group of worker nodes running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic. Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace. diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 51ab437a..1e1e9da5 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -11,4 +11,6 @@ module "bootkube" { pod_cidr = "${var.pod_cidr}" service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" + // temporary + apiserver_port = 443 } diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 4a93e35d..fc516b13 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -14,4 +14,7 @@ module "bootkube" { # Fedora trusted_certs_dir = "/etc/pki/tls/certs" + + // temporary + apiserver_port = 443 }