Change kube-apiserver port from 443 to 6443

* Adjust firewall rules, security groups, cloud load balancers,
and generated kubeconfig's
* Facilitates some future simplifications and cost reductions
* Bare-Metal users who exposed kube-apiserver on a WAN via their
router or load balancer will need to adjust its configuration.
This is uncommon, most apiserver are on LAN and/or behind VPN
so no routing infrastructure is configured with the port number

CHANGES.md

@@ -6,10 +6,23 @@ Notable changes between versions.
 
 * Update etcd from v3.3.6 to v3.3.8 ([#243](https://github.com/poseidon/typhoon/pull/243), [#247](https://github.com/poseidon/typhoon/pull/247))
 
+#### AWS
+
+* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
+  * Update NLB, security groups, and generated kubeconfig's
+  * Facilitates some future simplifications and cost reductions
+
 #### Bare-Metal
 
+* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
+  * Users who exposed kube-apiserver on a WAN via their router/load-balancer will need to adjust its configuration (e.g. DNAT 6443). Most apiservers are on a LAN (internal, VPN-only, etc) so if you didn't specially configure network gear for 443, no change is needed. (possible action required)
 * Fix possible deadlock when provisioning clusters larger than 10 nodes ([#244](https://github.com/poseidon/typhoon/pull/244)) 
 
+#### DigitalOcean
+
+* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
+  * Update firewall rules and generated kubeconfig's
+
 #### Addons
 
 * Update CLUO from v0.6.0 to v0.7.0 ([#242](https://github.com/poseidon/typhoon/pull/242))

aws/container-linux/kubernetes/apiserver.tf

@@ -28,7 +28,7 @@ resource "aws_lb" "apiserver" {
 resource "aws_lb_listener" "apiserver-https" {
   load_balancer_arn = "${aws_lb.apiserver.arn}"
   protocol          = "TCP"
-  port              = "443"
+  port              = "6443"
 
   default_action {
     type             = "forward"
@@ -43,12 +43,12 @@ resource "aws_lb_target_group" "controllers" {
   target_type = "instance"
 
   protocol = "TCP"
-  port     = 443
+  port     = 6443
 
   # TCP health check for apiserver
   health_check {
     protocol = "TCP"
-    port     = 443
+    port     = 6443
 
     # NLBs required to use same healthy and unhealthy thresholds
     healthy_threshold   = 3
@@ -65,5 +65,5 @@ resource "aws_lb_target_group_attachment" "controllers" {
 
   target_group_arn = "${aws_lb_target_group.controllers.arn}"
   target_id        = "${element(aws_instance.controllers.*.id, count.index)}"
-  port             = 443
+  port             = 6443
 }

aws/container-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

aws/container-linux/kubernetes/security.tf

@@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" {
 
   type        = "ingress"
   protocol    = "tcp"
-  from_port   = 443
+  from_port   = 6443
-  to_port     = 443
+  to_port     = 6443
   cidr_blocks = ["0.0.0.0/0"]
 }
 

aws/fedora-atomic/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

aws/fedora-atomic/kubernetes/security.tf

@@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" {
 
   type        = "ingress"
   protocol    = "tcp"
-  from_port   = 443
+  from_port   = 6443
-  to_port     = 443
+  to_port     = 6443
   cidr_blocks = ["0.0.0.0/0"]
 }
 

bare-metal/container-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name                    = "${var.cluster_name}"
   api_servers                     = ["${var.k8s_domain_name}"]

bare-metal/fedora-atomic/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${var.k8s_domain_name}"]

digital-ocean/container-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

digital-ocean/container-linux/kubernetes/network.tf

@@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" {
 
   tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 
-  # allow ssh, http/https ingress, and peer-to-peer traffic
+  # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic
   inbound_rule = [
     {
       protocol         = "tcp"
@@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" {
       port_range       = "443"
       source_addresses = ["0.0.0.0/0", "::/0"]
     },
+    {
+      protocol         = "tcp"
+      port_range       = "6443"
+      source_addresses = ["0.0.0.0/0", "::/0"]
+    },
     {
       protocol    = "udp"
       port_range  = "1-65535"

digital-ocean/fedora-atomic/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

digital-ocean/fedora-atomic/kubernetes/network.tf

@@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" {
 
   tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 
-  # allow ssh, http/https ingress, and peer-to-peer traffic
+  # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic
   inbound_rule = [
     {
       protocol         = "tcp"
@@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" {
       port_range       = "443"
       source_addresses = ["0.0.0.0/0", "::/0"]
     },
+    {
+      protocol         = "tcp"
+      port_range       = "6443"
+      source_addresses = ["0.0.0.0/0", "::/0"]
+    },
     {
       protocol    = "udp"
       port_range  = "1-65535"

docs/addons/ingress.md

@@ -4,7 +4,7 @@ Nginx Ingress controller pods accept and demultiplex HTTP, HTTPS, TCP, or UDP tr
 
 ## AWS
 
-On AWS, an elastic load balancer distributes traffic across worker nodes (i.e. an auto-scaling group) running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic.
+On AWS, a network load balancer (NLB) distributes traffic across a target group  of worker nodes running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic.
 
 Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace.
 

google-cloud/container-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
@@ -11,4 +11,6 @@ module "bootkube" {
   pod_cidr              = "${var.pod_cidr}"
   service_cidr          = "${var.service_cidr}"
   cluster_domain_suffix = "${var.cluster_domain_suffix}"
+  // temporary
+  apiserver_port = 443
 }

google-cloud/fedora-atomic/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0"
 
   cluster_name          = "${var.cluster_name}"
   api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
@@ -14,4 +14,7 @@ module "bootkube" {
 
   # Fedora
   trusted_certs_dir = "/etc/pki/tls/certs"
+  
+  // temporary
+  apiserver_port = 443
 }