Combine NLBs to use one NLB per cluster

* Simplify clusters to come with a single NLB
* Listen for apiserver traffic on port 6443 and forward
to controllers (with healthy apiserver)
* Listen for ingress traffic on ports 80/443 and forward
to workers (with healthy ingress controller)
* Reduce cost of default clusters by 1 NLB ($18.14/month)
* Keep using CNAME records to the `ingress_dns_name` NLB and
the nginx-ingress addon for Ingress (up to a few million RPS)
* Users with heavy traffic (many million RPS) can create their
own separate NLB(s) for Ingress and use the new output worker
target groups
* Fix issue where additional worker pools come with an
extraneous network load balancer

CHANGES.md

@@ -10,7 +10,13 @@ Notable changes between versions.
 #### AWS
 
 * Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
-  * Update NLB, security groups, and generated kubeconfig's
+* Combine apiserver and ingress NLBs ([#249](https://github.com/poseidon/typhoon/pull/249))
+  * Simplify clusters to come with one NLB. Reduce cost by ~$18/month per cluster.
+  * Users may keep using CNAME records to `ingress_dns_name` and the `nginx-ingress` addon for Ingress (up to a few million RPS)
+  * Users with heavy traffic (many million RPS) should create a separate NLB(s) for Ingress instead
+  * Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver)
+  * Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller)
+* Worker pools (advanced) no longer include an extraneous load balancer
 
 #### Bare-Metal
 

aws/container-linux/kubernetes/nlb.tf

@@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
 
   # AWS recommends their special "alias" records for ELBs
   alias {
-    name                   = "${aws_lb.apiserver.dns_name}"
-    zone_id                = "${aws_lb.apiserver.zone_id}"
+    name                   = "${aws_lb.nlb.dns_name}"
+    zone_id                = "${aws_lb.nlb.zone_id}"
     evaluate_target_health = true
   }
 }
 
-# Network Load Balancer for apiservers
-resource "aws_lb" "apiserver" {
-  name               = "${var.cluster_name}-apiserver"
+# Network Load Balancer for apiservers and ingress
+resource "aws_lb" "nlb" {
+  name               = "${var.cluster_name}-nlb"
   load_balancer_type = "network"
   internal           = false
 
@@ -24,9 +24,9 @@ resource "aws_lb" "apiserver" {
   enable_cross_zone_load_balancing = true
 }
 
-# Forward TCP traffic to controllers
+# Forward TCP apiserver traffic to controllers
 resource "aws_lb_listener" "apiserver-https" {
-  load_balancer_arn = "${aws_lb.apiserver.arn}"
+  load_balancer_arn = "${aws_lb.nlb.arn}"
   protocol          = "TCP"
   port              = "6443"
 
@@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
   }
 }
 
+# Forward HTTP ingress traffic to workers
+resource "aws_lb_listener" "ingress-http" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 80
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_http_arn}"
+  }
+}
+
+# Forward HTTPS ingress traffic to workers
+resource "aws_lb_listener" "ingress-https" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 443
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_https_arn}"
+  }
+}
+
 # Target group of controllers
 resource "aws_lb_target_group" "controllers" {
   name        = "${var.cluster_name}-controllers"

aws/container-linux/kubernetes/outputs.tf

@@ -1,8 +1,18 @@
 output "ingress_dns_name" {
-  value       = "${module.workers.ingress_dns_name}"
+  value       = "${aws_lb.nlb.dns_name}"
   description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
 }
 
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${module.workers.target_group_http_arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${module.workers.target_group_https_arn}"
+}
+
 # Outputs for worker pools
 
 output "vpc_id" {

aws/container-linux/kubernetes/workers/ingress.tf

@@ -1,39 +1,4 @@
-# Network Load Balancer for Ingress
-resource "aws_lb" "ingress" {
-  name               = "${var.name}-ingress"
-  load_balancer_type = "network"
-  internal           = false
-
-  subnets = ["${var.subnet_ids}"]
-
-  enable_cross_zone_load_balancing = true
-}
-
-# Forward HTTP traffic to workers
-resource "aws_lb_listener" "ingress-http" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 80
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-http.arn}"
-  }
-}
-
-# Forward HTTPS traffic to workers
-resource "aws_lb_listener" "ingress-https" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 443
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-https.arn}"
-  }
-}
-
-# Network Load Balancer target groups of instances
+# Target groups of instances for use with load balancers
 
 resource "aws_lb_target_group" "workers-http" {
   name        = "${var.name}-workers-http"

aws/container-linux/kubernetes/workers/outputs.tf

@@ -1,4 +1,9 @@
-output "ingress_dns_name" {
-  value       = "${aws_lb.ingress.dns_name}"
-  description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${aws_lb_target_group.workers-http.arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${aws_lb_target_group.workers-https.arn}"
 }

aws/fedora-atomic/kubernetes/nlb.tf

@@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
 
   # AWS recommends their special "alias" records for ELBs
   alias {
-    name                   = "${aws_lb.apiserver.dns_name}"
-    zone_id                = "${aws_lb.apiserver.zone_id}"
+    name                   = "${aws_lb.nlb.dns_name}"
+    zone_id                = "${aws_lb.nlb.zone_id}"
     evaluate_target_health = true
   }
 }
 
-# Network Load Balancer for apiservers
-resource "aws_lb" "apiserver" {
-  name               = "${var.cluster_name}-apiserver"
+# Network Load Balancer for apiservers and ingress
+resource "aws_lb" "nlb" {
+  name               = "${var.cluster_name}-nlb"
   load_balancer_type = "network"
   internal           = false
 
@@ -24,11 +24,11 @@ resource "aws_lb" "apiserver" {
   enable_cross_zone_load_balancing = true
 }
 
-# Forward TCP traffic to controllers
+# Forward TCP apiserver traffic to controllers
 resource "aws_lb_listener" "apiserver-https" {
-  load_balancer_arn = "${aws_lb.apiserver.arn}"
+  load_balancer_arn = "${aws_lb.nlb.arn}"
   protocol          = "TCP"
-  port              = "443"
+  port              = "6443"
 
   default_action {
     type             = "forward"
@@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
   }
 }
 
+# Forward HTTP ingress traffic to workers
+resource "aws_lb_listener" "ingress-http" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 80
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_http_arn}"
+  }
+}
+
+# Forward HTTPS ingress traffic to workers
+resource "aws_lb_listener" "ingress-https" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 443
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_https_arn}"
+  }
+}
+
 # Target group of controllers
 resource "aws_lb_target_group" "controllers" {
   name        = "${var.cluster_name}-controllers"
@@ -43,12 +67,12 @@ resource "aws_lb_target_group" "controllers" {
   target_type = "instance"
 
   protocol = "TCP"
-  port     = 443
+  port     = 6443
 
   # TCP health check for apiserver
   health_check {
     protocol = "TCP"
-    port     = 443
+    port     = 6443
 
     # NLBs required to use same healthy and unhealthy thresholds
     healthy_threshold   = 3
@@ -65,5 +89,5 @@ resource "aws_lb_target_group_attachment" "controllers" {
 
   target_group_arn = "${aws_lb_target_group.controllers.arn}"
   target_id        = "${element(aws_instance.controllers.*.id, count.index)}"
-  port             = 443
+  port             = 6443
 }

aws/fedora-atomic/kubernetes/outputs.tf

@@ -1,8 +1,18 @@
 output "ingress_dns_name" {
-  value       = "${module.workers.ingress_dns_name}"
+  value       = "${aws_lb.nlb.dns_name}"
   description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
 }
 
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${module.workers.target_group_http_arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${module.workers.target_group_https_arn}"
+}
+
 # Outputs for worker pools
 
 output "vpc_id" {

aws/fedora-atomic/kubernetes/workers/ingress.tf

@@ -1,39 +1,4 @@
-# Network Load Balancer for Ingress
-resource "aws_lb" "ingress" {
-  name               = "${var.name}-ingress"
-  load_balancer_type = "network"
-  internal           = false
-
-  subnets = ["${var.subnet_ids}"]
-
-  enable_cross_zone_load_balancing = true
-}
-
-# Forward HTTP traffic to workers
-resource "aws_lb_listener" "ingress-http" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 80
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-http.arn}"
-  }
-}
-
-# Forward HTTPS traffic to workers
-resource "aws_lb_listener" "ingress-https" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 443
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-https.arn}"
-  }
-}
-
-# Network Load Balancer target groups of instances
+# Target groups of instances for use with load balancers
 
 resource "aws_lb_target_group" "workers-http" {
   name        = "${var.name}-workers-http"

aws/fedora-atomic/kubernetes/workers/outputs.tf

@@ -1,4 +1,9 @@
-output "ingress_dns_name" {
-  value       = "${aws_lb.ingress.dns_name}"
-  description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${aws_lb_target_group.workers-http.arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${aws_lb_target_group.workers-https.arn}"
 }