Combine NLBs to use one NLB per cluster

* Simplify clusters to come with a single NLB
* Listen for apiserver traffic on port 6443 and forward
to controllers (with healthy apiserver)
* Listen for ingress traffic on ports 80/443 and forward
to workers (with healthy ingress controller)
* Reduce cost of default clusters by 1 NLB ($18.14/month)
* Keep using CNAME records to the `ingress_dns_name` NLB and
the nginx-ingress addon for Ingress (up to a few million RPS)
* Users with heavy traffic (many million RPS) can create their
own separate NLB(s) for Ingress and use the new output worker
target groups
* Fix issue where additional worker pools come with an
extraneous network load balancer

aws/container-linux/kubernetes/nlb.tf

@@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
 
   # AWS recommends their special "alias" records for ELBs
   alias {
-    name                   = "${aws_lb.apiserver.dns_name}"
-    zone_id                = "${aws_lb.apiserver.zone_id}"
+    name                   = "${aws_lb.nlb.dns_name}"
+    zone_id                = "${aws_lb.nlb.zone_id}"
     evaluate_target_health = true
   }
 }
 
-# Network Load Balancer for apiservers
-resource "aws_lb" "apiserver" {
-  name               = "${var.cluster_name}-apiserver"
+# Network Load Balancer for apiservers and ingress
+resource "aws_lb" "nlb" {
+  name               = "${var.cluster_name}-nlb"
   load_balancer_type = "network"
   internal           = false
 
@@ -24,9 +24,9 @@ resource "aws_lb" "apiserver" {
   enable_cross_zone_load_balancing = true
 }
 
-# Forward TCP traffic to controllers
+# Forward TCP apiserver traffic to controllers
 resource "aws_lb_listener" "apiserver-https" {
-  load_balancer_arn = "${aws_lb.apiserver.arn}"
+  load_balancer_arn = "${aws_lb.nlb.arn}"
   protocol          = "TCP"
   port              = "6443"
 
@@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
   }
 }
 
+# Forward HTTP ingress traffic to workers
+resource "aws_lb_listener" "ingress-http" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 80
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_http_arn}"
+  }
+}
+
+# Forward HTTPS ingress traffic to workers
+resource "aws_lb_listener" "ingress-https" {
+  load_balancer_arn = "${aws_lb.nlb.arn}"
+  protocol          = "TCP"
+  port              = 443
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${module.workers.target_group_https_arn}"
+  }
+}
+
 # Target group of controllers
 resource "aws_lb_target_group" "controllers" {
   name        = "${var.cluster_name}-controllers"

aws/container-linux/kubernetes/outputs.tf

@@ -1,8 +1,18 @@
 output "ingress_dns_name" {
-  value       = "${module.workers.ingress_dns_name}"
+  value       = "${aws_lb.nlb.dns_name}"
   description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
 }
 
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${module.workers.target_group_http_arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${module.workers.target_group_https_arn}"
+}
+
 # Outputs for worker pools
 
 output "vpc_id" {

aws/container-linux/kubernetes/workers/ingress.tf

@@ -1,39 +1,4 @@
-# Network Load Balancer for Ingress
-resource "aws_lb" "ingress" {
-  name               = "${var.name}-ingress"
-  load_balancer_type = "network"
-  internal           = false
-
-  subnets = ["${var.subnet_ids}"]
-
-  enable_cross_zone_load_balancing = true
-}
-
-# Forward HTTP traffic to workers
-resource "aws_lb_listener" "ingress-http" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 80
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-http.arn}"
-  }
-}
-
-# Forward HTTPS traffic to workers
-resource "aws_lb_listener" "ingress-https" {
-  load_balancer_arn = "${aws_lb.ingress.arn}"
-  protocol          = "TCP"
-  port              = 443
-
-  default_action {
-    type             = "forward"
-    target_group_arn = "${aws_lb_target_group.workers-https.arn}"
-  }
-}
-
-# Network Load Balancer target groups of instances
+# Target groups of instances for use with load balancers
 
 resource "aws_lb_target_group" "workers-http" {
   name        = "${var.name}-workers-http"

aws/container-linux/kubernetes/workers/outputs.tf

@@ -1,4 +1,9 @@
-output "ingress_dns_name" {
-  value       = "${aws_lb.ingress.dns_name}"
-  description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
+output "target_group_http_arn" {
+  description = "ARN of a target group of workers for HTTP traffic"
+  value = "${aws_lb_target_group.workers-http.arn}"
+}
+
+output "target_group_https_arn" {
+  description = "ARN of a target group of workers for HTTPS traffic"
+  value = "${aws_lb_target_group.workers-https.arn}"
 }