From ceb555522266000c861acda361ee0a4a349b5bf9 Mon Sep 17 00:00:00 2001
From: Paul Saunders <pms1969@gmail.com>
Date: Fri, 16 Feb 2018 13:18:27 +0000
Subject: [PATCH] Switch apiserver from ELB to a network load balancer

---
 aws/container-linux/kubernetes/apiserver.tf | 67 +++++++++++++++++++++
 aws/container-linux/kubernetes/elb.tf       | 43 -------------
 aws/container-linux/kubernetes/ingress.tf   |  4 +-
 3 files changed, 69 insertions(+), 45 deletions(-)
 create mode 100644 aws/container-linux/kubernetes/apiserver.tf
 delete mode 100644 aws/container-linux/kubernetes/elb.tf

diff --git a/aws/container-linux/kubernetes/apiserver.tf b/aws/container-linux/kubernetes/apiserver.tf
new file mode 100644
index 00000000..04e9f327
--- /dev/null
+++ b/aws/container-linux/kubernetes/apiserver.tf
@@ -0,0 +1,67 @@
+# kube-apiserver Network Load Balancer DNS Record
+resource "aws_route53_record" "apiserver" {
+  zone_id = "${var.dns_zone_id}"
+
+  name = "${format("%s.%s.", var.cluster_name, var.dns_zone)}"
+  type = "A"
+
+  # AWS recommends their special "alias" records for ELBs
+  alias {
+    name                   = "${aws_lb.apiserver.dns_name}"
+    zone_id                = "${aws_lb.apiserver.zone_id}"
+    evaluate_target_health = true
+  }
+}
+
+# Network Load Balancer for apiservers
+resource "aws_lb" "apiserver" {
+  name               = "${var.cluster_name}-apiserver"
+  load_balancer_type = "network"
+  internal           = false
+
+  subnets = ["${aws_subnet.public.*.id}"]
+}
+
+# Forward HTTP traffic to controllers
+resource "aws_lb_listener" "apiserver-https" {
+  load_balancer_arn = "${aws_lb.apiserver.arn}"
+  protocol          = "TCP"
+  port              = "443"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${aws_lb_target_group.controllers.arn}"
+  }
+}
+
+# Target group of controllers
+resource "aws_lb_target_group" "controllers" {
+  name        = "${var.cluster_name}-controllers"
+  vpc_id      = "${aws_vpc.network.id}"
+  target_type = "instance"
+
+  protocol = "TCP"
+  port     = 443
+
+  # Kubelet HTTP health check
+  health_check {
+    protocol = "TCP"
+    port     = 443
+
+    # NLBs required to use same healthy and unhealthy thresholds
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+
+    # Interval between health checks required to be 10 or 30
+    interval = 10
+  }
+}
+
+# Attach controller instances to apiserver NLB
+resource "aws_lb_target_group_attachment" "controllers" {
+  count = "${var.controller_count}"
+
+  target_group_arn = "${aws_lb_target_group.controllers.arn}"
+  target_id        = "${element(aws_instance.controllers.*.id, count.index)}"
+  port             = 443
+}
diff --git a/aws/container-linux/kubernetes/elb.tf b/aws/container-linux/kubernetes/elb.tf
deleted file mode 100644
index 2a448a69..00000000
--- a/aws/container-linux/kubernetes/elb.tf
+++ /dev/null
@@ -1,43 +0,0 @@
-# kube-apiserver Network Load Balancer DNS Record
-resource "aws_route53_record" "apiserver" {
-  zone_id = "${var.dns_zone_id}"
-
-  name = "${format("%s.%s.", var.cluster_name, var.dns_zone)}"
-  type = "A"
-
-  # AWS recommends their special "alias" records for ELBs
-  alias {
-    name                   = "${aws_elb.apiserver.dns_name}"
-    zone_id                = "${aws_elb.apiserver.zone_id}"
-    evaluate_target_health = true
-  }
-}
-
-# Controller Network Load Balancer
-resource "aws_elb" "apiserver" {
-  name            = "${var.cluster_name}-apiserver"
-  subnets         = ["${aws_subnet.public.*.id}"]
-  security_groups = ["${aws_security_group.controller.id}"]
-
-  listener {
-    lb_port           = 443
-    lb_protocol       = "tcp"
-    instance_port     = 443
-    instance_protocol = "tcp"
-  }
-
-  instances = ["${aws_instance.controllers.*.id}"]
-
-  # Kubelet HTTP health check
-  health_check {
-    target              = "SSL:443"
-    healthy_threshold   = 2
-    unhealthy_threshold = 4
-    timeout             = 5
-    interval            = 6
-  }
-
-  idle_timeout                = 3600
-  connection_draining         = true
-  connection_draining_timeout = 300
-}
diff --git a/aws/container-linux/kubernetes/ingress.tf b/aws/container-linux/kubernetes/ingress.tf
index 832748b0..acadcdce 100644
--- a/aws/container-linux/kubernetes/ingress.tf
+++ b/aws/container-linux/kubernetes/ingress.tf
@@ -7,7 +7,7 @@ resource "aws_lb" "ingress" {
   subnets = ["${aws_subnet.public.*.id}"]
 }
 
-# Forward HTTP traffic to instances
+# Forward HTTP traffic to workers
 resource "aws_lb_listener" "ingress-http" {
   load_balancer_arn = "${aws_lb.ingress.arn}"
   protocol          = "TCP"
@@ -19,7 +19,7 @@ resource "aws_lb_listener" "ingress-http" {
   }
 }
 
-# Forward HTTPS traffic to instances
+# Forward HTTPS traffic to workers
 resource "aws_lb_listener" "ingress-https" {
   load_balancer_arn = "${aws_lb.ingress.arn}"
   protocol          = "TCP"