From 316f06df06a8c7660020babc5df96d1dea468f13 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 Jun 2018 22:56:32 -0700 Subject: [PATCH] Combine NLBs to use one NLB per cluster * Simplify clusters to come with a single NLB * Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver) * Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller) * Reduce cost of default clusters by 1 NLB ($18.14/month) * Keep using CNAME records to the `ingress_dns_name` NLB and the nginx-ingress addon for Ingress (up to a few million RPS) * Users with heavy traffic (many million RPS) can create their own separate NLB(s) for Ingress and use the new output worker target groups * Fix issue where additional worker pools come with an extraneous network load balancer --- CHANGES.md | 8 +++- .../kubernetes/{apiserver.tf => nlb.tf} | 38 ++++++++++++--- aws/container-linux/kubernetes/outputs.tf | 12 ++++- .../kubernetes/workers/ingress.tf | 37 +-------------- .../kubernetes/workers/outputs.tf | 11 +++-- .../kubernetes/{apiserver.tf => nlb.tf} | 46 ++++++++++++++----- aws/fedora-atomic/kubernetes/outputs.tf | 12 ++++- .../kubernetes/workers/ingress.tf | 37 +-------------- .../kubernetes/workers/outputs.tf | 11 +++-- 9 files changed, 113 insertions(+), 99 deletions(-) rename aws/container-linux/kubernetes/{apiserver.tf => nlb.tf} (61%) rename aws/fedora-atomic/kubernetes/{apiserver.tf => nlb.tf} (58%) diff --git a/CHANGES.md b/CHANGES.md index c2efe2d9..bc9e2f60 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,7 +10,13 @@ Notable changes between versions. #### AWS * Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) - * Update NLB, security groups, and generated kubeconfig's +* Combine apiserver and ingress NLBs ([#249](https://github.com/poseidon/typhoon/pull/249)) + * Simplify clusters to come with one NLB. Reduce cost by ~$18/month per cluster. + * Users may keep using CNAME records to `ingress_dns_name` and the `nginx-ingress` addon for Ingress (up to a few million RPS) + * Users with heavy traffic (many million RPS) should create a separate NLB(s) for Ingress instead + * Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver) + * Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller) +* Worker pools (advanced) no longer include an extraneous load balancer #### Bare-Metal diff --git a/aws/container-linux/kubernetes/apiserver.tf b/aws/container-linux/kubernetes/nlb.tf similarity index 61% rename from aws/container-linux/kubernetes/apiserver.tf rename to aws/container-linux/kubernetes/nlb.tf index 93cbcee4..77d3bb14 100644 --- a/aws/container-linux/kubernetes/apiserver.tf +++ b/aws/container-linux/kubernetes/nlb.tf @@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" { # AWS recommends their special "alias" records for ELBs alias { - name = "${aws_lb.apiserver.dns_name}" - zone_id = "${aws_lb.apiserver.zone_id}" + name = "${aws_lb.nlb.dns_name}" + zone_id = "${aws_lb.nlb.zone_id}" evaluate_target_health = true } } -# Network Load Balancer for apiservers -resource "aws_lb" "apiserver" { - name = "${var.cluster_name}-apiserver" +# Network Load Balancer for apiservers and ingress +resource "aws_lb" "nlb" { + name = "${var.cluster_name}-nlb" load_balancer_type = "network" internal = false @@ -24,9 +24,9 @@ resource "aws_lb" "apiserver" { enable_cross_zone_load_balancing = true } -# Forward TCP traffic to controllers +# Forward TCP apiserver traffic to controllers resource "aws_lb_listener" "apiserver-https" { - load_balancer_arn = "${aws_lb.apiserver.arn}" + load_balancer_arn = "${aws_lb.nlb.arn}" protocol = "TCP" port = "6443" @@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" { } } +# Forward HTTP ingress traffic to workers +resource "aws_lb_listener" "ingress-http" { + load_balancer_arn = "${aws_lb.nlb.arn}" + protocol = "TCP" + port = 80 + + default_action { + type = "forward" + target_group_arn = "${module.workers.target_group_http_arn}" + } +} + +# Forward HTTPS ingress traffic to workers +resource "aws_lb_listener" "ingress-https" { + load_balancer_arn = "${aws_lb.nlb.arn}" + protocol = "TCP" + port = 443 + + default_action { + type = "forward" + target_group_arn = "${module.workers.target_group_https_arn}" + } +} + # Target group of controllers resource "aws_lb_target_group" "controllers" { name = "${var.cluster_name}-controllers" diff --git a/aws/container-linux/kubernetes/outputs.tf b/aws/container-linux/kubernetes/outputs.tf index 1d5e3535..3178129c 100644 --- a/aws/container-linux/kubernetes/outputs.tf +++ b/aws/container-linux/kubernetes/outputs.tf @@ -1,8 +1,18 @@ output "ingress_dns_name" { - value = "${module.workers.ingress_dns_name}" + value = "${aws_lb.nlb.dns_name}" description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" } +output "target_group_http_arn" { + description = "ARN of a target group of workers for HTTP traffic" + value = "${module.workers.target_group_http_arn}" +} + +output "target_group_https_arn" { + description = "ARN of a target group of workers for HTTPS traffic" + value = "${module.workers.target_group_https_arn}" +} + # Outputs for worker pools output "vpc_id" { diff --git a/aws/container-linux/kubernetes/workers/ingress.tf b/aws/container-linux/kubernetes/workers/ingress.tf index 64810efa..bdb7362f 100644 --- a/aws/container-linux/kubernetes/workers/ingress.tf +++ b/aws/container-linux/kubernetes/workers/ingress.tf @@ -1,39 +1,4 @@ -# Network Load Balancer for Ingress -resource "aws_lb" "ingress" { - name = "${var.name}-ingress" - load_balancer_type = "network" - internal = false - - subnets = ["${var.subnet_ids}"] - - enable_cross_zone_load_balancing = true -} - -# Forward HTTP traffic to workers -resource "aws_lb_listener" "ingress-http" { - load_balancer_arn = "${aws_lb.ingress.arn}" - protocol = "TCP" - port = 80 - - default_action { - type = "forward" - target_group_arn = "${aws_lb_target_group.workers-http.arn}" - } -} - -# Forward HTTPS traffic to workers -resource "aws_lb_listener" "ingress-https" { - load_balancer_arn = "${aws_lb.ingress.arn}" - protocol = "TCP" - port = 443 - - default_action { - type = "forward" - target_group_arn = "${aws_lb_target_group.workers-https.arn}" - } -} - -# Network Load Balancer target groups of instances +# Target groups of instances for use with load balancers resource "aws_lb_target_group" "workers-http" { name = "${var.name}-workers-http" diff --git a/aws/container-linux/kubernetes/workers/outputs.tf b/aws/container-linux/kubernetes/workers/outputs.tf index 1b9f8429..8a092681 100644 --- a/aws/container-linux/kubernetes/workers/outputs.tf +++ b/aws/container-linux/kubernetes/workers/outputs.tf @@ -1,4 +1,9 @@ -output "ingress_dns_name" { - value = "${aws_lb.ingress.dns_name}" - description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" +output "target_group_http_arn" { + description = "ARN of a target group of workers for HTTP traffic" + value = "${aws_lb_target_group.workers-http.arn}" +} + +output "target_group_https_arn" { + description = "ARN of a target group of workers for HTTPS traffic" + value = "${aws_lb_target_group.workers-https.arn}" } diff --git a/aws/fedora-atomic/kubernetes/apiserver.tf b/aws/fedora-atomic/kubernetes/nlb.tf similarity index 58% rename from aws/fedora-atomic/kubernetes/apiserver.tf rename to aws/fedora-atomic/kubernetes/nlb.tf index 8cc5eed6..77d3bb14 100644 --- a/aws/fedora-atomic/kubernetes/apiserver.tf +++ b/aws/fedora-atomic/kubernetes/nlb.tf @@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" { # AWS recommends their special "alias" records for ELBs alias { - name = "${aws_lb.apiserver.dns_name}" - zone_id = "${aws_lb.apiserver.zone_id}" + name = "${aws_lb.nlb.dns_name}" + zone_id = "${aws_lb.nlb.zone_id}" evaluate_target_health = true } } -# Network Load Balancer for apiservers -resource "aws_lb" "apiserver" { - name = "${var.cluster_name}-apiserver" +# Network Load Balancer for apiservers and ingress +resource "aws_lb" "nlb" { + name = "${var.cluster_name}-nlb" load_balancer_type = "network" internal = false @@ -24,11 +24,11 @@ resource "aws_lb" "apiserver" { enable_cross_zone_load_balancing = true } -# Forward TCP traffic to controllers +# Forward TCP apiserver traffic to controllers resource "aws_lb_listener" "apiserver-https" { - load_balancer_arn = "${aws_lb.apiserver.arn}" + load_balancer_arn = "${aws_lb.nlb.arn}" protocol = "TCP" - port = "443" + port = "6443" default_action { type = "forward" @@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" { } } +# Forward HTTP ingress traffic to workers +resource "aws_lb_listener" "ingress-http" { + load_balancer_arn = "${aws_lb.nlb.arn}" + protocol = "TCP" + port = 80 + + default_action { + type = "forward" + target_group_arn = "${module.workers.target_group_http_arn}" + } +} + +# Forward HTTPS ingress traffic to workers +resource "aws_lb_listener" "ingress-https" { + load_balancer_arn = "${aws_lb.nlb.arn}" + protocol = "TCP" + port = 443 + + default_action { + type = "forward" + target_group_arn = "${module.workers.target_group_https_arn}" + } +} + # Target group of controllers resource "aws_lb_target_group" "controllers" { name = "${var.cluster_name}-controllers" @@ -43,12 +67,12 @@ resource "aws_lb_target_group" "controllers" { target_type = "instance" protocol = "TCP" - port = 443 + port = 6443 # TCP health check for apiserver health_check { protocol = "TCP" - port = 443 + port = 6443 # NLBs required to use same healthy and unhealthy thresholds healthy_threshold = 3 @@ -65,5 +89,5 @@ resource "aws_lb_target_group_attachment" "controllers" { target_group_arn = "${aws_lb_target_group.controllers.arn}" target_id = "${element(aws_instance.controllers.*.id, count.index)}" - port = 443 + port = 6443 } diff --git a/aws/fedora-atomic/kubernetes/outputs.tf b/aws/fedora-atomic/kubernetes/outputs.tf index 1d5e3535..3178129c 100644 --- a/aws/fedora-atomic/kubernetes/outputs.tf +++ b/aws/fedora-atomic/kubernetes/outputs.tf @@ -1,8 +1,18 @@ output "ingress_dns_name" { - value = "${module.workers.ingress_dns_name}" + value = "${aws_lb.nlb.dns_name}" description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" } +output "target_group_http_arn" { + description = "ARN of a target group of workers for HTTP traffic" + value = "${module.workers.target_group_http_arn}" +} + +output "target_group_https_arn" { + description = "ARN of a target group of workers for HTTPS traffic" + value = "${module.workers.target_group_https_arn}" +} + # Outputs for worker pools output "vpc_id" { diff --git a/aws/fedora-atomic/kubernetes/workers/ingress.tf b/aws/fedora-atomic/kubernetes/workers/ingress.tf index 64810efa..bdb7362f 100644 --- a/aws/fedora-atomic/kubernetes/workers/ingress.tf +++ b/aws/fedora-atomic/kubernetes/workers/ingress.tf @@ -1,39 +1,4 @@ -# Network Load Balancer for Ingress -resource "aws_lb" "ingress" { - name = "${var.name}-ingress" - load_balancer_type = "network" - internal = false - - subnets = ["${var.subnet_ids}"] - - enable_cross_zone_load_balancing = true -} - -# Forward HTTP traffic to workers -resource "aws_lb_listener" "ingress-http" { - load_balancer_arn = "${aws_lb.ingress.arn}" - protocol = "TCP" - port = 80 - - default_action { - type = "forward" - target_group_arn = "${aws_lb_target_group.workers-http.arn}" - } -} - -# Forward HTTPS traffic to workers -resource "aws_lb_listener" "ingress-https" { - load_balancer_arn = "${aws_lb.ingress.arn}" - protocol = "TCP" - port = 443 - - default_action { - type = "forward" - target_group_arn = "${aws_lb_target_group.workers-https.arn}" - } -} - -# Network Load Balancer target groups of instances +# Target groups of instances for use with load balancers resource "aws_lb_target_group" "workers-http" { name = "${var.name}-workers-http" diff --git a/aws/fedora-atomic/kubernetes/workers/outputs.tf b/aws/fedora-atomic/kubernetes/workers/outputs.tf index 1b9f8429..8a092681 100644 --- a/aws/fedora-atomic/kubernetes/workers/outputs.tf +++ b/aws/fedora-atomic/kubernetes/workers/outputs.tf @@ -1,4 +1,9 @@ -output "ingress_dns_name" { - value = "${aws_lb.ingress.dns_name}" - description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" +output "target_group_http_arn" { + description = "ARN of a target group of workers for HTTP traffic" + value = "${aws_lb_target_group.workers-http.arn}" +} + +output "target_group_https_arn" { + description = "ARN of a target group of workers for HTTPS traffic" + value = "${aws_lb_target_group.workers-https.arn}" }