Combine NLBs to use one NLB per cluster

* Simplify clusters to come with a single NLB
* Listen for apiserver traffic on port 6443 and forward
to controllers (with healthy apiserver)
* Listen for ingress traffic on ports 80/443 and forward
to workers (with healthy ingress controller)
* Reduce cost of default clusters by 1 NLB ($18.14/month)
* Keep using CNAME records to the `ingress_dns_name` NLB and
the nginx-ingress addon for Ingress (up to a few million RPS)
* Users with heavy traffic (many million RPS) can create their
own separate NLB(s) for Ingress and use the new output worker
target groups
* Fix issue where additional worker pools come with an
extraneous network load balancer
This commit is contained in:
Dalton Hubble 2018-06-18 22:56:32 -07:00
parent f4d3059b00
commit 316f06df06
9 changed files with 113 additions and 99 deletions

View File

@ -10,7 +10,13 @@ Notable changes between versions.
#### AWS #### AWS
* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) * Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
* Update NLB, security groups, and generated kubeconfig's * Combine apiserver and ingress NLBs ([#249](https://github.com/poseidon/typhoon/pull/249))
* Simplify clusters to come with one NLB. Reduce cost by ~$18/month per cluster.
* Users may keep using CNAME records to `ingress_dns_name` and the `nginx-ingress` addon for Ingress (up to a few million RPS)
* Users with heavy traffic (many million RPS) should create a separate NLB(s) for Ingress instead
* Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver)
* Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller)
* Worker pools (advanced) no longer include an extraneous load balancer
#### Bare-Metal #### Bare-Metal

View File

@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
# AWS recommends their special "alias" records for ELBs # AWS recommends their special "alias" records for ELBs
alias { alias {
name = "${aws_lb.apiserver.dns_name}" name = "${aws_lb.nlb.dns_name}"
zone_id = "${aws_lb.apiserver.zone_id}" zone_id = "${aws_lb.nlb.zone_id}"
evaluate_target_health = true evaluate_target_health = true
} }
} }
# Network Load Balancer for apiservers # Network Load Balancer for apiservers and ingress
resource "aws_lb" "apiserver" { resource "aws_lb" "nlb" {
name = "${var.cluster_name}-apiserver" name = "${var.cluster_name}-nlb"
load_balancer_type = "network" load_balancer_type = "network"
internal = false internal = false
@ -24,9 +24,9 @@ resource "aws_lb" "apiserver" {
enable_cross_zone_load_balancing = true enable_cross_zone_load_balancing = true
} }
# Forward TCP traffic to controllers # Forward TCP apiserver traffic to controllers
resource "aws_lb_listener" "apiserver-https" { resource "aws_lb_listener" "apiserver-https" {
load_balancer_arn = "${aws_lb.apiserver.arn}" load_balancer_arn = "${aws_lb.nlb.arn}"
protocol = "TCP" protocol = "TCP"
port = "6443" port = "6443"
@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
} }
} }
# Forward HTTP ingress traffic to workers
resource "aws_lb_listener" "ingress-http" {
load_balancer_arn = "${aws_lb.nlb.arn}"
protocol = "TCP"
port = 80
default_action {
type = "forward"
target_group_arn = "${module.workers.target_group_http_arn}"
}
}
# Forward HTTPS ingress traffic to workers
resource "aws_lb_listener" "ingress-https" {
load_balancer_arn = "${aws_lb.nlb.arn}"
protocol = "TCP"
port = 443
default_action {
type = "forward"
target_group_arn = "${module.workers.target_group_https_arn}"
}
}
# Target group of controllers # Target group of controllers
resource "aws_lb_target_group" "controllers" { resource "aws_lb_target_group" "controllers" {
name = "${var.cluster_name}-controllers" name = "${var.cluster_name}-controllers"

View File

@ -1,8 +1,18 @@
output "ingress_dns_name" { output "ingress_dns_name" {
value = "${module.workers.ingress_dns_name}" value = "${aws_lb.nlb.dns_name}"
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
} }
output "target_group_http_arn" {
description = "ARN of a target group of workers for HTTP traffic"
value = "${module.workers.target_group_http_arn}"
}
output "target_group_https_arn" {
description = "ARN of a target group of workers for HTTPS traffic"
value = "${module.workers.target_group_https_arn}"
}
# Outputs for worker pools # Outputs for worker pools
output "vpc_id" { output "vpc_id" {

View File

@ -1,39 +1,4 @@
# Network Load Balancer for Ingress # Target groups of instances for use with load balancers
resource "aws_lb" "ingress" {
name = "${var.name}-ingress"
load_balancer_type = "network"
internal = false
subnets = ["${var.subnet_ids}"]
enable_cross_zone_load_balancing = true
}
# Forward HTTP traffic to workers
resource "aws_lb_listener" "ingress-http" {
load_balancer_arn = "${aws_lb.ingress.arn}"
protocol = "TCP"
port = 80
default_action {
type = "forward"
target_group_arn = "${aws_lb_target_group.workers-http.arn}"
}
}
# Forward HTTPS traffic to workers
resource "aws_lb_listener" "ingress-https" {
load_balancer_arn = "${aws_lb.ingress.arn}"
protocol = "TCP"
port = 443
default_action {
type = "forward"
target_group_arn = "${aws_lb_target_group.workers-https.arn}"
}
}
# Network Load Balancer target groups of instances
resource "aws_lb_target_group" "workers-http" { resource "aws_lb_target_group" "workers-http" {
name = "${var.name}-workers-http" name = "${var.name}-workers-http"

View File

@ -1,4 +1,9 @@
output "ingress_dns_name" { output "target_group_http_arn" {
value = "${aws_lb.ingress.dns_name}" description = "ARN of a target group of workers for HTTP traffic"
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" value = "${aws_lb_target_group.workers-http.arn}"
}
output "target_group_https_arn" {
description = "ARN of a target group of workers for HTTPS traffic"
value = "${aws_lb_target_group.workers-https.arn}"
} }

View File

@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
# AWS recommends their special "alias" records for ELBs # AWS recommends their special "alias" records for ELBs
alias { alias {
name = "${aws_lb.apiserver.dns_name}" name = "${aws_lb.nlb.dns_name}"
zone_id = "${aws_lb.apiserver.zone_id}" zone_id = "${aws_lb.nlb.zone_id}"
evaluate_target_health = true evaluate_target_health = true
} }
} }
# Network Load Balancer for apiservers # Network Load Balancer for apiservers and ingress
resource "aws_lb" "apiserver" { resource "aws_lb" "nlb" {
name = "${var.cluster_name}-apiserver" name = "${var.cluster_name}-nlb"
load_balancer_type = "network" load_balancer_type = "network"
internal = false internal = false
@ -24,11 +24,11 @@ resource "aws_lb" "apiserver" {
enable_cross_zone_load_balancing = true enable_cross_zone_load_balancing = true
} }
# Forward TCP traffic to controllers # Forward TCP apiserver traffic to controllers
resource "aws_lb_listener" "apiserver-https" { resource "aws_lb_listener" "apiserver-https" {
load_balancer_arn = "${aws_lb.apiserver.arn}" load_balancer_arn = "${aws_lb.nlb.arn}"
protocol = "TCP" protocol = "TCP"
port = "443" port = "6443"
default_action { default_action {
type = "forward" type = "forward"
@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
} }
} }
# Forward HTTP ingress traffic to workers
resource "aws_lb_listener" "ingress-http" {
load_balancer_arn = "${aws_lb.nlb.arn}"
protocol = "TCP"
port = 80
default_action {
type = "forward"
target_group_arn = "${module.workers.target_group_http_arn}"
}
}
# Forward HTTPS ingress traffic to workers
resource "aws_lb_listener" "ingress-https" {
load_balancer_arn = "${aws_lb.nlb.arn}"
protocol = "TCP"
port = 443
default_action {
type = "forward"
target_group_arn = "${module.workers.target_group_https_arn}"
}
}
# Target group of controllers # Target group of controllers
resource "aws_lb_target_group" "controllers" { resource "aws_lb_target_group" "controllers" {
name = "${var.cluster_name}-controllers" name = "${var.cluster_name}-controllers"
@ -43,12 +67,12 @@ resource "aws_lb_target_group" "controllers" {
target_type = "instance" target_type = "instance"
protocol = "TCP" protocol = "TCP"
port = 443 port = 6443
# TCP health check for apiserver # TCP health check for apiserver
health_check { health_check {
protocol = "TCP" protocol = "TCP"
port = 443 port = 6443
# NLBs required to use same healthy and unhealthy thresholds # NLBs required to use same healthy and unhealthy thresholds
healthy_threshold = 3 healthy_threshold = 3
@ -65,5 +89,5 @@ resource "aws_lb_target_group_attachment" "controllers" {
target_group_arn = "${aws_lb_target_group.controllers.arn}" target_group_arn = "${aws_lb_target_group.controllers.arn}"
target_id = "${element(aws_instance.controllers.*.id, count.index)}" target_id = "${element(aws_instance.controllers.*.id, count.index)}"
port = 443 port = 6443
} }

View File

@ -1,8 +1,18 @@
output "ingress_dns_name" { output "ingress_dns_name" {
value = "${module.workers.ingress_dns_name}" value = "${aws_lb.nlb.dns_name}"
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
} }
output "target_group_http_arn" {
description = "ARN of a target group of workers for HTTP traffic"
value = "${module.workers.target_group_http_arn}"
}
output "target_group_https_arn" {
description = "ARN of a target group of workers for HTTPS traffic"
value = "${module.workers.target_group_https_arn}"
}
# Outputs for worker pools # Outputs for worker pools
output "vpc_id" { output "vpc_id" {

View File

@ -1,39 +1,4 @@
# Network Load Balancer for Ingress # Target groups of instances for use with load balancers
resource "aws_lb" "ingress" {
name = "${var.name}-ingress"
load_balancer_type = "network"
internal = false
subnets = ["${var.subnet_ids}"]
enable_cross_zone_load_balancing = true
}
# Forward HTTP traffic to workers
resource "aws_lb_listener" "ingress-http" {
load_balancer_arn = "${aws_lb.ingress.arn}"
protocol = "TCP"
port = 80
default_action {
type = "forward"
target_group_arn = "${aws_lb_target_group.workers-http.arn}"
}
}
# Forward HTTPS traffic to workers
resource "aws_lb_listener" "ingress-https" {
load_balancer_arn = "${aws_lb.ingress.arn}"
protocol = "TCP"
port = 443
default_action {
type = "forward"
target_group_arn = "${aws_lb_target_group.workers-https.arn}"
}
}
# Network Load Balancer target groups of instances
resource "aws_lb_target_group" "workers-http" { resource "aws_lb_target_group" "workers-http" {
name = "${var.name}-workers-http" name = "${var.name}-workers-http"

View File

@ -1,4 +1,9 @@
output "ingress_dns_name" { output "target_group_http_arn" {
value = "${aws_lb.ingress.dns_name}" description = "ARN of a target group of workers for HTTP traffic"
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" value = "${aws_lb_target_group.workers-http.arn}"
}
output "target_group_https_arn" {
description = "ARN of a target group of workers for HTTPS traffic"
value = "${aws_lb_target_group.workers-https.arn}"
} }