Combine NLBs to use one NLB per cluster
* Simplify clusters to come with a single NLB * Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver) * Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller) * Reduce cost of default clusters by 1 NLB ($18.14/month) * Keep using CNAME records to the `ingress_dns_name` NLB and the nginx-ingress addon for Ingress (up to a few million RPS) * Users with heavy traffic (many million RPS) can create their own separate NLB(s) for Ingress and use the new output worker target groups * Fix issue where additional worker pools come with an extraneous network load balancer
This commit is contained in:
parent
f4d3059b00
commit
316f06df06
|
@ -10,7 +10,13 @@ Notable changes between versions.
|
||||||
#### AWS
|
#### AWS
|
||||||
|
|
||||||
* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
|
* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248))
|
||||||
* Update NLB, security groups, and generated kubeconfig's
|
* Combine apiserver and ingress NLBs ([#249](https://github.com/poseidon/typhoon/pull/249))
|
||||||
|
* Simplify clusters to come with one NLB. Reduce cost by ~$18/month per cluster.
|
||||||
|
* Users may keep using CNAME records to `ingress_dns_name` and the `nginx-ingress` addon for Ingress (up to a few million RPS)
|
||||||
|
* Users with heavy traffic (many million RPS) should create a separate NLB(s) for Ingress instead
|
||||||
|
* Listen for apiserver traffic on port 6443 and forward to controllers (with healthy apiserver)
|
||||||
|
* Listen for ingress traffic on ports 80/443 and forward to workers (with healthy ingress controller)
|
||||||
|
* Worker pools (advanced) no longer include an extraneous load balancer
|
||||||
|
|
||||||
#### Bare-Metal
|
#### Bare-Metal
|
||||||
|
|
||||||
|
|
|
@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
|
||||||
|
|
||||||
# AWS recommends their special "alias" records for ELBs
|
# AWS recommends their special "alias" records for ELBs
|
||||||
alias {
|
alias {
|
||||||
name = "${aws_lb.apiserver.dns_name}"
|
name = "${aws_lb.nlb.dns_name}"
|
||||||
zone_id = "${aws_lb.apiserver.zone_id}"
|
zone_id = "${aws_lb.nlb.zone_id}"
|
||||||
evaluate_target_health = true
|
evaluate_target_health = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# Network Load Balancer for apiservers
|
# Network Load Balancer for apiservers and ingress
|
||||||
resource "aws_lb" "apiserver" {
|
resource "aws_lb" "nlb" {
|
||||||
name = "${var.cluster_name}-apiserver"
|
name = "${var.cluster_name}-nlb"
|
||||||
load_balancer_type = "network"
|
load_balancer_type = "network"
|
||||||
internal = false
|
internal = false
|
||||||
|
|
||||||
|
@ -24,9 +24,9 @@ resource "aws_lb" "apiserver" {
|
||||||
enable_cross_zone_load_balancing = true
|
enable_cross_zone_load_balancing = true
|
||||||
}
|
}
|
||||||
|
|
||||||
# Forward TCP traffic to controllers
|
# Forward TCP apiserver traffic to controllers
|
||||||
resource "aws_lb_listener" "apiserver-https" {
|
resource "aws_lb_listener" "apiserver-https" {
|
||||||
load_balancer_arn = "${aws_lb.apiserver.arn}"
|
load_balancer_arn = "${aws_lb.nlb.arn}"
|
||||||
protocol = "TCP"
|
protocol = "TCP"
|
||||||
port = "6443"
|
port = "6443"
|
||||||
|
|
||||||
|
@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Forward HTTP ingress traffic to workers
|
||||||
|
resource "aws_lb_listener" "ingress-http" {
|
||||||
|
load_balancer_arn = "${aws_lb.nlb.arn}"
|
||||||
|
protocol = "TCP"
|
||||||
|
port = 80
|
||||||
|
|
||||||
|
default_action {
|
||||||
|
type = "forward"
|
||||||
|
target_group_arn = "${module.workers.target_group_http_arn}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Forward HTTPS ingress traffic to workers
|
||||||
|
resource "aws_lb_listener" "ingress-https" {
|
||||||
|
load_balancer_arn = "${aws_lb.nlb.arn}"
|
||||||
|
protocol = "TCP"
|
||||||
|
port = 443
|
||||||
|
|
||||||
|
default_action {
|
||||||
|
type = "forward"
|
||||||
|
target_group_arn = "${module.workers.target_group_https_arn}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# Target group of controllers
|
# Target group of controllers
|
||||||
resource "aws_lb_target_group" "controllers" {
|
resource "aws_lb_target_group" "controllers" {
|
||||||
name = "${var.cluster_name}-controllers"
|
name = "${var.cluster_name}-controllers"
|
|
@ -1,8 +1,18 @@
|
||||||
output "ingress_dns_name" {
|
output "ingress_dns_name" {
|
||||||
value = "${module.workers.ingress_dns_name}"
|
value = "${aws_lb.nlb.dns_name}"
|
||||||
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
|
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
output "target_group_http_arn" {
|
||||||
|
description = "ARN of a target group of workers for HTTP traffic"
|
||||||
|
value = "${module.workers.target_group_http_arn}"
|
||||||
|
}
|
||||||
|
|
||||||
|
output "target_group_https_arn" {
|
||||||
|
description = "ARN of a target group of workers for HTTPS traffic"
|
||||||
|
value = "${module.workers.target_group_https_arn}"
|
||||||
|
}
|
||||||
|
|
||||||
# Outputs for worker pools
|
# Outputs for worker pools
|
||||||
|
|
||||||
output "vpc_id" {
|
output "vpc_id" {
|
||||||
|
|
|
@ -1,39 +1,4 @@
|
||||||
# Network Load Balancer for Ingress
|
# Target groups of instances for use with load balancers
|
||||||
resource "aws_lb" "ingress" {
|
|
||||||
name = "${var.name}-ingress"
|
|
||||||
load_balancer_type = "network"
|
|
||||||
internal = false
|
|
||||||
|
|
||||||
subnets = ["${var.subnet_ids}"]
|
|
||||||
|
|
||||||
enable_cross_zone_load_balancing = true
|
|
||||||
}
|
|
||||||
|
|
||||||
# Forward HTTP traffic to workers
|
|
||||||
resource "aws_lb_listener" "ingress-http" {
|
|
||||||
load_balancer_arn = "${aws_lb.ingress.arn}"
|
|
||||||
protocol = "TCP"
|
|
||||||
port = 80
|
|
||||||
|
|
||||||
default_action {
|
|
||||||
type = "forward"
|
|
||||||
target_group_arn = "${aws_lb_target_group.workers-http.arn}"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Forward HTTPS traffic to workers
|
|
||||||
resource "aws_lb_listener" "ingress-https" {
|
|
||||||
load_balancer_arn = "${aws_lb.ingress.arn}"
|
|
||||||
protocol = "TCP"
|
|
||||||
port = 443
|
|
||||||
|
|
||||||
default_action {
|
|
||||||
type = "forward"
|
|
||||||
target_group_arn = "${aws_lb_target_group.workers-https.arn}"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Network Load Balancer target groups of instances
|
|
||||||
|
|
||||||
resource "aws_lb_target_group" "workers-http" {
|
resource "aws_lb_target_group" "workers-http" {
|
||||||
name = "${var.name}-workers-http"
|
name = "${var.name}-workers-http"
|
||||||
|
|
|
@ -1,4 +1,9 @@
|
||||||
output "ingress_dns_name" {
|
output "target_group_http_arn" {
|
||||||
value = "${aws_lb.ingress.dns_name}"
|
description = "ARN of a target group of workers for HTTP traffic"
|
||||||
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
|
value = "${aws_lb_target_group.workers-http.arn}"
|
||||||
|
}
|
||||||
|
|
||||||
|
output "target_group_https_arn" {
|
||||||
|
description = "ARN of a target group of workers for HTTPS traffic"
|
||||||
|
value = "${aws_lb_target_group.workers-https.arn}"
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,15 +7,15 @@ resource "aws_route53_record" "apiserver" {
|
||||||
|
|
||||||
# AWS recommends their special "alias" records for ELBs
|
# AWS recommends their special "alias" records for ELBs
|
||||||
alias {
|
alias {
|
||||||
name = "${aws_lb.apiserver.dns_name}"
|
name = "${aws_lb.nlb.dns_name}"
|
||||||
zone_id = "${aws_lb.apiserver.zone_id}"
|
zone_id = "${aws_lb.nlb.zone_id}"
|
||||||
evaluate_target_health = true
|
evaluate_target_health = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# Network Load Balancer for apiservers
|
# Network Load Balancer for apiservers and ingress
|
||||||
resource "aws_lb" "apiserver" {
|
resource "aws_lb" "nlb" {
|
||||||
name = "${var.cluster_name}-apiserver"
|
name = "${var.cluster_name}-nlb"
|
||||||
load_balancer_type = "network"
|
load_balancer_type = "network"
|
||||||
internal = false
|
internal = false
|
||||||
|
|
||||||
|
@ -24,11 +24,11 @@ resource "aws_lb" "apiserver" {
|
||||||
enable_cross_zone_load_balancing = true
|
enable_cross_zone_load_balancing = true
|
||||||
}
|
}
|
||||||
|
|
||||||
# Forward TCP traffic to controllers
|
# Forward TCP apiserver traffic to controllers
|
||||||
resource "aws_lb_listener" "apiserver-https" {
|
resource "aws_lb_listener" "apiserver-https" {
|
||||||
load_balancer_arn = "${aws_lb.apiserver.arn}"
|
load_balancer_arn = "${aws_lb.nlb.arn}"
|
||||||
protocol = "TCP"
|
protocol = "TCP"
|
||||||
port = "443"
|
port = "6443"
|
||||||
|
|
||||||
default_action {
|
default_action {
|
||||||
type = "forward"
|
type = "forward"
|
||||||
|
@ -36,6 +36,30 @@ resource "aws_lb_listener" "apiserver-https" {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Forward HTTP ingress traffic to workers
|
||||||
|
resource "aws_lb_listener" "ingress-http" {
|
||||||
|
load_balancer_arn = "${aws_lb.nlb.arn}"
|
||||||
|
protocol = "TCP"
|
||||||
|
port = 80
|
||||||
|
|
||||||
|
default_action {
|
||||||
|
type = "forward"
|
||||||
|
target_group_arn = "${module.workers.target_group_http_arn}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Forward HTTPS ingress traffic to workers
|
||||||
|
resource "aws_lb_listener" "ingress-https" {
|
||||||
|
load_balancer_arn = "${aws_lb.nlb.arn}"
|
||||||
|
protocol = "TCP"
|
||||||
|
port = 443
|
||||||
|
|
||||||
|
default_action {
|
||||||
|
type = "forward"
|
||||||
|
target_group_arn = "${module.workers.target_group_https_arn}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# Target group of controllers
|
# Target group of controllers
|
||||||
resource "aws_lb_target_group" "controllers" {
|
resource "aws_lb_target_group" "controllers" {
|
||||||
name = "${var.cluster_name}-controllers"
|
name = "${var.cluster_name}-controllers"
|
||||||
|
@ -43,12 +67,12 @@ resource "aws_lb_target_group" "controllers" {
|
||||||
target_type = "instance"
|
target_type = "instance"
|
||||||
|
|
||||||
protocol = "TCP"
|
protocol = "TCP"
|
||||||
port = 443
|
port = 6443
|
||||||
|
|
||||||
# TCP health check for apiserver
|
# TCP health check for apiserver
|
||||||
health_check {
|
health_check {
|
||||||
protocol = "TCP"
|
protocol = "TCP"
|
||||||
port = 443
|
port = 6443
|
||||||
|
|
||||||
# NLBs required to use same healthy and unhealthy thresholds
|
# NLBs required to use same healthy and unhealthy thresholds
|
||||||
healthy_threshold = 3
|
healthy_threshold = 3
|
||||||
|
@ -65,5 +89,5 @@ resource "aws_lb_target_group_attachment" "controllers" {
|
||||||
|
|
||||||
target_group_arn = "${aws_lb_target_group.controllers.arn}"
|
target_group_arn = "${aws_lb_target_group.controllers.arn}"
|
||||||
target_id = "${element(aws_instance.controllers.*.id, count.index)}"
|
target_id = "${element(aws_instance.controllers.*.id, count.index)}"
|
||||||
port = 443
|
port = 6443
|
||||||
}
|
}
|
|
@ -1,8 +1,18 @@
|
||||||
output "ingress_dns_name" {
|
output "ingress_dns_name" {
|
||||||
value = "${module.workers.ingress_dns_name}"
|
value = "${aws_lb.nlb.dns_name}"
|
||||||
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
|
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
output "target_group_http_arn" {
|
||||||
|
description = "ARN of a target group of workers for HTTP traffic"
|
||||||
|
value = "${module.workers.target_group_http_arn}"
|
||||||
|
}
|
||||||
|
|
||||||
|
output "target_group_https_arn" {
|
||||||
|
description = "ARN of a target group of workers for HTTPS traffic"
|
||||||
|
value = "${module.workers.target_group_https_arn}"
|
||||||
|
}
|
||||||
|
|
||||||
# Outputs for worker pools
|
# Outputs for worker pools
|
||||||
|
|
||||||
output "vpc_id" {
|
output "vpc_id" {
|
||||||
|
|
|
@ -1,39 +1,4 @@
|
||||||
# Network Load Balancer for Ingress
|
# Target groups of instances for use with load balancers
|
||||||
resource "aws_lb" "ingress" {
|
|
||||||
name = "${var.name}-ingress"
|
|
||||||
load_balancer_type = "network"
|
|
||||||
internal = false
|
|
||||||
|
|
||||||
subnets = ["${var.subnet_ids}"]
|
|
||||||
|
|
||||||
enable_cross_zone_load_balancing = true
|
|
||||||
}
|
|
||||||
|
|
||||||
# Forward HTTP traffic to workers
|
|
||||||
resource "aws_lb_listener" "ingress-http" {
|
|
||||||
load_balancer_arn = "${aws_lb.ingress.arn}"
|
|
||||||
protocol = "TCP"
|
|
||||||
port = 80
|
|
||||||
|
|
||||||
default_action {
|
|
||||||
type = "forward"
|
|
||||||
target_group_arn = "${aws_lb_target_group.workers-http.arn}"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Forward HTTPS traffic to workers
|
|
||||||
resource "aws_lb_listener" "ingress-https" {
|
|
||||||
load_balancer_arn = "${aws_lb.ingress.arn}"
|
|
||||||
protocol = "TCP"
|
|
||||||
port = 443
|
|
||||||
|
|
||||||
default_action {
|
|
||||||
type = "forward"
|
|
||||||
target_group_arn = "${aws_lb_target_group.workers-https.arn}"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Network Load Balancer target groups of instances
|
|
||||||
|
|
||||||
resource "aws_lb_target_group" "workers-http" {
|
resource "aws_lb_target_group" "workers-http" {
|
||||||
name = "${var.name}-workers-http"
|
name = "${var.name}-workers-http"
|
||||||
|
|
|
@ -1,4 +1,9 @@
|
||||||
output "ingress_dns_name" {
|
output "target_group_http_arn" {
|
||||||
value = "${aws_lb.ingress.dns_name}"
|
description = "ARN of a target group of workers for HTTP traffic"
|
||||||
description = "DNS name of the network load balancer for distributing traffic to Ingress controllers"
|
value = "${aws_lb_target_group.workers-http.arn}"
|
||||||
|
}
|
||||||
|
|
||||||
|
output "target_group_https_arn" {
|
||||||
|
description = "ARN of a target group of workers for HTTPS traffic"
|
||||||
|
value = "${aws_lb_target_group.workers-https.arn}"
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue