mirror of
https://github.com/puppetmaster/typhoon.git
synced 2024-12-25 17:39:33 +01:00
e9c8520359
* Accept experimental CNI `networking` mode "cilium"
* Run Cilium v1.8.0-rc4 with overlay vxlan tunnels and a
minimal set of features. We're interested in:
* IPAM: Divide pod_cidr into /24 subnets per node
* CNI networking pod-to-pod, pod-to-external
* BPF masquerade
* NetworkPolicy as defined by Kubernetes (no L7 Policy)
* Continue using kube-proxy with Cilium probe mode
* Firewall changes:
* Require UDP 8472 for vxlan (Linux kernel default) between nodes
* Optional ICMP echo(8) between nodes for host reachability
(health)
* Optional TCP 4240 between nodes for endpoint reachability (health)
Known Issues:
* Containers with `hostPort` don't listen on all host addresses,
these workloads must use `hostNetwork` for now
https://github.com/cilium/cilium/issues/12116
* Erroneous warning on Fedora CoreOS
https://github.com/cilium/cilium/issues/10256
Note: This is experimental. It is not listed in docs and may be
changed or removed without a deprecation notice
Related:
* https://github.com/poseidon/terraform-render-bootstrap/pull/192
* https://github.com/cilium/cilium/issues/12217
557 lines
14 KiB
HCL
557 lines
14 KiB
HCL
# Security Groups (instance firewalls)
|
|
|
|
# Controller security group
|
|
|
|
resource "aws_security_group" "controller" {
|
|
name = "${var.cluster_name}-controller"
|
|
description = "${var.cluster_name} controller security group"
|
|
|
|
vpc_id = aws_vpc.network.id
|
|
|
|
tags = {
|
|
"Name" = "${var.cluster_name}-controller"
|
|
}
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-icmp" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "icmp"
|
|
from_port = 8
|
|
to_port = 0
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-icmp-self" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "icmp"
|
|
from_port = 8
|
|
to_port = 0
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-ssh" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 22
|
|
to_port = 22
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-etcd" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 2379
|
|
to_port = 2380
|
|
self = true
|
|
}
|
|
|
|
# Allow Prometheus to scrape etcd metrics
|
|
resource "aws_security_group_rule" "controller-etcd-metrics" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 2381
|
|
to_port = 2381
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-cilium-health" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 4240
|
|
to_port = 4240
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-cilium-health-self" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 4240
|
|
to_port = 4240
|
|
self = true
|
|
}
|
|
|
|
# IANA VXLAN default
|
|
resource "aws_security_group_rule" "controller-vxlan" {
|
|
count = var.networking == "flannel" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 4789
|
|
to_port = 4789
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-vxlan-self" {
|
|
count = var.networking == "flannel" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 4789
|
|
to_port = 4789
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-apiserver" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 6443
|
|
to_port = 6443
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
}
|
|
|
|
# Linux VXLAN default
|
|
resource "aws_security_group_rule" "controller-linux-vxlan" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 8472
|
|
to_port = 8472
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-linux-vxlan-self" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 8472
|
|
to_port = 8472
|
|
self = true
|
|
}
|
|
|
|
# Allow Prometheus to scrape node-exporter daemonset
|
|
resource "aws_security_group_rule" "controller-node-exporter" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 9100
|
|
to_port = 9100
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
# Allow Prometheus to scrape kube-proxy
|
|
resource "aws_security_group_rule" "kube-proxy-metrics" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10249
|
|
to_port = 10249
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
# Allow apiserver to access kubelets for exec, log, port-forward
|
|
resource "aws_security_group_rule" "controller-kubelet" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10250
|
|
to_port = 10250
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-kubelet-self" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10250
|
|
to_port = 10250
|
|
self = true
|
|
}
|
|
|
|
# Allow Prometheus to scrape kube-scheduler
|
|
resource "aws_security_group_rule" "controller-scheduler-metrics" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10251
|
|
to_port = 10251
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
# Allow Prometheus to scrape kube-controller-manager
|
|
resource "aws_security_group_rule" "controller-manager-metrics" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10252
|
|
to_port = 10252
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-bgp" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 179
|
|
to_port = 179
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-bgp-self" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 179
|
|
to_port = 179
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-ipip" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = 4
|
|
from_port = 0
|
|
to_port = 0
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-ipip-self" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = 4
|
|
from_port = 0
|
|
to_port = 0
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-ipip-legacy" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = 94
|
|
from_port = 0
|
|
to_port = 0
|
|
source_security_group_id = aws_security_group.worker.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-ipip-legacy-self" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "ingress"
|
|
protocol = 94
|
|
from_port = 0
|
|
to_port = 0
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "controller-egress" {
|
|
security_group_id = aws_security_group.controller.id
|
|
|
|
type = "egress"
|
|
protocol = "-1"
|
|
from_port = 0
|
|
to_port = 0
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
ipv6_cidr_blocks = ["::/0"]
|
|
}
|
|
|
|
# Worker security group
|
|
|
|
resource "aws_security_group" "worker" {
|
|
name = "${var.cluster_name}-worker"
|
|
description = "${var.cluster_name} worker security group"
|
|
|
|
vpc_id = aws_vpc.network.id
|
|
|
|
tags = {
|
|
"Name" = "${var.cluster_name}-worker"
|
|
}
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-icmp" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "icmp"
|
|
from_port = 8
|
|
to_port = 0
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-icmp-self" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "icmp"
|
|
from_port = 8
|
|
to_port = 0
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-ssh" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 22
|
|
to_port = 22
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-http" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 80
|
|
to_port = 80
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-https" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 443
|
|
to_port = 443
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-cilium-health" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 4240
|
|
to_port = 4240
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-cilium-health-self" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 4240
|
|
to_port = 4240
|
|
self = true
|
|
}
|
|
|
|
# IANA VXLAN default
|
|
resource "aws_security_group_rule" "worker-vxlan" {
|
|
count = var.networking == "flannel" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 4789
|
|
to_port = 4789
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-vxlan-self" {
|
|
count = var.networking == "flannel" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 4789
|
|
to_port = 4789
|
|
self = true
|
|
}
|
|
|
|
# Linux VXLAN default
|
|
resource "aws_security_group_rule" "worker-linux-vxlan" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 8472
|
|
to_port = 8472
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-linux-vxlan-self" {
|
|
count = var.networking == "cilium" ? 1 : 0
|
|
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "udp"
|
|
from_port = 8472
|
|
to_port = 8472
|
|
self = true
|
|
}
|
|
|
|
# Allow Prometheus to scrape node-exporter daemonset
|
|
resource "aws_security_group_rule" "worker-node-exporter" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 9100
|
|
to_port = 9100
|
|
self = true
|
|
}
|
|
|
|
# Allow Prometheus to scrape kube-proxy
|
|
resource "aws_security_group_rule" "worker-kube-proxy" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10249
|
|
to_port = 10249
|
|
self = true
|
|
}
|
|
|
|
# Allow apiserver to access kubelets for exec, log, port-forward
|
|
resource "aws_security_group_rule" "worker-kubelet" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10250
|
|
to_port = 10250
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
# Allow Prometheus to scrape kubelet metrics
|
|
resource "aws_security_group_rule" "worker-kubelet-self" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10250
|
|
to_port = 10250
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "ingress-health" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 10254
|
|
to_port = 10254
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-bgp" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 179
|
|
to_port = 179
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-bgp-self" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = "tcp"
|
|
from_port = 179
|
|
to_port = 179
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-ipip" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = 4
|
|
from_port = 0
|
|
to_port = 0
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-ipip-self" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = 4
|
|
from_port = 0
|
|
to_port = 0
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-ipip-legacy" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = 94
|
|
from_port = 0
|
|
to_port = 0
|
|
source_security_group_id = aws_security_group.controller.id
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-ipip-legacy-self" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "ingress"
|
|
protocol = 94
|
|
from_port = 0
|
|
to_port = 0
|
|
self = true
|
|
}
|
|
|
|
resource "aws_security_group_rule" "worker-egress" {
|
|
security_group_id = aws_security_group.worker.id
|
|
|
|
type = "egress"
|
|
protocol = "-1"
|
|
from_port = 0
|
|
to_port = 0
|
|
cidr_blocks = ["0.0.0.0/0"]
|
|
ipv6_cidr_blocks = ["::/0"]
|
|
}
|
|
|