From 73126eb7f84108a2327efe9fc0474706c56915eb Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 26 Feb 2018 22:16:34 -0800 Subject: [PATCH] Add support for worker pools on AWS * Allow groups of workers to be defined and joined to a cluster (i.e. worker pools) * Move worker resources into a Terraform submodule * Output variables needed for passing to worker pools * Add usage docs for AWS worker pools (advanced) --- CHANGES.md | 6 +- README.md | 1 + aws/container-linux/kubernetes/README.md | 1 + aws/container-linux/kubernetes/controllers.tf | 182 --------- aws/container-linux/kubernetes/outputs.tf | 23 +- aws/container-linux/kubernetes/security.tf | 385 ++++++++++++++++++ aws/container-linux/kubernetes/workers.tf | 289 +------------ aws/container-linux/kubernetes/workers/ami.tf | 19 + .../{ => workers}/cl/worker.yaml.tmpl | 0 .../kubernetes/{ => workers}/ingress.tf | 6 +- .../kubernetes/workers/outputs.tf | 4 + .../kubernetes/workers/variables.tf | 73 ++++ .../kubernetes/workers/workers.tf | 74 ++++ docs/advanced/worker-pools.md | 106 ++++- docs/index.md | 1 + 15 files changed, 692 insertions(+), 478 deletions(-) create mode 100644 aws/container-linux/kubernetes/security.tf create mode 100644 aws/container-linux/kubernetes/workers/ami.tf rename aws/container-linux/kubernetes/{ => workers}/cl/worker.yaml.tmpl (100%) rename aws/container-linux/kubernetes/{ => workers}/ingress.tf (93%) create mode 100644 aws/container-linux/kubernetes/workers/outputs.tf create mode 100644 aws/container-linux/kubernetes/workers/variables.tf create mode 100644 aws/container-linux/kubernetes/workers/workers.tf diff --git a/CHANGES.md b/CHANGES.md index 7cf4e393..fac0c552 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,10 +4,11 @@ Notable changes between versions. ## Latest +* Add [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) for AWS and Google Cloud (advanced) +* [Recommend](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action recommended) * Upgrade etcd from v3.2.15 to v3.3.1 * Update Calico from v3.0.2 to v3.0.3 * Use kubernetes-incubator/bootkube v0.10.0 -* [Recommend](https://typhoon.psdn.io/topics/maintenance/#terraform-provider-ct-v021) updating `terraform-provider-ct` plugin from v0.2.0 to [v0.2.1](https://github.com/coreos/terraform-provider-ct/releases/tag/v0.2.1) (action recommended) #### AWS @@ -15,6 +16,7 @@ Notable changes between versions. * Switch Ingress elastic load balancer to a network load balancer ([#141](https://github.com/poseidon/typhoon/pull/141)) * AWS [NLBs](https://aws.amazon.com/blogs/aws/new-network-load-balancer-effortless-scaling-to-millions-of-requests-per-second/) can handle millions of RPS with high throughput and low latency. * Require terraform-provider-aws 1.7.0 or higher +* Allow groups of workers to be defined and joined to a cluster (i.e. worker pools) ([#150](https://github.com/poseidon/typhoon/pull/150)) * Add kubelet `--volume-plugin-dir` flag to allow flexvolume plugins ([#142](https://github.com/poseidon/typhoon/pull/142)) #### Digital Ocean @@ -24,7 +26,7 @@ Notable changes between versions. #### Google Cloud -* Add support for "worker pools" - groups of homogeneous workers joined to an existing cluster ([#148](https://github.com/poseidon/typhoon/pull/148)) +* Allow groups of workers to be defined and joined to a cluster (i.e. worker pools) ([#148](https://github.com/poseidon/typhoon/pull/148)) * Add kubelet `--volume-plugin-dir` flag to allow flexvolume plugins ([#142](https://github.com/poseidon/typhoon/pull/142)) * Add `kubeconfig` variable to `controllers` and `workers` submodules ([#147](https://github.com/poseidon/typhoon/pull/147)) * Remove `kubeconfig_*` variables from `controllers` and `workers` submodules ([#147](https://github.com/poseidon/typhoon/pull/147)) diff --git a/README.md b/README.md index 40fa8a3f..93263994 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.9.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) and [preemption](https://typhoon.psdn.io/google-cloud/#preemption) (varies by platform) * Ready for Ingress, Dashboards, Metrics, and other optional [addons](https://typhoon.psdn.io/addons/overview/) ## Modules diff --git a/aws/container-linux/kubernetes/README.md b/aws/container-linux/kubernetes/README.md index a71b0de2..8912379f 100644 --- a/aws/container-linux/kubernetes/README.md +++ b/aws/container-linux/kubernetes/README.md @@ -14,6 +14,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster * Kubernetes v1.9.3 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/) * Ready for Ingress, Dashboards, Metrics, and other optional [addons](https://typhoon.psdn.io/addons/overview/) ## Docs diff --git a/aws/container-linux/kubernetes/controllers.tf b/aws/container-linux/kubernetes/controllers.tf index 316dfb3f..951f6de9 100644 --- a/aws/container-linux/kubernetes/controllers.tf +++ b/aws/container-linux/kubernetes/controllers.tf @@ -79,185 +79,3 @@ data "ct_config" "controller_ign" { content = "${element(data.template_file.controller_config.*.rendered, count.index)}" pretty_print = false } - -# Security Group (instance firewall) - -resource "aws_security_group" "controller" { - name = "${var.cluster_name}-controller" - description = "${var.cluster_name} controller security group" - - vpc_id = "${aws_vpc.network.id}" - - tags = "${map("Name", "${var.cluster_name}-controller")}" -} - -resource "aws_security_group_rule" "controller-icmp" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "icmp" - from_port = 0 - to_port = 0 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "controller-ssh" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "controller-apiserver" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 443 - to_port = 443 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "controller-etcd" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 2379 - to_port = 2380 - self = true -} - -resource "aws_security_group_rule" "controller-flannel" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "udp" - from_port = 8472 - to_port = 8472 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-flannel-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "udp" - from_port = 8472 - to_port = 8472 - self = true -} - -resource "aws_security_group_rule" "controller-node-exporter" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 9100 - to_port = 9100 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-kubelet-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - self = true -} - -resource "aws_security_group_rule" "controller-kubelet-read" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10255 - to_port = 10255 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-kubelet-read-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10255 - to_port = 10255 - self = true -} - -resource "aws_security_group_rule" "controller-bgp" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-bgp-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - self = true -} - -resource "aws_security_group_rule" "controller-ipip" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-ipip-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "controller-ipip-legacy" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.worker.id}" -} - -resource "aws_security_group_rule" "controller-ipip-legacy-self" { - security_group_id = "${aws_security_group.controller.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "controller-egress" { - security_group_id = "${aws_security_group.controller.id}" - - type = "egress" - protocol = "-1" - from_port = 0 - to_port = 0 - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] -} diff --git a/aws/container-linux/kubernetes/outputs.tf b/aws/container-linux/kubernetes/outputs.tf index 1b9f8429..b126dd92 100644 --- a/aws/container-linux/kubernetes/outputs.tf +++ b/aws/container-linux/kubernetes/outputs.tf @@ -1,4 +1,25 @@ output "ingress_dns_name" { - value = "${aws_lb.ingress.dns_name}" + value = "${module.workers.ingress_dns_name}" description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" } + +# Outputs for worker pools + +output "vpc_id" { + value = "${aws_vpc.network.id}" + description = "ID of the VPC for creating worker instances" +} + +output "subnet_ids" { + value = ["${aws_subnet.public.*.id}"] + description = "List of subnet IDs for creating worker instances" +} + +output "worker_security_groups" { + value = ["${aws_security_group.worker.id}"] + description = "List of worker security group IDs" +} + +output "kubeconfig" { + value = "${module.bootkube.kubeconfig}" +} diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf new file mode 100644 index 00000000..8c71da6b --- /dev/null +++ b/aws/container-linux/kubernetes/security.tf @@ -0,0 +1,385 @@ +# Security Groups (instance firewalls) + +# Controller security group + +resource "aws_security_group" "controller" { + name = "${var.cluster_name}-controller" + description = "${var.cluster_name} controller security group" + + vpc_id = "${aws_vpc.network.id}" + + tags = "${map("Name", "${var.cluster_name}-controller")}" +} + +resource "aws_security_group_rule" "controller-icmp" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "icmp" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "controller-ssh" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "controller-apiserver" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "controller-etcd" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 2379 + to_port = 2380 + self = true +} + +resource "aws_security_group_rule" "controller-flannel" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = "${aws_security_group.worker.id}" +} + +resource "aws_security_group_rule" "controller-flannel-self" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + +resource "aws_security_group_rule" "controller-node-exporter" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + source_security_group_id = "${aws_security_group.worker.id}" +} + +resource "aws_security_group_rule" "controller-kubelet-self" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "controller-kubelet-read" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10255 + to_port = 10255 + source_security_group_id = "${aws_security_group.worker.id}" +} + +resource "aws_security_group_rule" "controller-kubelet-read-self" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10255 + to_port = 10255 + self = true +} + +resource "aws_security_group_rule" "controller-bgp" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + source_security_group_id = "${aws_security_group.worker.id}" +} + +resource "aws_security_group_rule" "controller-bgp-self" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + self = true +} + +resource "aws_security_group_rule" "controller-ipip" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + source_security_group_id = "${aws_security_group.worker.id}" +} + +resource "aws_security_group_rule" "controller-ipip-self" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "controller-ipip-legacy" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + source_security_group_id = "${aws_security_group.worker.id}" +} + +resource "aws_security_group_rule" "controller-ipip-legacy-self" { + security_group_id = "${aws_security_group.controller.id}" + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "controller-egress" { + security_group_id = "${aws_security_group.controller.id}" + + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] +} + +# Worker security group + +resource "aws_security_group" "worker" { + name = "${var.cluster_name}-worker" + description = "${var.cluster_name} worker security group" + + vpc_id = "${aws_vpc.network.id}" + + tags = "${map("Name", "${var.cluster_name}-worker")}" +} + +resource "aws_security_group_rule" "worker-icmp" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "icmp" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-ssh" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-http" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 80 + to_port = 80 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-https" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-flannel" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + source_security_group_id = "${aws_security_group.controller.id}" +} + +resource "aws_security_group_rule" "worker-flannel-self" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "udp" + from_port = 8472 + to_port = 8472 + self = true +} + +resource "aws_security_group_rule" "worker-node-exporter" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + self = true +} + +resource "aws_security_group_rule" "ingress-health" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10254 + to_port = 10254 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "worker-kubelet" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + source_security_group_id = "${aws_security_group.controller.id}" +} + +resource "aws_security_group_rule" "worker-kubelet-self" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + self = true +} + +resource "aws_security_group_rule" "worker-kubelet-read" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10255 + to_port = 10255 + source_security_group_id = "${aws_security_group.controller.id}" +} + +resource "aws_security_group_rule" "worker-kubelet-read-self" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 10255 + to_port = 10255 + self = true +} + +resource "aws_security_group_rule" "worker-bgp" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + source_security_group_id = "${aws_security_group.controller.id}" +} + +resource "aws_security_group_rule" "worker-bgp-self" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = "tcp" + from_port = 179 + to_port = 179 + self = true +} + +resource "aws_security_group_rule" "worker-ipip" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + source_security_group_id = "${aws_security_group.controller.id}" +} + +resource "aws_security_group_rule" "worker-ipip-self" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = 4 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "worker-ipip-legacy" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + source_security_group_id = "${aws_security_group.controller.id}" +} + +resource "aws_security_group_rule" "worker-ipip-legacy-self" { + security_group_id = "${aws_security_group.worker.id}" + + type = "ingress" + protocol = 94 + from_port = 0 + to_port = 0 + self = true +} + +resource "aws_security_group_rule" "worker-egress" { + security_group_id = "${aws_security_group.worker.id}" + + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] +} diff --git a/aws/container-linux/kubernetes/workers.tf b/aws/container-linux/kubernetes/workers.tf index e734da2d..a8ec605a 100644 --- a/aws/container-linux/kubernetes/workers.tf +++ b/aws/container-linux/kubernetes/workers.tf @@ -1,276 +1,19 @@ -# Workers AutoScaling Group -resource "aws_autoscaling_group" "workers" { - name = "${var.cluster_name}-worker ${aws_launch_configuration.worker.name}" - - # count - desired_capacity = "${var.worker_count}" - min_size = "${var.worker_count}" - max_size = "${var.worker_count + 2}" - default_cooldown = 30 - health_check_grace_period = 30 - - # network - vpc_zone_identifier = ["${aws_subnet.public.*.id}"] - - # template - launch_configuration = "${aws_launch_configuration.worker.name}" - - # target groups to which instances should be added - target_group_arns = [ - "${aws_lb_target_group.workers-http.id}", - "${aws_lb_target_group.workers-https.id}", - ] - - lifecycle { - # override the default destroy and replace update behavior - create_before_destroy = true - ignore_changes = ["image_id"] - } - - tags = [{ - key = "Name" - value = "${var.cluster_name}-worker" - propagate_at_launch = true - }] -} - -# Worker template -resource "aws_launch_configuration" "worker" { - image_id = "${data.aws_ami.coreos.image_id}" - instance_type = "${var.worker_type}" - - user_data = "${data.ct_config.worker_ign.rendered}" - - # storage - root_block_device { - volume_type = "standard" - volume_size = "${var.disk_size}" - } - - # network +module "workers" { + source = "workers" + cluster_name = "${var.cluster_name}" + + # AWS + vpc_id = "${aws_vpc.network.id}" + subnet_ids = ["${aws_subnet.public.*.id}"] security_groups = ["${aws_security_group.worker.id}"] + count = "${var.worker_count}" + instance_type = "${var.worker_type}" + os_channel = "${var.os_channel}" + disk_size = "${var.disk_size}" - lifecycle { - // Override the default destroy and replace update behavior - create_before_destroy = true - } -} - -# Worker Container Linux Config -data "template_file" "worker_config" { - template = "${file("${path.module}/cl/worker.yaml.tmpl")}" - - vars = { - kubeconfig = "${indent(10, module.bootkube.kubeconfig)}" - ssh_authorized_key = "${var.ssh_authorized_key}" - k8s_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" - cluster_domain_suffix = "${var.cluster_domain_suffix}" - } -} - -data "ct_config" "worker_ign" { - content = "${data.template_file.worker_config.rendered}" - pretty_print = false -} - -# Security Group (instance firewall) - -resource "aws_security_group" "worker" { - name = "${var.cluster_name}-worker" - description = "${var.cluster_name} worker security group" - - vpc_id = "${aws_vpc.network.id}" - - tags = "${map("Name", "${var.cluster_name}-worker")}" -} - -resource "aws_security_group_rule" "worker-icmp" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "icmp" - from_port = 0 - to_port = 0 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-ssh" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-http" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 80 - to_port = 80 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-https" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 443 - to_port = 443 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-flannel" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "udp" - from_port = 8472 - to_port = 8472 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-flannel-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "udp" - from_port = 8472 - to_port = 8472 - self = true -} - -resource "aws_security_group_rule" "worker-node-exporter" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 9100 - to_port = 9100 - self = true -} - -resource "aws_security_group_rule" "ingress-health" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10254 - to_port = 10254 - cidr_blocks = ["0.0.0.0/0"] -} - -resource "aws_security_group_rule" "worker-kubelet" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-kubelet-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10250 - to_port = 10250 - self = true -} - -resource "aws_security_group_rule" "worker-kubelet-read" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10255 - to_port = 10255 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-kubelet-read-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 10255 - to_port = 10255 - self = true -} - -resource "aws_security_group_rule" "worker-bgp" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-bgp-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = "tcp" - from_port = 179 - to_port = 179 - self = true -} - -resource "aws_security_group_rule" "worker-ipip" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-ipip-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 4 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "worker-ipip-legacy" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - source_security_group_id = "${aws_security_group.controller.id}" -} - -resource "aws_security_group_rule" "worker-ipip-legacy-self" { - security_group_id = "${aws_security_group.worker.id}" - - type = "ingress" - protocol = 94 - from_port = 0 - to_port = 0 - self = true -} - -resource "aws_security_group_rule" "worker-egress" { - security_group_id = "${aws_security_group.worker.id}" - - type = "egress" - protocol = "-1" - from_port = 0 - to_port = 0 - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] + # configuration + kubeconfig = "${module.bootkube.kubeconfig}" + ssh_authorized_key = "${var.ssh_authorized_key}" + service_cidr = "${var.service_cidr}" + cluster_domain_suffix = "${var.cluster_domain_suffix}" } diff --git a/aws/container-linux/kubernetes/workers/ami.tf b/aws/container-linux/kubernetes/workers/ami.tf new file mode 100644 index 00000000..03c6c0f6 --- /dev/null +++ b/aws/container-linux/kubernetes/workers/ami.tf @@ -0,0 +1,19 @@ +data "aws_ami" "coreos" { + most_recent = true + owners = ["595879546273"] + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "name" + values = ["CoreOS-${var.os_channel}-*"] + } +} diff --git a/aws/container-linux/kubernetes/cl/worker.yaml.tmpl b/aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl similarity index 100% rename from aws/container-linux/kubernetes/cl/worker.yaml.tmpl rename to aws/container-linux/kubernetes/workers/cl/worker.yaml.tmpl diff --git a/aws/container-linux/kubernetes/ingress.tf b/aws/container-linux/kubernetes/workers/ingress.tf similarity index 93% rename from aws/container-linux/kubernetes/ingress.tf rename to aws/container-linux/kubernetes/workers/ingress.tf index acadcdce..5b01f985 100644 --- a/aws/container-linux/kubernetes/ingress.tf +++ b/aws/container-linux/kubernetes/workers/ingress.tf @@ -4,7 +4,7 @@ resource "aws_lb" "ingress" { load_balancer_type = "network" internal = false - subnets = ["${aws_subnet.public.*.id}"] + subnets = ["${var.subnet_ids}"] } # Forward HTTP traffic to workers @@ -35,7 +35,7 @@ resource "aws_lb_listener" "ingress-https" { resource "aws_lb_target_group" "workers-http" { name = "${var.cluster_name}-workers-http" - vpc_id = "${aws_vpc.network.id}" + vpc_id = "${var.vpc_id}" target_type = "instance" protocol = "TCP" @@ -58,7 +58,7 @@ resource "aws_lb_target_group" "workers-http" { resource "aws_lb_target_group" "workers-https" { name = "${var.cluster_name}-workers-https" - vpc_id = "${aws_vpc.network.id}" + vpc_id = "${var.vpc_id}" target_type = "instance" protocol = "TCP" diff --git a/aws/container-linux/kubernetes/workers/outputs.tf b/aws/container-linux/kubernetes/workers/outputs.tf new file mode 100644 index 00000000..1b9f8429 --- /dev/null +++ b/aws/container-linux/kubernetes/workers/outputs.tf @@ -0,0 +1,4 @@ +output "ingress_dns_name" { + value = "${aws_lb.ingress.dns_name}" + description = "DNS name of the network load balancer for distributing traffic to Ingress controllers" +} diff --git a/aws/container-linux/kubernetes/workers/variables.tf b/aws/container-linux/kubernetes/workers/variables.tf new file mode 100644 index 00000000..64088b57 --- /dev/null +++ b/aws/container-linux/kubernetes/workers/variables.tf @@ -0,0 +1,73 @@ +variable "cluster_name" { + type = "string" + description = "Unique name" +} + +variable "vpc_id" { + type = "string" + description = "ID of the VPC for creating instances" +} + +variable "subnet_ids" { + type = "list" + description = "List of subnet IDs for creating instances" +} + +variable "security_groups" { + type = "list" + description = "List of security group IDs" +} + +# instances + +variable "count" { + type = "string" + default = "1" + description = "Number of instances" +} + +variable "instance_type" { + type = "string" + default = "t2.small" + description = "EC2 instance type" +} + +variable "os_channel" { + type = "string" + default = "stable" + description = "Container Linux AMI channel (stable, beta, alpha)" +} + +variable "disk_size" { + type = "string" + default = "40" + description = "Size of the disk in GB" +} + +# configuration + +variable "kubeconfig" { + type = "string" + description = "Generated Kubelet kubeconfig" +} + +variable "ssh_authorized_key" { + type = "string" + description = "SSH public key for user 'core'" +} + +variable "service_cidr" { + description = <