Rename container-linux modules to flatcar-linux

* CoreOS Container Linux was deprecated in v1.18.3
* Continue transitioning docs and modules from supporting
both CoreOS and Flatcar "variants" of Container Linux to
now supporting Flatcar Linux and equivalents

Action Required: Update the Flatcar Linux modules `source`
to replace `s/container-linux/flatcar-linux`. See docs for
examples

google-cloud/flatcar-linux/kubernetes/LICENSE

@@ -0,0 +1,23 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Typhoon Authors
+Copyright (c) 2017 Dalton Hubble
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

google-cloud/flatcar-linux/kubernetes/README.md

@@ -0,0 +1,23 @@
+# Typhoon <img align="right" src="https://storage.googleapis.com/poseidon/typhoon-logo.png">
+
+Typhoon is a minimal and free Kubernetes distribution.
+
+* Minimal, stable base Kubernetes distribution
+* Declarative infrastructure and configuration
+* Free (freedom and cost) and privacy-respecting
+* Practical for labs, datacenters, and clouds
+
+Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components.
+
+## Features <a href="https://www.cncf.io/certification/software-conformance/"><img align="right" src="https://storage.googleapis.com/poseidon/certified-kubernetes.png"></a>
+
+* Kubernetes v1.19.3 (upstream)
+* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking
+* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
+* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization
+* Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/)
+
+## Docs
+
+Please see the [official docs](https://typhoon.psdn.io) and the Google Cloud [tutorial](https://typhoon.psdn.io/cl/google-cloud/).
+

google-cloud/flatcar-linux/kubernetes/apiserver.tf

@@ -0,0 +1,93 @@
+# TCP Proxy load balancer DNS record
+resource "google_dns_record_set" "apiserver" {
+  # DNS Zone name where record should be created
+  managed_zone = var.dns_zone_name
+
+  # DNS record
+  name = format("%s.%s.", var.cluster_name, var.dns_zone)
+  type = "A"
+  ttl  = 300
+
+  # IPv4 address of apiserver TCP Proxy load balancer
+  rrdatas = [google_compute_global_address.apiserver-ipv4.address]
+}
+
+# Static IPv4 address for the TCP Proxy Load Balancer
+resource "google_compute_global_address" "apiserver-ipv4" {
+  name       = "${var.cluster_name}-apiserver-ip"
+  ip_version = "IPV4"
+}
+
+# Forward IPv4 TCP traffic to the TCP proxy load balancer
+resource "google_compute_global_forwarding_rule" "apiserver" {
+  name        = "${var.cluster_name}-apiserver"
+  ip_address  = google_compute_global_address.apiserver-ipv4.address
+  ip_protocol = "TCP"
+  port_range  = "443"
+  target      = google_compute_target_tcp_proxy.apiserver.self_link
+}
+
+# Global TCP Proxy Load Balancer for apiservers
+resource "google_compute_target_tcp_proxy" "apiserver" {
+  name            = "${var.cluster_name}-apiserver"
+  description     = "Distribute TCP load across ${var.cluster_name} controllers"
+  backend_service = google_compute_backend_service.apiserver.self_link
+}
+
+# Global backend service backed by unmanaged instance groups
+resource "google_compute_backend_service" "apiserver" {
+  name        = "${var.cluster_name}-apiserver"
+  description = "${var.cluster_name} apiserver service"
+
+  protocol         = "TCP"
+  port_name        = "apiserver"
+  session_affinity = "NONE"
+  timeout_sec      = "300"
+
+  # controller(s) spread across zonal instance groups
+  dynamic "backend" {
+    for_each = google_compute_instance_group.controllers
+    content {
+      group = backend.value.self_link
+    }
+  }
+
+  health_checks = [google_compute_health_check.apiserver.self_link]
+}
+
+# Instance group of heterogeneous (unmanged) controller instances
+resource "google_compute_instance_group" "controllers" {
+  count = min(var.controller_count, length(local.zones))
+
+  name = format("%s-controllers-%s", var.cluster_name, element(local.zones, count.index))
+  zone = element(local.zones, count.index)
+
+  named_port {
+    name = "apiserver"
+    port = "6443"
+  }
+
+  # add instances in the zone into the instance group
+  instances = matchkeys(
+    google_compute_instance.controllers.*.self_link,
+    google_compute_instance.controllers.*.zone,
+    [element(local.zones, count.index)],
+  )
+}
+
+# TCP health check for apiserver
+resource "google_compute_health_check" "apiserver" {
+  name        = "${var.cluster_name}-apiserver-tcp-health"
+  description = "TCP health check for kube-apiserver"
+
+  timeout_sec        = 5
+  check_interval_sec = 5
+
+  healthy_threshold   = 1
+  unhealthy_threshold = 3
+
+  tcp_health_check {
+    port = "6443"
+  }
+}
+

google-cloud/flatcar-linux/kubernetes/bootstrap.tf

@@ -0,0 +1,19 @@
+# Kubernetes assets (kubeconfig, manifests)
+module "bootstrap" {
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9037d7311b949439b217cd9c657d4500eab3e16b"
+
+  cluster_name          = var.cluster_name
+  api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]
+  etcd_servers          = google_dns_record_set.etcds.*.name
+  networking            = var.networking
+  network_mtu           = 1440
+  pod_cidr              = var.pod_cidr
+  service_cidr          = var.service_cidr
+  cluster_domain_suffix = var.cluster_domain_suffix
+  enable_reporting      = var.enable_reporting
+  enable_aggregation    = var.enable_aggregation
+
+  // temporary
+  external_apiserver_port = 443
+}
+

google-cloud/flatcar-linux/kubernetes/cl/controller.yaml

@@ -0,0 +1,192 @@
+---
+systemd:
+  units:
+    - name: etcd-member.service
+      enabled: true
+      dropins:
+        - name: 40-etcd-cluster.conf
+          contents: |
+            [Service]
+            Environment="ETCD_IMAGE_TAG=v3.4.12"
+            Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
+            Environment="RKT_RUN_ARGS=--insecure-options=image"
+            Environment="ETCD_NAME=${etcd_name}"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
+            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
+            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
+            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
+            Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
+            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
+            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
+            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
+            Environment="ETCD_CLIENT_CERT_AUTH=true"
+            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
+            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
+            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
+            Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
+    - name: docker.service
+      enabled: true
+    - name: locksmithd.service
+      mask: true
+    - name: wait-for-dns.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+        RequiredBy=etcd-member.service
+    - name: kubelet.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubelet
+        Requires=docker.service
+        After=docker.service
+        Wants=rpc-statd.service
+        [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/bin/mkdir -p /opt/cni/bin
+        ExecStartPre=/bin/mkdir -p /var/lib/calico
+        ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
+          --anonymous-auth=false \
+          --authentication-token-webhook \
+          --authorization-mode=Webhook \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --cluster_dns=${cluster_dns_service_ip} \
+          --cluster_domain=${cluster_domain_suffix} \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --healthz-port=0 \
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --network-plugin=cni \
+          --node-labels=node.kubernetes.io/controller="true" \
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
+          --read-only-port=0 \
+          --rotate-certificates \
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
+        Restart=always
+        RestartSec=10
+        [Install]
+        WantedBy=multi-user.target
+    - name: bootstrap.service
+      contents: |
+        [Unit]
+        Description=Kubernetes control plane
+        ConditionPathExists=!/opt/bootstrap/bootstrap.done
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        WorkingDirectory=/opt/bootstrap
+        ExecStart=/usr/bin/rkt run \
+            --trust-keys-from-https \
+            --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \
+            --mount volume=config,target=/etc/kubernetes/secrets \
+            --volume assets,kind=host,source=/opt/bootstrap/assets \
+            --mount volume=assets,target=/assets \
+            --volume script,kind=host,source=/opt/bootstrap/apply \
+            --mount volume=script,target=/apply \
+            --insecure-options=image \
+            docker://quay.io/poseidon/kubelet:v1.19.3 \
+            --net=host \
+            --dns=host \
+            --exec=/apply
+        ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
+        [Install]
+        WantedBy=multi-user.target
+storage:
+  directories:
+    - path: /var/lib/etcd
+      filesystem: root
+      mode: 0700
+      overwrite: true
+  files:
+    - path: /etc/kubernetes/kubeconfig
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          ${kubeconfig}
+    - path: /opt/bootstrap/layout
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
+          awk '/#####/ {filename=$2; next} {print > filename}' assets
+          mkdir -p /etc/ssl/etcd/etcd
+          mkdir -p /etc/kubernetes/bootstrap-secrets
+          mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
+          mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/
+          chown -R etcd:etcd /etc/ssl/etcd
+          chmod -R 500 /etc/ssl/etcd
+          chmod -R 700 /var/lib/etcd
+          mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/
+          mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/
+          mkdir -p /etc/kubernetes/manifests
+          mv static-manifests/* /etc/kubernetes/manifests/
+          mkdir -p /opt/bootstrap/assets
+          mv manifests /opt/bootstrap/assets/manifests
+          mv manifests-networking/* /opt/bootstrap/assets/manifests/
+          rm -rf assets auth static-manifests tls manifests-networking
+    - path: /opt/bootstrap/apply
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig
+          until kubectl version; do
+            echo "Waiting for static pod control plane"
+            sleep 5
+          done
+          until kubectl apply -f /assets/manifests -R; do
+             echo "Retry applying manifests"
+             sleep 5
+          done
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - "${ssh_authorized_key}"

google-cloud/flatcar-linux/kubernetes/controllers.tf

@@ -0,0 +1,103 @@
+# Discrete DNS records for each controller's private IPv4 for etcd usage
+resource "google_dns_record_set" "etcds" {
+  count = var.controller_count
+
+  # DNS Zone name where record should be created
+  managed_zone = var.dns_zone_name
+
+  # DNS record
+  name = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)
+  type = "A"
+  ttl  = 300
+
+  # private IPv4 address for etcd
+  rrdatas = [google_compute_instance.controllers.*.network_interface.0.network_ip[count.index]]
+}
+
+# Zones in the region
+data "google_compute_zones" "all" {
+  region = var.region
+}
+
+locals {
+  zones = data.google_compute_zones.all.names
+
+  controllers_ipv4_public = google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip
+}
+
+# Controller instances
+resource "google_compute_instance" "controllers" {
+  count = var.controller_count
+
+  name = "${var.cluster_name}-controller-${count.index}"
+  # use a zone in the region and wrap around (e.g. controllers > zones)
+  zone         = element(local.zones, count.index)
+  machine_type = var.controller_type
+
+  metadata = {
+    user-data = data.ct_config.controller-ignitions.*.rendered[count.index]
+  }
+
+  boot_disk {
+    auto_delete = true
+
+    initialize_params {
+      image = var.os_image
+      size  = var.disk_size
+    }
+  }
+
+  network_interface {
+    network = google_compute_network.network.name
+
+    # Ephemeral external IP
+    access_config {
+    }
+  }
+
+  can_ip_forward = true
+  tags           = ["${var.cluster_name}-controller"]
+
+  lifecycle {
+    ignore_changes = [metadata]
+  }
+}
+
+# Controller Ignition configs
+data "ct_config" "controller-ignitions" {
+  count    = var.controller_count
+  content  = data.template_file.controller-configs.*.rendered[count.index]
+  strict   = true
+  snippets = var.controller_snippets
+}
+
+# Controller Container Linux configs
+data "template_file" "controller-configs" {
+  count = var.controller_count
+
+  template = file("${path.module}/cl/controller.yaml")
+
+  vars = {
+    # Cannot use cyclic dependencies on controllers or their DNS records
+    etcd_name   = "etcd${count.index}"
+    etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
+    # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
+    etcd_initial_cluster   = join(",", data.template_file.etcds.*.rendered)
+    kubeconfig             = indent(10, module.bootstrap.kubeconfig-kubelet)
+    ssh_authorized_key     = var.ssh_authorized_key
+    cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+    cluster_domain_suffix  = var.cluster_domain_suffix
+  }
+}
+
+data "template_file" "etcds" {
+  count    = var.controller_count
+  template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380"
+
+  vars = {
+    index        = count.index
+    cluster_name = var.cluster_name
+    dns_zone     = var.dns_zone
+  }
+}
+

google-cloud/flatcar-linux/kubernetes/ingress.tf

@@ -0,0 +1,123 @@
+# Static IPv4 address for Ingress Load Balancing
+resource "google_compute_global_address" "ingress-ipv4" {
+  name       = "${var.cluster_name}-ingress-ipv4"
+  ip_version = "IPV4"
+}
+
+# Static IPv6 address for Ingress Load Balancing
+resource "google_compute_global_address" "ingress-ipv6" {
+  name       = "${var.cluster_name}-ingress-ipv6"
+  ip_version = "IPV6"
+}
+
+# Forward IPv4 TCP traffic to the HTTP proxy load balancer
+# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy.
+resource "google_compute_global_forwarding_rule" "ingress-http-ipv4" {
+  name        = "${var.cluster_name}-ingress-http-ipv4"
+  ip_address  = google_compute_global_address.ingress-ipv4.address
+  ip_protocol = "TCP"
+  port_range  = "80"
+  target      = google_compute_target_http_proxy.ingress-http.self_link
+}
+
+# Forward IPv4 TCP traffic to the TCP proxy load balancer
+resource "google_compute_global_forwarding_rule" "ingress-https-ipv4" {
+  name        = "${var.cluster_name}-ingress-https-ipv4"
+  ip_address  = google_compute_global_address.ingress-ipv4.address
+  ip_protocol = "TCP"
+  port_range  = "443"
+  target      = google_compute_target_tcp_proxy.ingress-https.self_link
+}
+
+# Forward IPv6 TCP traffic to the HTTP proxy load balancer
+# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy.
+resource "google_compute_global_forwarding_rule" "ingress-http-ipv6" {
+  name        = "${var.cluster_name}-ingress-http-ipv6"
+  ip_address  = google_compute_global_address.ingress-ipv6.address
+  ip_protocol = "TCP"
+  port_range  = "80"
+  target      = google_compute_target_http_proxy.ingress-http.self_link
+}
+
+# Forward IPv6 TCP traffic to the TCP proxy load balancer
+resource "google_compute_global_forwarding_rule" "ingress-https-ipv6" {
+  name        = "${var.cluster_name}-ingress-https-ipv6"
+  ip_address  = google_compute_global_address.ingress-ipv6.address
+  ip_protocol = "TCP"
+  port_range  = "443"
+  target      = google_compute_target_tcp_proxy.ingress-https.self_link
+}
+
+# HTTP proxy load balancer for ingress controllers
+resource "google_compute_target_http_proxy" "ingress-http" {
+  name        = "${var.cluster_name}-ingress-http"
+  description = "Distribute HTTP load across ${var.cluster_name} workers"
+  url_map     = google_compute_url_map.ingress-http.self_link
+}
+
+# TCP proxy load balancer for ingress controllers
+resource "google_compute_target_tcp_proxy" "ingress-https" {
+  name            = "${var.cluster_name}-ingress-https"
+  description     = "Distribute HTTPS load across ${var.cluster_name} workers"
+  backend_service = google_compute_backend_service.ingress-https.self_link
+}
+
+# HTTP URL Map (required)
+resource "google_compute_url_map" "ingress-http" {
+  name = "${var.cluster_name}-ingress-http"
+
+  # Do not add host/path rules for applications here. Use Ingress resources.
+  default_service = google_compute_backend_service.ingress-http.self_link
+}
+
+# Backend service backed by managed instance group of workers
+resource "google_compute_backend_service" "ingress-http" {
+  name        = "${var.cluster_name}-ingress-http"
+  description = "${var.cluster_name} ingress service"
+
+  protocol         = "HTTP"
+  port_name        = "http"
+  session_affinity = "NONE"
+  timeout_sec      = "60"
+
+  backend {
+    group = module.workers.instance_group
+  }
+
+  health_checks = [google_compute_health_check.ingress.self_link]
+}
+
+# Backend service backed by managed instance group of workers
+resource "google_compute_backend_service" "ingress-https" {
+  name        = "${var.cluster_name}-ingress-https"
+  description = "${var.cluster_name} ingress service"
+
+  protocol         = "TCP"
+  port_name        = "https"
+  session_affinity = "NONE"
+  timeout_sec      = "60"
+
+  backend {
+    group = module.workers.instance_group
+  }
+
+  health_checks = [google_compute_health_check.ingress.self_link]
+}
+
+# Ingress HTTP Health Check
+resource "google_compute_health_check" "ingress" {
+  name        = "${var.cluster_name}-ingress-health"
+  description = "Health check for Ingress controller"
+
+  timeout_sec        = 5
+  check_interval_sec = 5
+
+  healthy_threshold   = 2
+  unhealthy_threshold = 4
+
+  http_health_check {
+    port         = 10254
+    request_path = "/healthz"
+  }
+}
+

google-cloud/flatcar-linux/kubernetes/network.tf

@@ -0,0 +1,219 @@
+resource "google_compute_network" "network" {
+  name                    = var.cluster_name
+  description             = "Network for the ${var.cluster_name} cluster"
+  auto_create_subnetworks = true
+
+  timeouts {
+    delete = "6m"
+  }
+}
+
+resource "google_compute_firewall" "allow-ssh" {
+  name    = "${var.cluster_name}-allow-ssh"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [22]
+  }
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+}
+
+resource "google_compute_firewall" "internal-etcd" {
+  name    = "${var.cluster_name}-internal-etcd"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [2379, 2380]
+  }
+
+  source_tags = ["${var.cluster_name}-controller"]
+  target_tags = ["${var.cluster_name}-controller"]
+}
+
+# Allow Prometheus to scrape etcd metrics
+resource "google_compute_firewall" "internal-etcd-metrics" {
+  name    = "${var.cluster_name}-internal-etcd-metrics"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [2381]
+  }
+
+  source_tags = ["${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller"]
+}
+
+# Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics
+resource "google_compute_firewall" "internal-kube-metrics" {
+  name    = "${var.cluster_name}-internal-kube-metrics"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [10251, 10252]
+  }
+
+  source_tags = ["${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller"]
+}
+
+resource "google_compute_firewall" "allow-apiserver" {
+  name    = "${var.cluster_name}-allow-apiserver"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [6443]
+  }
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["${var.cluster_name}-controller"]
+}
+
+# BGP and IPIP
+# https://docs.projectcalico.org/latest/reference/public-cloud/gce
+resource "google_compute_firewall" "internal-bgp" {
+  count = var.networking != "flannel" ? 1 : 0
+
+  name    = "${var.cluster_name}-internal-bgp"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = ["179"]
+  }
+
+  allow {
+    protocol = "ipip"
+  }
+
+  source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+}
+
+# flannel VXLAN
+resource "google_compute_firewall" "internal-vxlan" {
+  count = var.networking == "flannel" ? 1 : 0
+
+  name    = "${var.cluster_name}-internal-vxlan"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "udp"
+    ports    = [4789]
+  }
+
+  source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+}
+
+# Cilium VXLAN
+resource "google_compute_firewall" "internal-linux-vxlan" {
+  count = var.networking == "cilium" ? 1 : 0
+
+  name    = "${var.cluster_name}-linux-vxlan"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "udp"
+    ports    = [8472]
+  }
+
+  # Cilium health
+  allow {
+    protocol = "icmp"
+  }
+
+  allow {
+    protocol = "tcp"
+    ports    = [4240]
+  }
+
+  source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+}
+
+# Allow Prometheus to scrape node-exporter daemonset
+resource "google_compute_firewall" "internal-node-exporter" {
+  name    = "${var.cluster_name}-internal-node-exporter"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [9100]
+  }
+
+  source_tags = ["${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+}
+
+# Allow Prometheus to scrape kube-proxy metrics
+resource "google_compute_firewall" "internal-kube-proxy" {
+  name    = "${var.cluster_name}-internal-kube-proxy"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [10249]
+  }
+
+  source_tags = ["${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+}
+
+# Allow apiserver to access kubelets for exec, log, port-forward
+resource "google_compute_firewall" "internal-kubelet" {
+  name    = "${var.cluster_name}-internal-kubelet"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [10250]
+  }
+
+  # allow Prometheus to scrape kubelet metrics too
+  source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+}
+
+# Workers
+
+resource "google_compute_firewall" "allow-ingress" {
+  name    = "${var.cluster_name}-allow-ingress"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [80, 443]
+  }
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["${var.cluster_name}-worker"]
+}
+
+resource "google_compute_firewall" "google-ingress-health-checks" {
+  name    = "${var.cluster_name}-ingress-health"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [10254]
+  }
+
+  # https://cloud.google.com/load-balancing/docs/health-check-concepts#method
+  source_ranges = [
+    "35.191.0.0/16",
+    "130.211.0.0/22",
+    "35.191.0.0/16",
+    "209.85.152.0/22",
+    "209.85.204.0/22",
+  ]
+
+  target_tags = ["${var.cluster_name}-worker"]
+}
+

google-cloud/flatcar-linux/kubernetes/outputs.tf

@@ -0,0 +1,50 @@
+output "kubeconfig-admin" {
+  value = module.bootstrap.kubeconfig-admin
+}
+
+# Outputs for Kubernetes Ingress
+
+output "ingress_static_ipv4" {
+  description = "Global IPv4 address for proxy load balancing to the nearest Ingress controller"
+  value       = google_compute_global_address.ingress-ipv4.address
+}
+
+output "ingress_static_ipv6" {
+  description = "Global IPv6 address for proxy load balancing to the nearest Ingress controller"
+  value       = google_compute_global_address.ingress-ipv6.address
+}
+
+# Outputs for worker pools
+
+output "network_name" {
+  value = google_compute_network.network.name
+}
+
+output "kubeconfig" {
+  value = module.bootstrap.kubeconfig-kubelet
+}
+
+# Outputs for custom firewalling
+
+output "network_self_link" {
+  value = google_compute_network.network.self_link
+}
+
+# Outputs for custom load balancing
+
+output "worker_instance_group" {
+  description = "Worker managed instance group full URL"
+  value       = module.workers.instance_group
+}
+
+output "worker_target_pool" {
+  description = "Worker target pool self link"
+  value       = module.workers.target_pool
+}
+
+# Outputs for debug
+
+output "assets_dist" {
+  value = module.bootstrap.assets_dist
+}
+

google-cloud/flatcar-linux/kubernetes/ssh.tf

@@ -0,0 +1,58 @@
+locals {
+  # format assets for distribution
+  assets_bundle = [
+    # header with the unpack location
+    for key, value in module.bootstrap.assets_dist :
+    format("##### %s\n%s", key, value)
+  ]
+}
+
+# Secure copy assets to controllers.
+resource "null_resource" "copy-controller-secrets" {
+  count = var.controller_count
+
+  depends_on = [
+    module.bootstrap,
+  ]
+
+  connection {
+    type    = "ssh"
+    host    = local.controllers_ipv4_public[count.index]
+    user    = "core"
+    timeout = "15m"
+  }
+
+  provisioner "file" {
+    content     = join("\n", local.assets_bundle)
+    destination = "$HOME/assets"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo /opt/bootstrap/layout",
+    ]
+  }
+}
+
+# Connect to a controller to perform one-time cluster bootstrap.
+resource "null_resource" "bootstrap" {
+  depends_on = [
+    null_resource.copy-controller-secrets,
+    module.workers,
+    google_dns_record_set.apiserver,
+  ]
+
+  connection {
+    type    = "ssh"
+    host    = local.controllers_ipv4_public[0]
+    user    = "core"
+    timeout = "15m"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo systemctl start bootstrap",
+    ]
+  }
+}
+

google-cloud/flatcar-linux/kubernetes/variables.tf

@@ -0,0 +1,132 @@
+variable "cluster_name" {
+  type        = string
+  description = "Unique cluster name (prepended to dns_zone)"
+}
+
+# Google Cloud
+
+variable "region" {
+  type        = string
+  description = "Google Cloud Region (e.g. us-central1, see `gcloud compute regions list`)"
+}
+
+variable "dns_zone" {
+  type        = string
+  description = "Google Cloud DNS Zone (e.g. google-cloud.example.com)"
+}
+
+variable "dns_zone_name" {
+  type        = string
+  description = "Google Cloud DNS Zone name (e.g. example-zone)"
+}
+
+# instances
+
+variable "controller_count" {
+  type        = number
+  description = "Number of controllers (i.e. masters)"
+  default     = 1
+}
+
+variable "worker_count" {
+  type        = number
+  description = "Number of workers"
+  default     = 1
+}
+
+variable "controller_type" {
+  type        = string
+  description = "Machine type for controllers (see `gcloud compute machine-types list`)"
+  default     = "n1-standard-1"
+}
+
+variable "worker_type" {
+  type        = string
+  description = "Machine type for controllers (see `gcloud compute machine-types list`)"
+  default     = "n1-standard-1"
+}
+
+variable "os_image" {
+  type        = string
+  description = "Flatcar Linux image for compute instances (e.g. custom-image)"
+}
+
+variable "disk_size" {
+  type        = number
+  description = "Size of the disk in GB"
+  default     = 40
+}
+
+variable "worker_preemptible" {
+  type        = bool
+  description = "If enabled, Compute Engine will terminate workers randomly within 24 hours"
+  default     = false
+}
+
+variable "controller_snippets" {
+  type        = list(string)
+  description = "Controller Container Linux Config snippets"
+  default     = []
+}
+
+variable "worker_snippets" {
+  type        = list(string)
+  description = "Worker Container Linux Config snippets"
+  default     = []
+}
+
+# configuration
+
+variable "ssh_authorized_key" {
+  type        = string
+  description = "SSH public key for user 'core'"
+}
+
+variable "networking" {
+  type        = string
+  description = "Choice of networking provider (flannel or calico)"
+  default     = "calico"
+}
+
+variable "pod_cidr" {
+  type        = string
+  description = "CIDR IPv4 range to assign Kubernetes pods"
+  default     = "10.2.0.0/16"
+}
+
+variable "service_cidr" {
+  type        = string
+  description = <<EOD
+CIDR IPv4 range to assign Kubernetes services.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
+EOD
+  default     = "10.3.0.0/16"
+}
+
+
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable usage or analytics reporting to upstreams (Calico)"
+  default     = false
+}
+
+variable "enable_aggregation" {
+  type        = bool
+  description = "Enable the Kubernetes Aggregation Layer (defaults to false)"
+  default     = false
+}
+
+variable "worker_node_labels" {
+  type        = list(string)
+  description = "List of initial worker node labels"
+  default     = []
+}
+
+# unofficial, undocumented, unsupported
+
+variable "cluster_domain_suffix" {
+  type        = string
+  description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  default     = "cluster.local"
+}
+

google-cloud/flatcar-linux/kubernetes/versions.tf

@@ -0,0 +1,15 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.12.26, < 0.14.0"
+  required_providers {
+    google   = ">= 2.19, < 4.0"
+    template = "~> 2.1"
+    null     = "~> 2.1"
+
+    ct = {
+      source  = "poseidon/ct"
+      version = "~> 0.6.1"
+    }
+  }
+}

google-cloud/flatcar-linux/kubernetes/workers.tf

@@ -0,0 +1,23 @@
+module "workers" {
+  source       = "./workers"
+  name         = var.cluster_name
+  cluster_name = var.cluster_name
+
+  # GCE
+  region       = var.region
+  network      = google_compute_network.network.name
+  worker_count = var.worker_count
+  machine_type = var.worker_type
+  os_image     = var.os_image
+  disk_size    = var.disk_size
+  preemptible  = var.worker_preemptible
+
+  # configuration
+  kubeconfig            = module.bootstrap.kubeconfig-kubelet
+  ssh_authorized_key    = var.ssh_authorized_key
+  service_cidr          = var.service_cidr
+  cluster_domain_suffix = var.cluster_domain_suffix
+  snippets              = var.worker_snippets
+  node_labels           = var.worker_node_labels
+}
+

google-cloud/flatcar-linux/kubernetes/workers/cl/worker.yaml

@@ -0,0 +1,115 @@
+---
+systemd:
+  units:
+    - name: docker.service
+      enabled: true
+    - name: locksmithd.service
+      mask: true
+    - name: wait-for-dns.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+    - name: kubelet.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubelet
+        Requires=docker.service
+        After=docker.service
+        Wants=rpc-statd.service
+        [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/bin/mkdir -p /opt/cni/bin
+        ExecStartPre=/bin/mkdir -p /var/lib/calico
+        ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        # Podman, rkt, or runc run container processes, whereas docker run
+        # is a client to a daemon and requires workarounds to use within a
+        # systemd unit. https://github.com/moby/moby/issues/6791
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
+          --anonymous-auth=false \
+          --authentication-token-webhook \
+          --authorization-mode=Webhook \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --cluster_dns=${cluster_dns_service_ip} \
+          --cluster_domain=${cluster_domain_suffix} \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --healthz-port=0 \
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --network-plugin=cni \
+          --node-labels=node.kubernetes.io/node \
+          %{~ for label in split(",", node_labels) ~}
+          --node-labels=${label} \
+          %{~ endfor ~}
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --read-only-port=0 \
+          --rotate-certificates \
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
+        Restart=always
+        RestartSec=5
+        [Install]
+        WantedBy=multi-user.target
+    - name: delete-node.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Delete Kubernetes node on shutdown
+        [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/true
+        ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
+        [Install]
+        WantedBy=multi-user.target
+storage:
+  files:
+    - path: /etc/kubernetes/kubeconfig
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          ${kubeconfig}
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - "${ssh_authorized_key}"

google-cloud/flatcar-linux/kubernetes/workers/outputs.tf

@@ -0,0 +1,14 @@
+# Outputs for global load balancing
+
+output "instance_group" {
+  description = "Worker managed instance group full URL"
+  value       = google_compute_region_instance_group_manager.workers.instance_group
+}
+
+# Outputs for regional load balancing
+
+output "target_pool" {
+  description = "Worker target pool self link"
+  value       = google_compute_target_pool.workers.self_link
+}
+

google-cloud/flatcar-linux/kubernetes/workers/target_pool.tf

@@ -0,0 +1,23 @@
+# Target pool for TCP/UDP load balancing
+resource "google_compute_target_pool" "workers" {
+  name             = "${var.name}-worker-pool"
+  region           = var.region
+  session_affinity = "NONE"
+
+  health_checks = [
+    google_compute_http_health_check.workers.name,
+  ]
+}
+
+# HTTP Health Check (for TCP/UDP load balancing)
+# Forward rules (regional) to target pools don't support different external
+# and internal ports. Health check for nodes with Ingress controllers that
+# may support proxying or otherwise satisfy the check.
+resource "google_compute_http_health_check" "workers" {
+  name        = "${var.name}-target-pool-health"
+  description = "Health check for the worker target pool"
+
+  port         = 10254
+  request_path = "/healthz"
+}
+

google-cloud/flatcar-linux/kubernetes/workers/variables.tf

@@ -0,0 +1,106 @@
+variable "name" {
+  type        = string
+  description = "Unique name for the worker pool"
+}
+
+variable "cluster_name" {
+  type        = string
+  description = "Must be set to `cluster_name of cluster`"
+}
+
+# Google Cloud
+
+variable "region" {
+  type        = string
+  description = "Must be set to `region` of cluster"
+}
+
+variable "network" {
+  type        = string
+  description = "Must be set to `network_name` output by cluster"
+}
+
+# instances
+
+variable "worker_count" {
+  type        = number
+  description = "Number of worker compute instances the instance group should manage"
+  default     = 1
+}
+
+variable "machine_type" {
+  type        = string
+  description = "Machine type for compute instances (e.g. gcloud compute machine-types list)"
+  default     = "n1-standard-1"
+}
+
+variable "os_image" {
+  type        = string
+  description = "Flatcar Linux image for compute instanges (e.g. gcloud compute images list)"
+}
+
+variable "disk_size" {
+  type        = number
+  description = "Size of the disk in GB"
+  default     = 40
+}
+
+variable "preemptible" {
+  type        = bool
+  description = "If enabled, Compute Engine will terminate instances randomly within 24 hours"
+  default     = false
+}
+
+variable "snippets" {
+  type        = list(string)
+  description = "Container Linux Config snippets"
+  default     = []
+}
+
+# configuration
+
+variable "kubeconfig" {
+  type        = string
+  description = "Must be set to `kubeconfig` output by cluster"
+}
+
+variable "ssh_authorized_key" {
+  type        = string
+  description = "SSH public key for user 'core'"
+}
+
+variable "service_cidr" {
+  type        = string
+  description = <<EOD
+CIDR IPv4 range to assign Kubernetes services.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
+EOD
+  default     = "10.3.0.0/16"
+}
+
+variable "node_labels" {
+  type        = list(string)
+  description = "List of initial node labels"
+  default     = []
+}
+
+# unofficial, undocumented, unsupported, temporary
+
+variable "cluster_domain_suffix" {
+  type        = string
+  description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  default     = "cluster.local"
+}
+
+variable "accelerator_type" {
+  type        = string
+  default     = ""
+  description = "Google Compute Engine accelerator type (e.g. nvidia-tesla-k80, see gcloud compute accelerator-types list)"
+}
+
+variable "accelerator_count" {
+  type        = string
+  default     = "0"
+  description = "Number of compute engine accelerators"
+}
+

google-cloud/flatcar-linux/kubernetes/workers/versions.tf

@@ -0,0 +1,14 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.12.26, < 0.14.0"
+  required_providers {
+    google   = ">= 2.19, < 4.0"
+    template = "~> 2.1"
+
+    ct = {
+      source  = "poseidon/ct"
+      version = "~> 0.6.1"
+    }
+  }
+}

google-cloud/flatcar-linux/kubernetes/workers/workers.tf

@@ -0,0 +1,91 @@
+# Managed instance group of workers
+resource "google_compute_region_instance_group_manager" "workers" {
+  name        = "${var.name}-worker-group"
+  description = "Compute instance group of ${var.name} workers"
+
+  # instance name prefix for instances in the group
+  base_instance_name = "${var.name}-worker"
+  region             = var.region
+  version {
+    name              = "default"
+    instance_template = google_compute_instance_template.worker.self_link
+  }
+
+  target_size  = var.worker_count
+  target_pools = [google_compute_target_pool.workers.self_link]
+
+  named_port {
+    name = "http"
+    port = "80"
+  }
+
+  named_port {
+    name = "https"
+    port = "443"
+  }
+}
+
+# Worker instance template
+resource "google_compute_instance_template" "worker" {
+  name_prefix  = "${var.name}-worker-"
+  description  = "Worker Instance template"
+  machine_type = var.machine_type
+
+  metadata = {
+    user-data = data.ct_config.worker-ignition.rendered
+  }
+
+  scheduling {
+    automatic_restart = var.preemptible ? false : true
+    preemptible       = var.preemptible
+  }
+
+  disk {
+    auto_delete  = true
+    boot         = true
+    source_image = var.os_image
+    disk_size_gb = var.disk_size
+  }
+
+  network_interface {
+    network = var.network
+
+    # Ephemeral external IP
+    access_config {
+    }
+  }
+
+  can_ip_forward = true
+  tags           = ["worker", "${var.cluster_name}-worker", "${var.name}-worker"]
+
+  guest_accelerator {
+    count = var.accelerator_count
+    type  = var.accelerator_type
+  }
+
+  lifecycle {
+    # To update an Instance Template, Terraform should replace the existing resource
+    create_before_destroy = true
+  }
+}
+
+# Worker Ignition config
+data "ct_config" "worker-ignition" {
+  content  = data.template_file.worker-config.rendered
+  strict   = true
+  snippets = var.snippets
+}
+
+# Worker Container Linux config
+data "template_file" "worker-config" {
+  template = file("${path.module}/cl/worker.yaml")
+
+  vars = {
+    kubeconfig             = indent(10, var.kubeconfig)
+    ssh_authorized_key     = var.ssh_authorized_key
+    cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+    cluster_domain_suffix  = var.cluster_domain_suffix
+    node_labels            = join(",", var.node_labels)
+  }
+}
+