Rename container-linux modules to flatcar-linux

* CoreOS Container Linux was deprecated in v1.18.3
* Continue transitioning docs and modules from supporting
both CoreOS and Flatcar "variants" of Container Linux to
now supporting Flatcar Linux and equivalents

Action Required: Update the Flatcar Linux modules `source`
to replace `s/container-linux/flatcar-linux`. See docs for
examples

azure/flatcar-linux/kubernetes/LICENSE

@@ -0,0 +1,23 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Typhoon Authors
+Copyright (c) 2017 Dalton Hubble
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

azure/flatcar-linux/kubernetes/README.md

@@ -0,0 +1,23 @@
+# Typhoon <img align="right" src="https://storage.googleapis.com/poseidon/typhoon-logo.png">
+
+Typhoon is a minimal and free Kubernetes distribution.
+
+* Minimal, stable base Kubernetes distribution
+* Declarative infrastructure and configuration
+* Free (freedom and cost) and privacy-respecting
+* Practical for labs, datacenters, and clouds
+
+Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components.
+
+## Features <a href="https://www.cncf.io/certification/software-conformance/"><img align="right" src="https://storage.googleapis.com/poseidon/certified-kubernetes.png"></a>
+
+* Kubernetes v1.19.3 (upstream)
+* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking
+* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
+* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/cl/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization
+* Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/)
+
+## Docs
+
+Please see the [official docs](https://typhoon.psdn.io) and the Azure [tutorial](https://typhoon.psdn.io/cl/azure/).
+

azure/flatcar-linux/kubernetes/bootstrap.tf

@@ -0,0 +1,22 @@
+# Kubernetes assets (kubeconfig, manifests)
+module "bootstrap" {
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9037d7311b949439b217cd9c657d4500eab3e16b"
+
+  cluster_name = var.cluster_name
+  api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]
+  etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone)
+
+  networking = var.networking
+
+  # only effective with Calico networking
+  # we should be able to use 1450 MTU, but in practice, 1410 was needed
+  network_encapsulation = "vxlan"
+  network_mtu           = "1410"
+
+  pod_cidr              = var.pod_cidr
+  service_cidr          = var.service_cidr
+  cluster_domain_suffix = var.cluster_domain_suffix
+  enable_reporting      = var.enable_reporting
+  enable_aggregation    = var.enable_aggregation
+}
+

azure/flatcar-linux/kubernetes/cl/controller.yaml

@@ -0,0 +1,194 @@
+---
+systemd:
+  units:
+    - name: etcd-member.service
+      enabled: true
+      dropins:
+        - name: 40-etcd-cluster.conf
+          contents: |
+            [Service]
+            Environment="ETCD_IMAGE_TAG=v3.4.12"
+            Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
+            Environment="RKT_RUN_ARGS=--insecure-options=image"
+            Environment="ETCD_NAME=${etcd_name}"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
+            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
+            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
+            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
+            Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
+            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
+            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
+            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
+            Environment="ETCD_CLIENT_CERT_AUTH=true"
+            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
+            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
+            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
+            Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
+    - name: docker.service
+      enabled: true
+    - name: locksmithd.service
+      mask: true
+    - name: wait-for-dns.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+        RequiredBy=etcd-member.service
+    - name: kubelet.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubelet
+        Requires=docker.service
+        After=docker.service
+        Wants=rpc-statd.service
+        [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/bin/mkdir -p /opt/cni/bin
+        ExecStartPre=/bin/mkdir -p /var/lib/calico
+        ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
+          --anonymous-auth=false \
+          --authentication-token-webhook \
+          --authorization-mode=Webhook \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --cluster_dns=${cluster_dns_service_ip} \
+          --cluster_domain=${cluster_domain_suffix} \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --healthz-port=0 \
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --network-plugin=cni \
+          --node-labels=node.kubernetes.io/controller="true" \
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --read-only-port=0 \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
+          --rotate-certificates \
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
+        Restart=always
+        RestartSec=10
+        [Install]
+        WantedBy=multi-user.target
+    - name: bootstrap.service
+      contents: |
+        [Unit]
+        Description=Kubernetes control plane
+        ConditionPathExists=!/opt/bootstrap/bootstrap.done
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        WorkingDirectory=/opt/bootstrap
+        ExecStart=/usr/bin/rkt run \
+            --trust-keys-from-https \
+            --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \
+            --mount volume=config,target=/etc/kubernetes/secrets \
+            --volume assets,kind=host,source=/opt/bootstrap/assets \
+            --mount volume=assets,target=/assets \
+            --volume script,kind=host,source=/opt/bootstrap/apply \
+            --mount volume=script,target=/apply \
+            --insecure-options=image \
+            docker://quay.io/poseidon/kubelet:v1.19.3 \
+            --net=host \
+            --dns=host \
+            --exec=/apply
+        ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
+        [Install]
+        WantedBy=multi-user.target
+storage:
+  directories:
+    - path: /var/lib/etcd
+      filesystem: root
+      mode: 0700
+      overwrite: true
+  files:
+    - path: /etc/kubernetes/kubeconfig
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          ${kubeconfig}
+    - path: /opt/bootstrap/layout
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
+          awk '/#####/ {filename=$2; next} {print > filename}' assets
+          mkdir -p /etc/ssl/etcd/etcd
+          mkdir -p /etc/kubernetes/bootstrap-secrets
+          mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
+          mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/
+          chown -R etcd:etcd /etc/ssl/etcd
+          chmod -R 500 /etc/ssl/etcd
+          chmod -R 700 /var/lib/etcd
+          mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/
+          mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/
+          mkdir -p /etc/kubernetes/manifests
+          mv static-manifests/* /etc/kubernetes/manifests/
+          mkdir -p /opt/bootstrap/assets
+          mv manifests /opt/bootstrap/assets/manifests
+          mv manifests-networking/* /opt/bootstrap/assets/manifests/
+          rm -rf assets auth static-manifests tls manifests-networking
+    - path: /opt/bootstrap/apply
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig
+          until kubectl version; do
+            echo "Waiting for static pod control plane"
+            sleep 5
+          done
+          until kubectl apply -f /assets/manifests -R; do
+             echo "Retry applying manifests"
+             sleep 5
+          done
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - "${ssh_authorized_key}"

azure/flatcar-linux/kubernetes/controllers.tf

@@ -0,0 +1,171 @@
+# Discrete DNS records for each controller's private IPv4 for etcd usage
+resource "azurerm_dns_a_record" "etcds" {
+  count               = var.controller_count
+  resource_group_name = var.dns_zone_group
+
+  # DNS Zone name where record should be created
+  zone_name = var.dns_zone
+
+  # DNS record
+  name = format("%s-etcd%d", var.cluster_name, count.index)
+  ttl  = 300
+
+  # private IPv4 address for etcd
+  records = [azurerm_network_interface.controllers.*.private_ip_address[count.index]]
+}
+
+locals {
+  # Container Linux derivative
+  # flatcar-stable -> Flatcar Linux Stable
+  channel = split("-", var.os_image)[1]
+}
+
+# Controller availability set to spread controllers
+resource "azurerm_availability_set" "controllers" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                         = "${var.cluster_name}-controllers"
+  location                     = var.region
+  platform_fault_domain_count  = 2
+  platform_update_domain_count = 4
+  managed                      = true
+}
+
+# Controller instances
+resource "azurerm_linux_virtual_machine" "controllers" {
+  count               = var.controller_count
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                = "${var.cluster_name}-controller-${count.index}"
+  location            = var.region
+  availability_set_id = azurerm_availability_set.controllers.id
+
+  size        = var.controller_type
+  custom_data = base64encode(data.ct_config.controller-ignitions.*.rendered[count.index])
+
+  # storage
+  os_disk {
+    name                 = "${var.cluster_name}-controller-${count.index}"
+    caching              = "None"
+    disk_size_gb         = var.disk_size
+    storage_account_type = "Premium_LRS"
+  }
+
+  # Flatcar Container Linux
+  source_image_reference {
+    publisher = "Kinvolk"
+    offer     = "flatcar-container-linux-free"
+    sku       = local.channel
+    version   = "latest"
+  }
+
+  plan {
+    name      = local.channel
+    publisher = "kinvolk"
+    product   = "flatcar-container-linux-free"
+  }
+
+  # network
+  network_interface_ids = [
+    azurerm_network_interface.controllers.*.id[count.index]
+  ]
+
+  # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too
+  admin_username = "core"
+  admin_ssh_key {
+    username   = "core"
+    public_key = var.ssh_authorized_key
+  }
+
+  lifecycle {
+    ignore_changes = [
+      os_disk,
+      custom_data,
+    ]
+  }
+}
+
+# Controller public IPv4 addresses
+resource "azurerm_public_ip" "controllers" {
+  count               = var.controller_count
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name              = "${var.cluster_name}-controller-${count.index}"
+  location          = azurerm_resource_group.cluster.location
+  sku               = "Standard"
+  allocation_method = "Static"
+}
+
+# Controller NICs with public and private IPv4
+resource "azurerm_network_interface" "controllers" {
+  count               = var.controller_count
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name     = "${var.cluster_name}-controller-${count.index}"
+  location = azurerm_resource_group.cluster.location
+
+  ip_configuration {
+    name                          = "ip0"
+    subnet_id                     = azurerm_subnet.controller.id
+    private_ip_address_allocation = "Dynamic"
+    # instance public IPv4
+    public_ip_address_id = azurerm_public_ip.controllers.*.id[count.index]
+  }
+}
+
+# Associate controller network interface with controller security group
+resource "azurerm_network_interface_security_group_association" "controllers" {
+  count = var.controller_count
+
+  network_interface_id      = azurerm_network_interface.controllers[count.index].id
+  network_security_group_id = azurerm_network_security_group.controller.id
+}
+
+# Associate controller network interface with controller backend address pool
+resource "azurerm_network_interface_backend_address_pool_association" "controllers" {
+  count = var.controller_count
+
+  network_interface_id    = azurerm_network_interface.controllers[count.index].id
+  ip_configuration_name   = "ip0"
+  backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id
+}
+
+# Controller Ignition configs
+data "ct_config" "controller-ignitions" {
+  count    = var.controller_count
+  content  = data.template_file.controller-configs.*.rendered[count.index]
+  strict   = true
+  snippets = var.controller_snippets
+}
+
+# Controller Container Linux configs
+data "template_file" "controller-configs" {
+  count = var.controller_count
+
+  template = file("${path.module}/cl/controller.yaml")
+
+  vars = {
+    # Cannot use cyclic dependencies on controllers or their DNS records
+    etcd_name   = "etcd${count.index}"
+    etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
+    # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
+    etcd_initial_cluster   = join(",", data.template_file.etcds.*.rendered)
+    cgroup_driver          = local.channel == "edge" ? "systemd" : "cgroupfs"
+    kubeconfig             = indent(10, module.bootstrap.kubeconfig-kubelet)
+    ssh_authorized_key     = var.ssh_authorized_key
+    cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+    cluster_domain_suffix  = var.cluster_domain_suffix
+  }
+}
+
+data "template_file" "etcds" {
+  count    = var.controller_count
+  template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380"
+
+  vars = {
+    index        = count.index
+    cluster_name = var.cluster_name
+    dns_zone     = var.dns_zone
+  }
+}
+

azure/flatcar-linux/kubernetes/lb.tf

@@ -0,0 +1,161 @@
+# DNS record for the apiserver load balancer
+resource "azurerm_dns_a_record" "apiserver" {
+  resource_group_name = var.dns_zone_group
+
+  # DNS Zone name where record should be created
+  zone_name = var.dns_zone
+
+  # DNS record
+  name = var.cluster_name
+  ttl  = 300
+
+  # IPv4 address of apiserver load balancer
+  records = [azurerm_public_ip.apiserver-ipv4.ip_address]
+}
+
+# Static IPv4 address for the apiserver frontend
+resource "azurerm_public_ip" "apiserver-ipv4" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name              = "${var.cluster_name}-apiserver-ipv4"
+  location          = var.region
+  sku               = "Standard"
+  allocation_method = "Static"
+}
+
+# Static IPv4 address for the ingress frontend
+resource "azurerm_public_ip" "ingress-ipv4" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name              = "${var.cluster_name}-ingress-ipv4"
+  location          = var.region
+  sku               = "Standard"
+  allocation_method = "Static"
+}
+
+# Network Load Balancer for apiservers and ingress
+resource "azurerm_lb" "cluster" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name     = var.cluster_name
+  location = var.region
+  sku      = "Standard"
+
+  frontend_ip_configuration {
+    name                 = "apiserver"
+    public_ip_address_id = azurerm_public_ip.apiserver-ipv4.id
+  }
+
+  frontend_ip_configuration {
+    name                 = "ingress"
+    public_ip_address_id = azurerm_public_ip.ingress-ipv4.id
+  }
+}
+
+resource "azurerm_lb_rule" "apiserver" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                           = "apiserver"
+  loadbalancer_id                = azurerm_lb.cluster.id
+  frontend_ip_configuration_name = "apiserver"
+
+  protocol                = "Tcp"
+  frontend_port           = 6443
+  backend_port            = 6443
+  backend_address_pool_id = azurerm_lb_backend_address_pool.controller.id
+  probe_id                = azurerm_lb_probe.apiserver.id
+}
+
+resource "azurerm_lb_rule" "ingress-http" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                           = "ingress-http"
+  loadbalancer_id                = azurerm_lb.cluster.id
+  frontend_ip_configuration_name = "ingress"
+  disable_outbound_snat          = true
+
+  protocol                = "Tcp"
+  frontend_port           = 80
+  backend_port            = 80
+  backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
+  probe_id                = azurerm_lb_probe.ingress.id
+}
+
+resource "azurerm_lb_rule" "ingress-https" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                           = "ingress-https"
+  loadbalancer_id                = azurerm_lb.cluster.id
+  frontend_ip_configuration_name = "ingress"
+  disable_outbound_snat          = true
+
+  protocol                = "Tcp"
+  frontend_port           = 443
+  backend_port            = 443
+  backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
+  probe_id                = azurerm_lb_probe.ingress.id
+}
+
+# Worker outbound TCP/UDP SNAT
+resource "azurerm_lb_outbound_rule" "worker-outbound" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name            = "worker"
+  loadbalancer_id = azurerm_lb.cluster.id
+  frontend_ip_configuration {
+    name = "ingress"
+  }
+
+  protocol                = "All"
+  backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
+}
+
+# Address pool of controllers
+resource "azurerm_lb_backend_address_pool" "controller" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name            = "controller"
+  loadbalancer_id = azurerm_lb.cluster.id
+}
+
+# Address pool of workers
+resource "azurerm_lb_backend_address_pool" "worker" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name            = "worker"
+  loadbalancer_id = azurerm_lb.cluster.id
+}
+
+# Health checks / probes
+
+# TCP health check for apiserver
+resource "azurerm_lb_probe" "apiserver" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name            = "apiserver"
+  loadbalancer_id = azurerm_lb.cluster.id
+  protocol        = "Tcp"
+  port            = 6443
+
+  # unhealthy threshold
+  number_of_probes = 3
+
+  interval_in_seconds = 5
+}
+
+# HTTP health check for ingress
+resource "azurerm_lb_probe" "ingress" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name            = "ingress"
+  loadbalancer_id = azurerm_lb.cluster.id
+  protocol        = "Http"
+  port            = 10254
+  request_path    = "/healthz"
+
+  # unhealthy threshold
+  number_of_probes = 3
+
+  interval_in_seconds = 5
+}
+

azure/flatcar-linux/kubernetes/network.tf

@@ -0,0 +1,44 @@
+# Organize cluster into a resource group
+resource "azurerm_resource_group" "cluster" {
+  name     = var.cluster_name
+  location = var.region
+}
+
+resource "azurerm_virtual_network" "network" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name          = var.cluster_name
+  location      = azurerm_resource_group.cluster.location
+  address_space = [var.host_cidr]
+}
+
+# Subnets - separate subnets for controller and workers because Azure
+# network security groups are based on IPv4 CIDR rather than instance
+# tags like GCP or security group membership like AWS
+
+resource "azurerm_subnet" "controller" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                 = "controller"
+  virtual_network_name = azurerm_virtual_network.network.name
+  address_prefixes     = [cidrsubnet(var.host_cidr, 1, 0)]
+}
+
+resource "azurerm_subnet_network_security_group_association" "controller" {
+  subnet_id                 = azurerm_subnet.controller.id
+  network_security_group_id = azurerm_network_security_group.controller.id
+}
+
+resource "azurerm_subnet" "worker" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                 = "worker"
+  virtual_network_name = azurerm_virtual_network.network.name
+  address_prefixes     = [cidrsubnet(var.host_cidr, 1, 1)]
+}
+
+resource "azurerm_subnet_network_security_group_association" "worker" {
+  subnet_id                 = azurerm_subnet.worker.id
+  network_security_group_id = azurerm_network_security_group.worker.id
+}
+

azure/flatcar-linux/kubernetes/outputs.tf

@@ -0,0 +1,66 @@
+output "kubeconfig-admin" {
+  value = module.bootstrap.kubeconfig-admin
+}
+
+# Outputs for Kubernetes Ingress
+
+output "ingress_static_ipv4" {
+  value       = azurerm_public_ip.ingress-ipv4.ip_address
+  description = "IPv4 address of the load balancer for distributing traffic to Ingress controllers"
+}
+
+# Outputs for worker pools
+
+output "region" {
+  value = azurerm_resource_group.cluster.location
+}
+
+output "resource_group_name" {
+  value = azurerm_resource_group.cluster.name
+}
+
+output "resource_group_id" {
+  value = azurerm_resource_group.cluster.id
+}
+
+output "subnet_id" {
+  value = azurerm_subnet.worker.id
+}
+
+output "security_group_id" {
+  value = azurerm_network_security_group.worker.id
+}
+
+output "kubeconfig" {
+  value = module.bootstrap.kubeconfig-kubelet
+}
+
+# Outputs for custom firewalling
+
+output "worker_security_group_name" {
+  value = azurerm_network_security_group.worker.name
+}
+
+output "worker_address_prefix" {
+  description = "Worker network subnet CIDR address (for source/destination)"
+  value       = azurerm_subnet.worker.address_prefix
+}
+
+# Outputs for custom load balancing
+
+output "loadbalancer_id" {
+  description = "ID of the cluster load balancer"
+  value       = azurerm_lb.cluster.id
+}
+
+output "backend_address_pool_id" {
+  description = "ID of the worker backend address pool"
+  value       = azurerm_lb_backend_address_pool.worker.id
+}
+
+# Outputs for debug
+
+output "assets_dist" {
+  value = module.bootstrap.assets_dist
+}
+

azure/flatcar-linux/kubernetes/security.tf

@@ -0,0 +1,428 @@
+# Controller security group
+
+resource "azurerm_network_security_group" "controller" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name     = "${var.cluster_name}-controller"
+  location = azurerm_resource_group.cluster.location
+}
+
+resource "azurerm_network_security_rule" "controller-icmp" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-icmp"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "1995"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Icmp"
+  source_port_range           = "*"
+  destination_port_range      = "*"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+resource "azurerm_network_security_rule" "controller-ssh" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-ssh"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2000"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "22"
+  source_address_prefix       = "*"
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+resource "azurerm_network_security_rule" "controller-etcd" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-etcd"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2005"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "2379-2380"
+  source_address_prefix       = azurerm_subnet.controller.address_prefix
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+# Allow Prometheus to scrape etcd metrics
+resource "azurerm_network_security_rule" "controller-etcd-metrics" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-etcd-metrics"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2010"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "2381"
+  source_address_prefix       = azurerm_subnet.worker.address_prefix
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+# Allow Prometheus to scrape kube-proxy metrics
+resource "azurerm_network_security_rule" "controller-kube-proxy" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-kube-proxy-metrics"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2011"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "10249"
+  source_address_prefix       = azurerm_subnet.worker.address_prefix
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+# Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics
+resource "azurerm_network_security_rule" "controller-kube-metrics" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-kube-metrics"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2012"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "10251-10252"
+  source_address_prefix       = azurerm_subnet.worker.address_prefix
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+resource "azurerm_network_security_rule" "controller-apiserver" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-apiserver"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2015"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "6443"
+  source_address_prefix       = "*"
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+resource "azurerm_network_security_rule" "controller-cilium-health" {
+  resource_group_name = azurerm_resource_group.cluster.name
+  count               = var.networking == "cilium" ? 1 : 0
+
+  name                        = "allow-cilium-health"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2019"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "4240"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+resource "azurerm_network_security_rule" "controller-vxlan" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-vxlan"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2020"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Udp"
+  source_port_range           = "*"
+  destination_port_range      = "4789"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+resource "azurerm_network_security_rule" "controller-linux-vxlan" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-linux-vxlan"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2021"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Udp"
+  source_port_range           = "*"
+  destination_port_range      = "8472"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+# Allow Prometheus to scrape node-exporter daemonset
+resource "azurerm_network_security_rule" "controller-node-exporter" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-node-exporter"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2025"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "9100"
+  source_address_prefix       = azurerm_subnet.worker.address_prefix
+  destination_address_prefix  = azurerm_subnet.controller.address_prefix
+}
+
+# Allow apiserver to access kubelet's for exec, log, port-forward
+resource "azurerm_network_security_rule" "controller-kubelet" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-kubelet"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "2030"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "10250"
+
+  # allow Prometheus to scrape kubelet metrics too
+  source_address_prefixes    = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix = azurerm_subnet.controller.address_prefix
+}
+
+# Override Azure AllowVNetInBound and AllowAzureLoadBalancerInBound
+# https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#default-security-rules
+
+resource "azurerm_network_security_rule" "controller-allow-loadblancer" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-loadbalancer"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "3000"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "*"
+  source_port_range           = "*"
+  destination_port_range      = "*"
+  source_address_prefix       = "AzureLoadBalancer"
+  destination_address_prefix  = "*"
+}
+
+resource "azurerm_network_security_rule" "controller-deny-all" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "deny-all"
+  network_security_group_name = azurerm_network_security_group.controller.name
+  priority                    = "3005"
+  access                      = "Deny"
+  direction                   = "Inbound"
+  protocol                    = "*"
+  source_port_range           = "*"
+  destination_port_range      = "*"
+  source_address_prefix       = "*"
+  destination_address_prefix  = "*"
+}
+
+# Worker security group
+
+resource "azurerm_network_security_group" "worker" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name     = "${var.cluster_name}-worker"
+  location = azurerm_resource_group.cluster.location
+}
+
+resource "azurerm_network_security_rule" "worker-icmp" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-icmp"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "1995"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Icmp"
+  source_port_range           = "*"
+  destination_port_range      = "*"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+resource "azurerm_network_security_rule" "worker-ssh" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-ssh"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2000"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "22"
+  source_address_prefix       = azurerm_subnet.controller.address_prefix
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+resource "azurerm_network_security_rule" "worker-http" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-http"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2005"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "80"
+  source_address_prefix       = "*"
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+resource "azurerm_network_security_rule" "worker-https" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-https"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2010"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "443"
+  source_address_prefix       = "*"
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+resource "azurerm_network_security_rule" "worker-cilium-health" {
+  resource_group_name = azurerm_resource_group.cluster.name
+  count               = var.networking == "cilium" ? 1 : 0
+
+  name                        = "allow-cilium-health"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2014"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "4240"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+resource "azurerm_network_security_rule" "worker-vxlan" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-vxlan"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2015"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Udp"
+  source_port_range           = "*"
+  destination_port_range      = "4789"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+resource "azurerm_network_security_rule" "worker-linux-vxlan" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-linux-vxlan"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2016"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Udp"
+  source_port_range           = "*"
+  destination_port_range      = "8472"
+  source_address_prefixes     = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+# Allow Prometheus to scrape node-exporter daemonset
+resource "azurerm_network_security_rule" "worker-node-exporter" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-node-exporter"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2020"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "9100"
+  source_address_prefix       = azurerm_subnet.worker.address_prefix
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+# Allow Prometheus to scrape kube-proxy
+resource "azurerm_network_security_rule" "worker-kube-proxy" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-kube-proxy"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2024"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "10249"
+  source_address_prefix       = azurerm_subnet.worker.address_prefix
+  destination_address_prefix  = azurerm_subnet.worker.address_prefix
+}
+
+# Allow apiserver to access kubelet's for exec, log, port-forward
+resource "azurerm_network_security_rule" "worker-kubelet" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-kubelet"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "2025"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "10250"
+
+  # allow Prometheus to scrape kubelet metrics too
+  source_address_prefixes    = [azurerm_subnet.controller.address_prefix, azurerm_subnet.worker.address_prefix]
+  destination_address_prefix = azurerm_subnet.worker.address_prefix
+}
+
+# Override Azure AllowVNetInBound and AllowAzureLoadBalancerInBound
+# https://docs.microsoft.com/en-us/azure/virtual-network/security-overview#default-security-rules
+
+resource "azurerm_network_security_rule" "worker-allow-loadblancer" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "allow-loadbalancer"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "3000"
+  access                      = "Allow"
+  direction                   = "Inbound"
+  protocol                    = "*"
+  source_port_range           = "*"
+  destination_port_range      = "*"
+  source_address_prefix       = "AzureLoadBalancer"
+  destination_address_prefix  = "*"
+}
+
+resource "azurerm_network_security_rule" "worker-deny-all" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name                        = "deny-all"
+  network_security_group_name = azurerm_network_security_group.worker.name
+  priority                    = "3005"
+  access                      = "Deny"
+  direction                   = "Inbound"
+  protocol                    = "*"
+  source_port_range           = "*"
+  destination_port_range      = "*"
+  source_address_prefix       = "*"
+  destination_address_prefix  = "*"
+}
+

azure/flatcar-linux/kubernetes/ssh.tf

@@ -0,0 +1,59 @@
+locals {
+  # format assets for distribution
+  assets_bundle = [
+    # header with the unpack location
+    for key, value in module.bootstrap.assets_dist :
+    format("##### %s\n%s", key, value)
+  ]
+}
+
+# Secure copy assets to controllers.
+resource "null_resource" "copy-controller-secrets" {
+  count = var.controller_count
+
+  depends_on = [
+    module.bootstrap,
+    azurerm_linux_virtual_machine.controllers
+  ]
+
+  connection {
+    type    = "ssh"
+    host    = azurerm_public_ip.controllers.*.ip_address[count.index]
+    user    = "core"
+    timeout = "15m"
+  }
+
+  provisioner "file" {
+    content     = join("\n", local.assets_bundle)
+    destination = "$HOME/assets"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo /opt/bootstrap/layout",
+    ]
+  }
+}
+
+# Connect to a controller to perform one-time cluster bootstrap.
+resource "null_resource" "bootstrap" {
+  depends_on = [
+    null_resource.copy-controller-secrets,
+    module.workers,
+    azurerm_dns_a_record.apiserver,
+  ]
+
+  connection {
+    type    = "ssh"
+    host    = azurerm_public_ip.controllers.*.ip_address[0]
+    user    = "core"
+    timeout = "15m"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo systemctl start bootstrap",
+    ]
+  }
+}
+

azure/flatcar-linux/kubernetes/variables.tf

@@ -0,0 +1,138 @@
+variable "cluster_name" {
+  type        = string
+  description = "Unique cluster name (prepended to dns_zone)"
+}
+
+# Azure
+
+variable "region" {
+  type        = string
+  description = "Azure Region (e.g. centralus , see `az account list-locations --output table`)"
+}
+
+variable "dns_zone" {
+  type        = string
+  description = "Azure DNS Zone (e.g. azure.example.com)"
+}
+
+variable "dns_zone_group" {
+  type        = string
+  description = "Resource group where the Azure DNS Zone resides (e.g. global)"
+}
+
+# instances
+
+variable "controller_count" {
+  type        = number
+  description = "Number of controllers (i.e. masters)"
+  default     = 1
+}
+
+variable "worker_count" {
+  type        = number
+  description = "Number of workers"
+  default     = 1
+}
+
+variable "controller_type" {
+  type        = string
+  description = "Machine type for controllers (see `az vm list-skus --location centralus`)"
+  default     = "Standard_B2s"
+}
+
+variable "worker_type" {
+  type        = string
+  description = "Machine type for workers (see `az vm list-skus --location centralus`)"
+  default     = "Standard_DS1_v2"
+}
+
+variable "os_image" {
+  type        = string
+  description = "Channel for a Container Linux derivative (flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)"
+  default     = "flatcar-stable"
+}
+
+variable "disk_size" {
+  type        = number
+  description = "Size of the disk in GB"
+  default     = 40
+}
+
+variable "worker_priority" {
+  type        = string
+  description = "Set worker priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time."
+  default     = "Regular"
+}
+
+variable "controller_snippets" {
+  type        = list(string)
+  description = "Controller Container Linux Config snippets"
+  default     = []
+}
+
+variable "worker_snippets" {
+  type        = list(string)
+  description = "Worker Container Linux Config snippets"
+  default     = []
+}
+
+# configuration
+
+variable "ssh_authorized_key" {
+  type        = string
+  description = "SSH public key for user 'core'"
+}
+
+variable "networking" {
+  type        = string
+  description = "Choice of networking provider (flannel or calico)"
+  default     = "calico"
+}
+
+variable "host_cidr" {
+  type        = string
+  description = "CIDR IPv4 range to assign to instances"
+  default     = "10.0.0.0/16"
+}
+
+variable "pod_cidr" {
+  type        = string
+  description = "CIDR IPv4 range to assign Kubernetes pods"
+  default     = "10.2.0.0/16"
+}
+
+variable "service_cidr" {
+  type        = string
+  description = <<EOD
+CIDR IPv4 range to assign Kubernetes services.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
+EOD
+  default     = "10.3.0.0/16"
+}
+
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable usage or analytics reporting to upstreams (Calico)"
+  default     = false
+}
+
+variable "enable_aggregation" {
+  type        = bool
+  description = "Enable the Kubernetes Aggregation Layer (defaults to false)"
+  default     = false
+}
+
+variable "worker_node_labels" {
+  type        = list(string)
+  description = "List of initial worker node labels"
+  default     = []
+}
+
+# unofficial, undocumented, unsupported
+
+variable "cluster_domain_suffix" {
+  type        = string
+  description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  default     = "cluster.local"
+}
+

azure/flatcar-linux/kubernetes/versions.tf

@@ -0,0 +1,16 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.12.26, < 0.14.0"
+  required_providers {
+    azurerm  = "~> 2.8"
+    template = "~> 2.1"
+    null     = "~> 2.1"
+
+    ct = {
+      source  = "poseidon/ct"
+      version = "~> 0.6.1"
+    }
+  }
+}
+

azure/flatcar-linux/kubernetes/workers.tf

@@ -0,0 +1,24 @@
+module "workers" {
+  source = "./workers"
+  name   = var.cluster_name
+
+  # Azure
+  resource_group_name     = azurerm_resource_group.cluster.name
+  region                  = azurerm_resource_group.cluster.location
+  subnet_id               = azurerm_subnet.worker.id
+  security_group_id       = azurerm_network_security_group.worker.id
+  backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
+
+  worker_count = var.worker_count
+  vm_type      = var.worker_type
+  os_image     = var.os_image
+  priority     = var.worker_priority
+
+  # configuration
+  kubeconfig            = module.bootstrap.kubeconfig-kubelet
+  ssh_authorized_key    = var.ssh_authorized_key
+  service_cidr          = var.service_cidr
+  cluster_domain_suffix = var.cluster_domain_suffix
+  snippets              = var.worker_snippets
+  node_labels           = var.worker_node_labels
+}

azure/flatcar-linux/kubernetes/workers/cl/worker.yaml

@@ -0,0 +1,117 @@
+---
+systemd:
+  units:
+    - name: docker.service
+      enabled: true
+    - name: locksmithd.service
+      mask: true
+    - name: wait-for-dns.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Wait for DNS entries
+        Wants=systemd-resolved.service
+        Before=kubelet.service
+        [Service]
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
+        [Install]
+        RequiredBy=kubelet.service
+    - name: kubelet.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubelet
+        Requires=docker.service
+        After=docker.service
+        Wants=rpc-statd.service
+        [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
+        Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver}
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+        ExecStartPre=/bin/mkdir -p /opt/cni/bin
+        ExecStartPre=/bin/mkdir -p /var/lib/calico
+        ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        # Podman, rkt, or runc run container processes, whereas docker run
+        # is a client to a daemon and requires workarounds to use within a
+        # systemd unit. https://github.com/moby/moby/issues/6791
+        ExecStartPre=/usr/bin/docker run -d \
+          --name kubelet \
+          --privileged \
+          --pid host \
+          --network host \
+          -v /etc/kubernetes:/etc/kubernetes:ro \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /usr/lib/os-release:/etc/os-release:ro \
+          -v /lib/modules:/lib/modules:ro \
+          -v /run:/run \
+          -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+          -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log:/var/log \
+          -v /opt/cni/bin:/opt/cni/bin \
+          $${KUBELET_IMAGE} \
+          --anonymous-auth=false \
+          --authentication-token-webhook \
+          --authorization-mode=Webhook \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \
+          --client-ca-file=/etc/kubernetes/ca.crt \
+          --cluster_dns=${cluster_dns_service_ip} \
+          --cluster_domain=${cluster_domain_suffix} \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --healthz-port=0 \
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --network-plugin=cni \
+          --node-labels=node.kubernetes.io/node \
+          %{~ for label in split(",", node_labels) ~}
+          --node-labels=${label} \
+          %{~ endfor ~}
+          --pod-manifest-path=/etc/kubernetes/manifests \
+          --read-only-port=0 \
+          --rotate-certificates \
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+        ExecStart=docker logs -f kubelet
+        ExecStop=docker stop kubelet
+        ExecStopPost=docker rm kubelet
+        Restart=always
+        RestartSec=5
+        [Install]
+        WantedBy=multi-user.target
+    - name: delete-node.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Delete Kubernetes node on shutdown
+        [Service]
+        Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
+        Type=oneshot
+        RemainAfterExit=true
+        ExecStart=/bin/true
+        ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
+        [Install]
+        WantedBy=multi-user.target
+storage:
+  files:
+    - path: /etc/kubernetes/kubeconfig
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          ${kubeconfig}
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - "${ssh_authorized_key}"

azure/flatcar-linux/kubernetes/workers/variables.tf

@@ -0,0 +1,99 @@
+variable "name" {
+  type        = string
+  description = "Unique name for the worker pool"
+}
+
+# Azure
+
+variable "region" {
+  type        = string
+  description = "Must be set to the Azure Region of cluster"
+}
+
+variable "resource_group_name" {
+  type        = string
+  description = "Must be set to the resource group name of cluster"
+}
+
+variable "subnet_id" {
+  type        = string
+  description = "Must be set to the `worker_subnet_id` output by cluster"
+}
+
+variable "security_group_id" {
+  type        = string
+  description = "Must be set to the `worker_security_group_id` output by cluster"
+}
+
+variable "backend_address_pool_id" {
+  type        = string
+  description = "Must be set to the `worker_backend_address_pool_id` output by cluster"
+}
+
+# instances
+
+variable "worker_count" {
+  type        = number
+  description = "Number of instances"
+  default     = 1
+}
+
+variable "vm_type" {
+  type        = string
+  description = "Machine type for instances (see `az vm list-skus --location centralus`)"
+  default     = "Standard_DS1_v2"
+}
+
+variable "os_image" {
+  type        = string
+  description = "Channel for a Container Linux derivative (flatcar-stable, flatcar-beta, flatcar-alpha, flatcar-edge)"
+  default     = "flatcar-stable"
+}
+
+variable "priority" {
+  type        = string
+  description = "Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be evicted at any time."
+  default     = "Regular"
+}
+
+variable "snippets" {
+  type        = list(string)
+  description = "Container Linux Config snippets"
+  default     = []
+}
+
+# configuration
+
+variable "kubeconfig" {
+  type        = string
+  description = "Must be set to `kubeconfig` output by cluster"
+}
+
+variable "ssh_authorized_key" {
+  type        = string
+  description = "SSH public key for user 'core'"
+}
+
+variable "service_cidr" {
+  type        = string
+  description = <<EOD
+CIDR IPv4 range to assign Kubernetes services.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
+EOD
+  default     = "10.3.0.0/16"
+}
+
+variable "node_labels" {
+  type        = list(string)
+  description = "List of initial node labels"
+  default     = []
+}
+
+# unofficial, undocumented, unsupported
+
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = string
+  default     = "cluster.local"
+}
+

azure/flatcar-linux/kubernetes/workers/versions.tf

@@ -0,0 +1,14 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.12.26, < 0.14.0"
+  required_providers {
+    azurerm  = "~> 2.8"
+    template = "~> 2.1"
+
+    ct = {
+      source  = "poseidon/ct"
+      version = "~> 0.6.1"
+    }
+  }
+}

azure/flatcar-linux/kubernetes/workers/workers.tf

@@ -0,0 +1,111 @@
+locals {
+  # flatcar-stable -> Flatcar Linux Stable
+  channel = split("-", var.os_image)[1]
+}
+
+# Workers scale set
+resource "azurerm_linux_virtual_machine_scale_set" "workers" {
+  resource_group_name = var.resource_group_name
+
+  name      = "${var.name}-worker"
+  location  = var.region
+  sku       = var.vm_type
+  instances = var.worker_count
+  # instance name prefix for instances in the set
+  computer_name_prefix   = "${var.name}-worker"
+  single_placement_group = false
+  custom_data            = base64encode(data.ct_config.worker-ignition.rendered)
+
+  # storage
+  os_disk {
+    storage_account_type = "Standard_LRS"
+    caching              = "ReadWrite"
+  }
+
+  # Flatcar Container Linux
+  source_image_reference {
+    publisher = "Kinvolk"
+    offer     = "flatcar-container-linux-free"
+    sku       = local.channel
+    version   = "latest"
+  }
+
+  plan {
+    name      = local.channel
+    publisher = "kinvolk"
+    product   = "flatcar-container-linux-free"
+  }
+
+  # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too
+  admin_username = "core"
+  admin_ssh_key {
+    username   = "core"
+    public_key = var.ssh_authorized_key
+  }
+
+  # network
+  network_interface {
+    name                      = "nic0"
+    primary                   = true
+    network_security_group_id = var.security_group_id
+
+    ip_configuration {
+      name      = "ip0"
+      primary   = true
+      subnet_id = var.subnet_id
+
+      # backend address pool to which the NIC should be added
+      load_balancer_backend_address_pool_ids = [var.backend_address_pool_id]
+    }
+  }
+
+  # lifecycle
+  upgrade_mode = "Manual"
+  # eviction policy may only be set when priority is Spot
+  priority        = var.priority
+  eviction_policy = var.priority == "Spot" ? "Delete" : null
+}
+
+# Scale up or down to maintain desired number, tolerating deallocations.
+resource "azurerm_monitor_autoscale_setting" "workers" {
+  resource_group_name = var.resource_group_name
+
+  name     = "${var.name}-maintain-desired"
+  location = var.region
+
+  # autoscale
+  enabled            = true
+  target_resource_id = azurerm_linux_virtual_machine_scale_set.workers.id
+
+  profile {
+    name = "default"
+
+    capacity {
+      minimum = var.worker_count
+      default = var.worker_count
+      maximum = var.worker_count
+    }
+  }
+}
+
+# Worker Ignition configs
+data "ct_config" "worker-ignition" {
+  content  = data.template_file.worker-config.rendered
+  strict   = true
+  snippets = var.snippets
+}
+
+# Worker Container Linux configs
+data "template_file" "worker-config" {
+  template = file("${path.module}/cl/worker.yaml")
+
+  vars = {
+    kubeconfig             = indent(10, var.kubeconfig)
+    ssh_authorized_key     = var.ssh_authorized_key
+    cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+    cluster_domain_suffix  = var.cluster_domain_suffix
+    cgroup_driver          = local.channel == "edge" ? "systemd" : "cgroupfs"
+    node_labels            = join(",", var.node_labels)
+  }
+}
+