Add experimental Fedora CoreOS arm64 support on AWS

* Add experimental `arch` variable to Fedora CoreOS AWS,
accepting amd64 (default) or arm64 to support native
arm64/aarch64 clusters or mixed/hybrid clusters with
a worker pool of arm64 workers
* Add `daemonset_tolerations` variable to cluster module
(experimental)
* Add `node_taints` variable to workers module
* Requires flannel CNI and experimental Poseidon-built
arm64 Fedora CoreOS AMIs (published to us-east-1, us-east-2,
and us-west-1)

WARN:

* Our AMIs are experimental, may be removed at any time, and
will be removed when Fedora CoreOS publishes official arm64
AMIs. Do NOT use in production

Related:

* https://github.com/poseidon/typhoon/pull/682

aws/fedora-coreos/kubernetes/ami.tf

@@ -18,3 +18,27 @@ data "aws_ami" "fedora-coreos" {
     values = ["Fedora CoreOS ${var.os_stream} *"]
   }
 }
+
+# Experimental Fedora CoreOS arm64 / aarch64 AMIs from Poseidon
+# WARNING: These AMIs will be removed when Fedora CoreOS publishes arm64 AMIs
+# and may be removed for any reason before then as well. Do not use.
+data "aws_ami" "fedora-coreos-arm" {
+  most_recent = true
+  owners      = ["099663496933"]
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "name"
+    values = ["fedora-coreos-*"]
+  }
+}
+

aws/fedora-coreos/kubernetes/bootstrap.tf

@@ -12,6 +12,7 @@ module "bootstrap" {
   cluster_domain_suffix = var.cluster_domain_suffix
   enable_reporting      = var.enable_reporting
   enable_aggregation    = var.enable_aggregation
+  daemonset_tolerations = var.daemonset_tolerations
 
   trusted_certs_dir = "/etc/pki/tls/certs"
 }

aws/fedora-coreos/kubernetes/controllers.tf

@@ -22,9 +22,8 @@ resource "aws_instance" "controllers" {
   }
 
   instance_type = var.controller_type
-
-  ami       = data.aws_ami.fedora-coreos.image_id
-  user_data = data.ct_config.controller-ignitions.*.rendered[count.index]
+  ami           = var.arch == "arm64" ? data.aws_ami.fedora-coreos-arm.image_id : data.aws_ami.fedora-coreos.image_id
+  user_data     = data.ct_config.controller-ignitions.*.rendered[count.index]
 
   # storage
   root_block_device {
@@ -63,6 +62,7 @@ data "template_file" "controller-configs" {
 
   vars = {
     # Cannot use cyclic dependencies on controllers or their DNS records
+    etcd_arch   = var.arch == "arm64" ? "-arm64" : ""
     etcd_name   = "etcd${count.index}"
     etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
     # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...

aws/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -12,7 +12,7 @@ systemd:
         Wants=network-online.target network.target
         After=network-online.target
         [Service]
-        Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12
+        Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.4.12${etcd_arch}
         Type=exec
         ExecStartPre=/bin/mkdir -p /var/lib/etcd
         ExecStartPre=-/usr/bin/podman rm etcd
@@ -214,6 +214,7 @@ storage:
           ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
           ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
           ETCD_PEER_CLIENT_CERT_AUTH=true
+          ETCD_UNSUPPORTED_ARCH=arm64
 passwd:
   users:
     - name: core

aws/fedora-coreos/kubernetes/variables.tf

@@ -155,3 +155,15 @@ variable "cluster_domain_suffix" {
   default     = "cluster.local"
 }
 
+variable "arch" {
+  type        = string
+  description = "Container architecture (amd64 or arm64)"
+  default     = "amd64"
+}
+
+variable "daemonset_tolerations" {
+  type        = list(string)
+  description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
+  default     = []
+}
+

aws/fedora-coreos/kubernetes/workers.tf

@@ -9,6 +9,7 @@ module "workers" {
   worker_count    = var.worker_count
   instance_type   = var.worker_type
   os_stream       = var.os_stream
+  arch            = var.arch
   disk_size       = var.disk_size
   spot_price      = var.worker_price
   target_groups   = var.worker_target_groups

aws/fedora-coreos/kubernetes/workers/ami.tf

@@ -18,3 +18,27 @@ data "aws_ami" "fedora-coreos" {
     values = ["Fedora CoreOS ${var.os_stream} *"]
   }
 }
+
+# Experimental Fedora CoreOS arm64 / aarch64 AMIs from Poseidon
+# WARNING: These AMIs will be removed when Fedora CoreOS publishes arm64 AMIs
+# and may be removed for any reason before then as well. Do not use.
+data "aws_ami" "fedora-coreos-arm" {
+  most_recent = true
+  owners      = ["099663496933"]
+
+  filter {
+    name   = "architecture"
+    values = ["arm64"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name   = "name"
+    values = ["fedora-coreos-*"]
+  }
+}
+

aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml

@@ -68,6 +68,9 @@ systemd:
           %{~ for label in split(",", node_labels) ~}
           --node-labels=${label} \
           %{~ endfor ~}
+          %{~ for taint in split(",", node_taints) ~}
+          --register-with-taints=${taint} \
+          %{~ endfor ~}
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --rotate-certificates \

aws/fedora-coreos/kubernetes/workers/variables.tf

@@ -108,3 +108,17 @@ variable "node_labels" {
   description = "List of initial node labels"
   default     = []
 }
+
+variable "node_taints" {
+  type        = list(string)
+  description = "List of initial node taints"
+  default     = []
+}
+
+# unofficial, undocumented, unsupported
+
+variable "arch" {
+  type        = string
+  description = "Container architecture (amd64 or arm64)"
+  default     = "amd64"
+}

aws/fedora-coreos/kubernetes/workers/workers.tf

@@ -44,7 +44,7 @@ resource "aws_autoscaling_group" "workers" {
 
 # Worker template
 resource "aws_launch_configuration" "worker" {
-  image_id          = data.aws_ami.fedora-coreos.image_id
+  image_id          = var.arch == "arm64" ? data.aws_ami.fedora-coreos-arm.image_id : data.aws_ami.fedora-coreos.image_id
   instance_type     = var.instance_type
   spot_price        = var.spot_price > 0 ? var.spot_price : null
   enable_monitoring = false
@@ -86,6 +86,7 @@ data "template_file" "worker-config" {
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
     node_labels            = join(",", var.node_labels)
+    node_taints            = join(",", var.node_taints)
   }
 }