Migrate Flatcar Linux from Ignition spec v2.3.0 to v3.3.0

* Requires poseidon v0.11+ and Flatcar Linux 3185.0.0+ (action required)
* Previously, Flatcar Linux configs have been parsed as Container
Linux Configs to Ignition v2.2.0 specs by poseidon/ct
* Flatcar Linux starting in 3185.0.0 now supports Ignition v3.x specs
(which are rendered from Butane Configs, like Fedora CoreOS)
* poseidon/ct v0.11.0 adds support for the flatcar Butane Config
variant so that Flatcar Linux can use Ignition v3.x

Rel:

* [Flatcar Support](https://flatcar-linux.org/docs/latest/provisioning/ignition/specification/#ignition-v3)
* [poseidon/ct support](https://github.com/poseidon/terraform-provider-ct/pull/131)

CHANGES.md

@@ -7,6 +7,20 @@ Notable changes between versions.
 * Update Calico from v3.23.1 to [v3.23.3](https://github.com/projectcalico/calico/releases/tag/v3.23.3)
 * Remove use of deprecated Terraform [template](https://registry.terraform.io/providers/hashicorp/template) provider ([#1194](https://github.com/poseidon/typhoon/pull/1194))
 
+### Flatcar Linux
+
+* Migrate Flatcar Linux from Ignition spec v2.3.0 to v3.3.0 (**action required**)
+  * Flatcar Linux 3185.0.0+ [supports](https://flatcar-linux.org/docs/latest/provisioning/ignition/specification/#ignition-v3) Ignition v3.x specs (which are rendered from Butane Configs, like Fedora CoreOS)
+  * `poseidon/ct` v0.11.0 [supports](https://github.com/poseidon/terraform-provider-ct/pull/131) the `flatcar` Butane Config variant
+  * Require poseidon v0.11+ and Flatcar Linux 3185.0.0+
+* Modify any Flatcar Linux snippets to use the [Butane Config](https://coreos.github.io/butane/config-flatcar-v1_0/) format (**action required**):
+
+```tf
+variant: flatcar
+version: 1.0.0
+...
+```
+
 ### Google
 
 * Fix bug provisioning clusters with multiple controller nodes ([#1195](https://github.com/poseidon/typhoon/pull/1195))

aws/flatcar-linux/kubernetes/butane/controller.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: etcd-member.service
@@ -134,18 +135,15 @@ systemd:
 storage:
   directories:
     - path: /var/lib/etcd
-      filesystem: root
       mode: 0700
       overwrite: true
   files:
     - path: /etc/kubernetes/kubeconfig
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           ${kubeconfig}
     - path: /opt/bootstrap/layout
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -168,7 +166,6 @@ storage:
           mv manifests-networking/* /opt/bootstrap/assets/manifests/
           rm -rf assets auth static-manifests tls manifests-networking
     - path: /opt/bootstrap/apply
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -183,13 +180,11 @@ storage:
              sleep 5
           done
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
     - path: /etc/etcd/etcd.env
-      filesystem: root
       mode: 0644
       contents:
           inline: |

aws/flatcar-linux/kubernetes/controllers.tf

@@ -50,7 +50,7 @@ resource "aws_instance" "controllers" {
 # Flatcar Linux controllers
 data "ct_config" "controllers" {
   count = var.controller_count
-  content = templatefile("${path.module}/cl/controller.yaml", {
+  content = templatefile("${path.module}/butane/controller.yaml", {
     # Cannot use cyclic dependencies on controllers or their DNS records
     etcd_name   = "etcd${count.index}"
     etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"

aws/flatcar-linux/kubernetes/versions.tf

@@ -7,7 +7,7 @@ terraform {
     null = ">= 2.1"
     ct = {
       source  = "poseidon/ct"
-      version = "~> 0.9"
+      version = "~> 0.11"
     }
   }
 }

aws/flatcar-linux/kubernetes/workers/butane/worker.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: docker.service
@@ -106,13 +107,11 @@ systemd:
 storage:
   files:
     - path: /etc/kubernetes/kubeconfig
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           ${kubeconfig}
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |

aws/flatcar-linux/kubernetes/workers/versions.tf

@@ -6,7 +6,7 @@ terraform {
     aws = ">= 2.23, <= 5.0"
     ct = {
       source  = "poseidon/ct"
-      version = "~> 0.9"
+      version = "~> 0.11"
     }
   }
 }

aws/flatcar-linux/kubernetes/workers/workers.tf

@@ -69,7 +69,7 @@ resource "aws_launch_configuration" "worker" {
 
 # Flatcar Linux worker
 data "ct_config" "worker" {
-  content = templatefile("${path.module}/cl/worker.yaml", {
+  content = templatefile("${path.module}/butane/worker.yaml", {
     kubeconfig             = indent(10, var.kubeconfig)
     ssh_authorized_key     = var.ssh_authorized_key
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)

azure/flatcar-linux/kubernetes/butane/controller.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: etcd-member.service
@@ -94,9 +95,9 @@ systemd:
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
           --pod-manifest-path=/etc/kubernetes/manifests \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --read-only-port=0 \
           --resolv-conf=/run/systemd/resolve/resolv.conf \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
@@ -130,18 +131,15 @@ systemd:
 storage:
   directories:
     - path: /var/lib/etcd
-      filesystem: root
       mode: 0700
       overwrite: true
   files:
     - path: /etc/kubernetes/kubeconfig
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           ${kubeconfig}
     - path: /opt/bootstrap/layout
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -164,7 +162,6 @@ storage:
           mv manifests-networking/* /opt/bootstrap/assets/manifests/
           rm -rf assets auth static-manifests tls manifests-networking
     - path: /opt/bootstrap/apply
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -179,13 +176,11 @@ storage:
              sleep 5
           done
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
     - path: /etc/etcd/etcd.env
-      filesystem: root
       mode: 0644
       contents:
           inline: |

azure/flatcar-linux/kubernetes/controllers.tf

@@ -133,7 +133,7 @@ resource "azurerm_network_interface_backend_address_pool_association" "controlle
 # Flatcar Linux controllers
 data "ct_config" "controllers" {
   count = var.controller_count
-  content = templatefile("${path.module}/cl/controller.yaml", {
+  content = templatefile("${path.module}/butane/controller.yaml", {
     # Cannot use cyclic dependencies on controllers or their DNS records
     etcd_name   = "etcd${count.index}"
     etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"

azure/flatcar-linux/kubernetes/versions.tf

@@ -7,7 +7,7 @@ terraform {
     null    = ">= 2.1"
     ct = {
       source  = "poseidon/ct"
-      version = "~> 0.9"
+      version = "~> 0.11"
     }
   }
 }

azure/flatcar-linux/kubernetes/workers/butane/worker.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: docker.service
@@ -102,13 +103,11 @@ systemd:
 storage:
   files:
     - path: /etc/kubernetes/kubeconfig
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           ${kubeconfig}
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |

azure/flatcar-linux/kubernetes/workers/versions.tf

@@ -6,7 +6,7 @@ terraform {
     azurerm = ">= 2.8, < 4.0"
     ct = {
       source  = "poseidon/ct"
-      version = "~> 0.9"
+      version = "~> 0.11"
     }
   }
 }

azure/flatcar-linux/kubernetes/workers/workers.tf

@@ -90,7 +90,7 @@ resource "azurerm_monitor_autoscale_setting" "workers" {
 
 # Flatcar Linux worker
 data "ct_config" "worker" {
-  content = templatefile("${path.module}/cl/worker.yaml", {
+  content = templatefile("${path.module}/butane/worker.yaml", {
     kubeconfig             = indent(10, var.kubeconfig)
     ssh_authorized_key     = var.ssh_authorized_key
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)

bare-metal/fedora-coreos/kubernetes/profiles.tf

@@ -43,7 +43,7 @@ resource "matchbox_profile" "controllers" {
 
 # Fedora CoreOS controllers
 data "ct_config" "controllers" {
-  count = var.controller_count
+  count = length(var.controllers)
   content = templatefile("${path.module}/fcc/controller.yaml", {
     domain_name            = var.controllers.*.domain[count.index]
     etcd_name              = var.controllers.*.name[count.index]

bare-metal/flatcar-linux/kubernetes/butane/controller.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: etcd-member.service
@@ -139,21 +140,17 @@ systemd:
 storage:
   directories:
     - path: /var/lib/etcd
-      filesystem: root
       mode: 0700
       overwrite: true
     - path: /etc/kubernetes
-      filesystem: root
       mode: 0755
   files:
     - path: /etc/hostname
-      filesystem: root
       mode: 0644
       contents:
         inline:
           ${domain_name}
     - path: /opt/bootstrap/layout
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -176,7 +173,6 @@ storage:
           mv manifests-networking/* /opt/bootstrap/assets/manifests/
           rm -rf assets auth static-manifests tls manifests-networking
     - path: /opt/bootstrap/apply
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -191,13 +187,11 @@ storage:
              sleep 5
           done
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
     - path: /etc/etcd/etcd.env
-      filesystem: root
       mode: 0644
       contents:
           inline: |

bare-metal/flatcar-linux/kubernetes/butane/install.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: installer.service
@@ -25,12 +26,11 @@ systemd:
 storage:
   files:
     - path: /opt/installer
-      filesystem: root
       mode: 0500
       contents:
         inline: |
           #!/bin/bash -ex
-          curl --retry 10 "${ignition_endpoint}?{{.request.raw_query}}&os=installed" -o ignition.json
+          curl --retry 10 "${ignition_endpoint}?mac=${mac}&os=installed" -o ignition.json
           flatcar-install \
             -d ${install_disk} \
             -C ${os_channel} \

bare-metal/flatcar-linux/kubernetes/butane/worker.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: docker.service
@@ -99,17 +100,14 @@ systemd:
 storage:
   directories:
     - path: /etc/kubernetes
-      filesystem: root
       mode: 0755
   files:
     - path: /etc/hostname
-      filesystem: root
       mode: 0644
       contents:
         inline:
           ${domain_name}
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |

bare-metal/flatcar-linux/kubernetes/profiles.tf

@@ -21,7 +21,7 @@ resource "matchbox_profile" "flatcar-install" {
     var.kernel_args,
   ])
 
-  container_linux_config = data.template_file.install-configs.*.rendered[count.index]
+  raw_ignition = data.ct_config.install.*.rendered[count.index]
 }
 
 // Flatcar Linux Install profile (from matchbox /assets cache)
@@ -43,41 +43,40 @@ resource "matchbox_profile" "cached-flatcar-install" {
     var.kernel_args,
   ])
 
-  container_linux_config = data.template_file.cached-install-configs.*.rendered[count.index]
+  raw_ignition = data.ct_config.cached-install.*.rendered[count.index]
 }
 
-data "template_file" "install-configs" {
+# Flatcar Linux install
+data "ct_config" "install" {
   count = length(var.controllers) + length(var.workers)
-
+  content = templatefile("${path.module}/butane/install.yaml", {
-  template = file("${path.module}/cl/install.yaml")
-
-  vars = {
     os_channel         = local.channel
     os_version         = var.os_version
     ignition_endpoint  = format("%s/ignition", var.matchbox_http_endpoint)
+    mac                = concat(var.controllers.*.mac, var.workers.*.mac)[count.index]
     install_disk       = var.install_disk
     ssh_authorized_key = var.ssh_authorized_key
     # only cached profile adds -b baseurl
     baseurl_flag = ""
-  }
+  })
+  strict = true
 }
 
-data "template_file" "cached-install-configs" {
+# Flatcar Linux cached install
+data "ct_config" "cached-install" {
   count = length(var.controllers) + length(var.workers)
-
+  content = templatefile("${path.module}/butane/install.yaml", {
-  template = file("${path.module}/cl/install.yaml")
-
-  vars = {
     os_channel         = local.channel
     os_version         = var.os_version
     ignition_endpoint  = format("%s/ignition", var.matchbox_http_endpoint)
+    mac                = concat(var.controllers.*.mac, var.workers.*.mac)[count.index]
     install_disk       = var.install_disk
     ssh_authorized_key = var.ssh_authorized_key
     # profile uses -b baseurl to install from matchbox cache
     baseurl_flag = "-b ${var.matchbox_http_endpoint}/assets/flatcar"
+  })
+  strict = true
 }
-}
-
 
 // Kubernetes Controller profiles
 resource "matchbox_profile" "controllers" {
@@ -88,8 +87,8 @@ resource "matchbox_profile" "controllers" {
 
 # Flatcar Linux controllers
 data "ct_config" "controllers" {
-  count = var.controller_count
+  count = length(var.controllers)
-  content = templatefile("${path.module}/cl/controller.yaml", {
+  content = templatefile("${path.module}/butane/controller.yaml", {
     domain_name            = var.controllers.*.domain[count.index]
     etcd_name              = var.controllers.*.name[count.index]
     etcd_initial_cluster   = join(",", formatlist("%s=https://%s:2380", var.controllers.*.name, var.controllers.*.domain))
@@ -111,7 +110,7 @@ resource "matchbox_profile" "workers" {
 # Flatcar Linux workers
 data "ct_config" "workers" {
   count = length(var.workers)
-  content = templatefile("${path.module}/cl/worker.yaml", {
+  content = templatefile("${path.module}/butane/worker.yaml", {
     domain_name            = var.workers.*.domain[count.index]
     cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip
     cluster_domain_suffix  = var.cluster_domain_suffix

digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: etcd-member.service
@@ -142,15 +143,12 @@ systemd:
 storage:
   directories:
     - path: /var/lib/etcd
-      filesystem: root
       mode: 0700
       overwrite: true
     - path: /etc/kubernetes
-      filesystem: root
       mode: 0755
   files:
     - path: /opt/bootstrap/layout
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -173,7 +171,6 @@ storage:
           mv manifests-networking/* /opt/bootstrap/assets/manifests/
           rm -rf assets auth static-manifests tls manifests-networking
     - path: /opt/bootstrap/apply
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -188,13 +185,11 @@ storage:
              sleep 5
           done
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
     - path: /etc/etcd/etcd.env
-      filesystem: root
       mode: 0644
       contents:
           inline: |

digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: docker.service
@@ -108,11 +109,9 @@ systemd:
 storage:
   directories:
     - path: /etc/kubernetes
-      filesystem: root
       mode: 0755
   files:
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |

digital-ocean/flatcar-linux/kubernetes/controllers.tf

@@ -70,7 +70,7 @@ resource "digitalocean_tag" "controllers" {
 # Flatcar Linux controllers
 data "ct_config" "controllers" {
   count = var.controller_count
-  content = templatefile("${path.module}/cl/controller.yaml", {
+  content = templatefile("${path.module}/butane/controller.yaml", {
     # Cannot use cyclic dependencies on controllers or their DNS records
     etcd_name   = "etcd${count.index}"
     etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"

digital-ocean/flatcar-linux/kubernetes/versions.tf

@@ -6,7 +6,7 @@ terraform {
     null = ">= 2.1"
     ct = {
       source  = "poseidon/ct"
-      version = "~> 0.9"
+      version = "~> 0.11"
     }
     digitalocean = {
       source  = "digitalocean/digitalocean"

digital-ocean/flatcar-linux/kubernetes/workers.tf

@@ -58,7 +58,7 @@ resource "digitalocean_tag" "workers" {
 
 # Flatcar Linux worker
 data "ct_config" "worker" {
-  content = templatefile("${path.module}/cl/worker.yaml", {
+  content = templatefile("${path.module}/butane/worker.yaml", {
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
   })

google-cloud/flatcar-linux/kubernetes/butane/controller.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: etcd-member.service
@@ -94,9 +95,9 @@ systemd:
           --kubeconfig=/var/lib/kubelet/kubeconfig \
           --node-labels=node.kubernetes.io/controller="true" \
           --pod-manifest-path=/etc/kubernetes/manifests \
+          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --read-only-port=0 \
           --resolv-conf=/run/systemd/resolve/resolv.conf \
-          --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
           --rotate-certificates \
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins
         ExecStart=docker logs -f kubelet
@@ -130,18 +131,15 @@ systemd:
 storage:
   directories:
     - path: /var/lib/etcd
-      filesystem: root
       mode: 0700
       overwrite: true
   files:
     - path: /etc/kubernetes/kubeconfig
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           ${kubeconfig}
     - path: /opt/bootstrap/layout
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -164,7 +162,6 @@ storage:
           mv manifests-networking/* /opt/bootstrap/assets/manifests/
           rm -rf assets auth static-manifests tls manifests-networking
     - path: /opt/bootstrap/apply
-      filesystem: root
       mode: 0544
       contents:
         inline: |
@@ -179,13 +176,11 @@ storage:
              sleep 5
           done
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
     - path: /etc/etcd/etcd.env
-      filesystem: root
       mode: 0644
       contents:
           inline: |

google-cloud/flatcar-linux/kubernetes/controllers.tf

@@ -69,7 +69,7 @@ resource "google_compute_instance" "controllers" {
 # Flatcar Linux controllers
 data "ct_config" "controllers" {
   count = var.controller_count
-  content = templatefile("${path.module}/cl/controller.yaml", {
+  content = templatefile("${path.module}/butane/controller.yaml", {
     # Cannot use cyclic dependencies on controllers or their DNS records
     etcd_name   = "etcd${count.index}"
     etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"

google-cloud/flatcar-linux/kubernetes/versions.tf

@@ -7,7 +7,7 @@ terraform {
     null   = ">= 2.1"
     ct = {
       source  = "poseidon/ct"
-      version = "~> 0.9"
+      version = "~> 0.11"
     }
   }
 }

google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml

@@ -1,4 +1,5 @@
----
+variant: flatcar
+version: 1.0.0
 systemd:
   units:
     - name: docker.service
@@ -102,13 +103,11 @@ systemd:
 storage:
   files:
     - path: /etc/kubernetes/kubeconfig
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           ${kubeconfig}
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       mode: 0644
       contents:
         inline: |

google-cloud/flatcar-linux/kubernetes/workers/versions.tf

@@ -6,7 +6,7 @@ terraform {
     google = ">= 2.19, < 5.0"
     ct = {
       source  = "poseidon/ct"
-      version = "~> 0.9"
+      version = "~> 0.11"
     }
   }
 }

google-cloud/flatcar-linux/kubernetes/workers/workers.tf

@@ -71,7 +71,7 @@ resource "google_compute_instance_template" "worker" {
 
 # Flatcar Linux worker
 data "ct_config" "worker" {
-  content = templatefile("${path.module}/cl/worker.yaml", {
+  content = templatefile("${path.module}/butane/worker.yaml", {
     kubeconfig             = indent(10, var.kubeconfig)
     ssh_authorized_key     = var.ssh_authorized_key
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)