Start prototype of Fedora CoreOS on bare-metal

* Use terraform-provider-ct v0.4.0 with Fedora CoreOS Config
support (not yet released)

bare-metal/fedora-coreos/kubernetes/bootkube.tf

@@ -0,0 +1,19 @@
+# Self-hosted Kubernetes assets (kubeconfig, manifests)
+module "bootkube" {
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6"
+
+  cluster_name                    = var.cluster_name
+  api_servers                     = [var.k8s_domain_name]
+  etcd_servers                    = var.controller_domains
+  asset_dir                       = var.asset_dir
+  networking                      = var.networking
+  network_mtu                     = var.network_mtu
+  network_ip_autodetection_method = var.network_ip_autodetection_method
+  pod_cidr                        = var.pod_cidr
+  service_cidr                    = var.service_cidr
+  cluster_domain_suffix           = var.cluster_domain_suffix
+  enable_reporting                = var.enable_reporting
+  enable_aggregation              = var.enable_aggregation
+}
+
+

bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -0,0 +1,28 @@
+---
+variant: fcos
+version: 1.0.0
+storage:
+  files:
+    - path: /etc/kubernetes/kubelet.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
+          KUBELET_IMAGE_TAG=v1.15.0
+    - path: /etc/hostname
+      filesystem: root
+      mode: 0644
+      contents:
+        inline:
+          ${domain_name}
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - ${ssh_authorized_key}

bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml

@@ -0,0 +1,29 @@
+---
+variant: fcos
+version: 1.0.0
+storage:
+  files:
+    - path: /etc/kubernetes/kubelet.env
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
+          KUBELET_IMAGE_TAG=v1.15.0
+    - path: /etc/hostname
+      filesystem: root
+      mode: 0644
+      contents:
+        inline:
+          ${domain_name}
+    - path: /etc/sysctl.d/max-user-watches.conf
+      filesystem: root
+      contents:
+        inline: |
+          fs.inotify.max_user_watches=16184
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - ${ssh_authorized_key}
+

bare-metal/fedora-coreos/kubernetes/groups.tf

@@ -0,0 +1,22 @@
+# Match each controller or worker to a profile
+
+resource "matchbox_group" "controller" {
+  count   = length(var.controller_names)
+  name    = format("%s-%s", var.cluster_name, var.controller_names[count.index])
+  profile = matchbox_profile.controllers.*.name[count.index]
+
+  selector = {
+    mac = var.controller_macs[count.index]
+  }
+}
+
+resource "matchbox_group" "worker" {
+  count   = length(var.worker_names)
+  name    = format("%s-%s", var.cluster_name, var.worker_names[count.index])
+  profile = matchbox_profile.workers.*.name[count.index]
+
+  selector = {
+    mac = var.worker_macs[count.index]
+  }
+}
+

bare-metal/fedora-coreos/kubernetes/profiles.tf

@@ -0,0 +1,95 @@
+locals {
+  remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-kernel"
+  remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-initramfs.img"
+  remote_args = [
+    "ip=dhcp",
+    "rd.neednet=1",
+    "coreos.inst=yes",
+    "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-metal.raw.gz",
+    "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
+    "coreos.inst.install_dev=${var.install_disk}"
+  ]
+
+  cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel"
+  cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.img"
+  cached_args = [
+    "ip=dhcp",
+    "rd.neednet=1",
+    "coreos.inst=yes",
+    "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal-bios.raw.gz",
+    "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
+    "coreos.inst.install_dev=${var.install_disk}"
+  ]
+
+  kernel = var.cached_install == "true" ? local.cached_kernel : local.remote_kernel
+  initrd = var.cached_install == "true" ? local.cached_initrd : local.remote_initrd
+  args   = var.cached_install == "true" ? local.cached_args : local.remote_args
+}
+
+
+// Fedora CoreOS controller profile
+resource "matchbox_profile" "controllers" {
+  count = length(var.controller_names)
+  name  = format("%s-controller-%s", var.cluster_name, var.controller_names[count.index])
+
+  kernel = local.kernel
+  initrd = [
+    local.initrd
+  ]
+  args = concat(local.args, var.kernel_args)
+
+  raw_ignition = data.ct_config.controller-ignitions.*.rendered[count.index]
+}
+
+data "ct_config" "controller-ignitions" {
+  count = length(var.controller_names)
+
+  content      = data.template_file.controller-configs.*.rendered[count.index]
+}
+
+data "template_file" "controller-configs" {
+  count = length(var.controller_names)
+
+  template = file("${path.module}/fcc/controller.yaml")
+  vars = {
+    domain_name            = var.controller_domains[count.index]
+    etcd_name              = var.controller_names[count.index]
+    etcd_initial_cluster   = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))
+    cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip
+    cluster_domain_suffix  = var.cluster_domain_suffix
+    ssh_authorized_key     = var.ssh_authorized_key
+  }
+}
+
+// Fedora CoreOS worker profile
+resource "matchbox_profile" "workers" {
+  count = length(var.worker_names)
+  name  = format("%s-worker-%s", var.cluster_name, var.worker_names[count.index])
+
+  kernel = local.kernel
+  initrd = [
+    local.initrd
+  ]
+  args = concat(local.args, var.kernel_args)
+
+  raw_ignition = data.ct_config.worker-ignitions.*.rendered[count.index]
+}
+
+data "ct_config" "worker-ignitions" {
+  count = length(var.worker_names)
+
+  content      = data.template_file.worker-configs.*.rendered[count.index]
+}
+
+data "template_file" "worker-configs" {
+  count = length(var.worker_names)
+
+  template = file("${path.module}/fcc/worker.yaml")
+  vars = {
+    domain_name            = var.worker_domains[count.index]
+    cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip
+    cluster_domain_suffix  = var.cluster_domain_suffix
+    ssh_authorized_key     = var.ssh_authorized_key
+  }
+}
+

bare-metal/fedora-coreos/kubernetes/variables.tf

@@ -0,0 +1,158 @@
+variable "cluster_name" {
+  type        = string
+  description = "Unique cluster name"
+}
+
+# bare-metal
+
+variable "matchbox_http_endpoint" {
+  type        = string
+  description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)"
+}
+
+variable "os_stream" {
+  type        = string
+  description = "Fedora CoreOS release stream (e.g. testing, stable)"
+  default     = "testing"
+}
+
+variable "os_version" {
+  type        = string
+  description = "Fedora CoreOS version to PXE and install (e.g. 30.20190712.0)"
+}
+
+# machines
+# Terraform's crude "type system" does not properly support lists of maps so we do this.
+
+variable "controller_names" {
+  type        = list(string)
+  description = "Ordered list of controller names (e.g. [node1])"
+}
+
+variable "controller_macs" {
+  type        = list(string)
+  description = "Ordered list of controller identifying MAC addresses (e.g. [52:54:00:a1:9c:ae])"
+}
+
+variable "controller_domains" {
+  type        = list(string)
+  description = "Ordered list of controller FQDNs (e.g. [node1.example.com])"
+}
+
+variable "worker_names" {
+  type        = list(string)
+  description = "Ordered list of worker names (e.g. [node2, node3])"
+}
+
+variable "worker_macs" {
+  type        = list(string)
+  description = "Ordered list of worker identifying MAC addresses (e.g. [52:54:00:b2:2f:86, 52:54:00:c3:61:77])"
+}
+
+variable "worker_domains" {
+  type        = list(string)
+  description = "Ordered list of worker FQDNs (e.g. [node2.example.com, node3.example.com])"
+}
+
+variable "snippets" {
+  type        = map(list(string))
+  description = "Map from machine names to lists of Fedora CoreOS Config snippets"
+  default     = {}
+}
+
+# configuration
+
+variable "k8s_domain_name" {
+  description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)"
+  type        = string
+}
+
+variable "ssh_authorized_key" {
+  type        = string
+  description = "SSH public key for user 'core'"
+}
+
+variable "asset_dir" {
+  description = "Path to a directory where generated assets should be placed (contains secrets)"
+  type        = string
+}
+
+variable "networking" {
+  description = "Choice of networking provider (flannel or calico)"
+  type        = string
+  default     = "calico"
+}
+
+variable "network_mtu" {
+  description = "CNI interface MTU (applies to calico only)"
+  type        = string
+  default     = "1480"
+}
+
+variable "network_ip_autodetection_method" {
+  description = "Method to autodetect the host IPv4 address (applies to calico only)"
+  type        = string
+  default     = "first-found"
+}
+
+variable "pod_cidr" {
+  description = "CIDR IPv4 range to assign Kubernetes pods"
+  type        = string
+  default     = "10.2.0.0/16"
+}
+
+variable "service_cidr" {
+  description = <<EOD
+CIDR IPv4 range to assign Kubernetes services.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
+EOD
+
+
+  type = string
+  default = "10.3.0.0/16"
+}
+
+# optional
+
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type = string
+  default = "cluster.local"
+}
+
+variable "download_protocol" {
+  type = string
+  default = "https"
+  description = "Protocol iPXE should use to download the kernel and initrd. Defaults to https, which requires iPXE compiled with crypto support. Unused if cached_install is true."
+}
+
+variable "cached_install" {
+  type = string
+  default = "false"
+  description = "Whether Fedora CoreOS should PXE boot and install from matchbox /assets cache. Note that the admin must have downloaded the os_version into matchbox assets."
+}
+
+variable "install_disk" {
+  type = string
+  default = "sda"
+  description = "Disk device to install Fedora CoreOS (e.g. sda)"
+}
+
+variable "kernel_args" {
+  description = "Additional kernel arguments to provide at PXE boot."
+  type = list(string)
+  default = []
+}
+
+variable "enable_reporting" {
+  type = string
+  description = "Enable usage or analytics reporting to upstreams (Calico)"
+  default = "false"
+}
+
+variable "enable_aggregation" {
+  description = "Enable the Kubernetes Aggregation Layer (defaults to false)"
+  type = string
+  default = "false"
+}
+

bare-metal/fedora-coreos/kubernetes/versions.tf

@@ -0,0 +1,11 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = "~> 0.12.0"
+  required_providers {
+    matchbox = "~> 0.3.0"
+    ct       = "~> 0.4"
+    template = "~> 2.1"
+    null     = "~> 2.1"
+  }
+}