Start prototype of Fedora CoreOS on bare-metal

* Use terraform-provider-ct v0.4.0 with Fedora CoreOS Config
support (not yet released)
This commit is contained in:
Dalton Hubble 2019-07-07 15:41:04 -07:00
parent dfa6bcfecf
commit eb92f67125
7 changed files with 362 additions and 0 deletions

View File

@ -0,0 +1,19 @@
# Self-hosted Kubernetes assets (kubeconfig, manifests)
module "bootkube" {
source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6"
cluster_name = var.cluster_name
api_servers = [var.k8s_domain_name]
etcd_servers = var.controller_domains
asset_dir = var.asset_dir
networking = var.networking
network_mtu = var.network_mtu
network_ip_autodetection_method = var.network_ip_autodetection_method
pod_cidr = var.pod_cidr
service_cidr = var.service_cidr
cluster_domain_suffix = var.cluster_domain_suffix
enable_reporting = var.enable_reporting
enable_aggregation = var.enable_aggregation
}

View File

@ -0,0 +1,28 @@
---
variant: fcos
version: 1.0.0
storage:
files:
- path: /etc/kubernetes/kubelet.env
filesystem: root
mode: 0644
contents:
inline: |
KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
KUBELET_IMAGE_TAG=v1.15.0
- path: /etc/hostname
filesystem: root
mode: 0644
contents:
inline:
${domain_name}
- path: /etc/sysctl.d/max-user-watches.conf
filesystem: root
contents:
inline: |
fs.inotify.max_user_watches=16184
passwd:
users:
- name: core
ssh_authorized_keys:
- ${ssh_authorized_key}

View File

@ -0,0 +1,29 @@
---
variant: fcos
version: 1.0.0
storage:
files:
- path: /etc/kubernetes/kubelet.env
filesystem: root
mode: 0644
contents:
inline: |
KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
KUBELET_IMAGE_TAG=v1.15.0
- path: /etc/hostname
filesystem: root
mode: 0644
contents:
inline:
${domain_name}
- path: /etc/sysctl.d/max-user-watches.conf
filesystem: root
contents:
inline: |
fs.inotify.max_user_watches=16184
passwd:
users:
- name: core
ssh_authorized_keys:
- ${ssh_authorized_key}

View File

@ -0,0 +1,22 @@
# Match each controller or worker to a profile
resource "matchbox_group" "controller" {
count = length(var.controller_names)
name = format("%s-%s", var.cluster_name, var.controller_names[count.index])
profile = matchbox_profile.controllers.*.name[count.index]
selector = {
mac = var.controller_macs[count.index]
}
}
resource "matchbox_group" "worker" {
count = length(var.worker_names)
name = format("%s-%s", var.cluster_name, var.worker_names[count.index])
profile = matchbox_profile.workers.*.name[count.index]
selector = {
mac = var.worker_macs[count.index]
}
}

View File

@ -0,0 +1,95 @@
locals {
remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-kernel"
remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-initramfs.img"
remote_args = [
"ip=dhcp",
"rd.neednet=1",
"coreos.inst=yes",
"coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-metal.raw.gz",
"coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
"coreos.inst.install_dev=${var.install_disk}"
]
cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel"
cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.img"
cached_args = [
"ip=dhcp",
"rd.neednet=1",
"coreos.inst=yes",
"coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal-bios.raw.gz",
"coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
"coreos.inst.install_dev=${var.install_disk}"
]
kernel = var.cached_install == "true" ? local.cached_kernel : local.remote_kernel
initrd = var.cached_install == "true" ? local.cached_initrd : local.remote_initrd
args = var.cached_install == "true" ? local.cached_args : local.remote_args
}
// Fedora CoreOS controller profile
resource "matchbox_profile" "controllers" {
count = length(var.controller_names)
name = format("%s-controller-%s", var.cluster_name, var.controller_names[count.index])
kernel = local.kernel
initrd = [
local.initrd
]
args = concat(local.args, var.kernel_args)
raw_ignition = data.ct_config.controller-ignitions.*.rendered[count.index]
}
data "ct_config" "controller-ignitions" {
count = length(var.controller_names)
content = data.template_file.controller-configs.*.rendered[count.index]
}
data "template_file" "controller-configs" {
count = length(var.controller_names)
template = file("${path.module}/fcc/controller.yaml")
vars = {
domain_name = var.controller_domains[count.index]
etcd_name = var.controller_names[count.index]
etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))
cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip
cluster_domain_suffix = var.cluster_domain_suffix
ssh_authorized_key = var.ssh_authorized_key
}
}
// Fedora CoreOS worker profile
resource "matchbox_profile" "workers" {
count = length(var.worker_names)
name = format("%s-worker-%s", var.cluster_name, var.worker_names[count.index])
kernel = local.kernel
initrd = [
local.initrd
]
args = concat(local.args, var.kernel_args)
raw_ignition = data.ct_config.worker-ignitions.*.rendered[count.index]
}
data "ct_config" "worker-ignitions" {
count = length(var.worker_names)
content = data.template_file.worker-configs.*.rendered[count.index]
}
data "template_file" "worker-configs" {
count = length(var.worker_names)
template = file("${path.module}/fcc/worker.yaml")
vars = {
domain_name = var.worker_domains[count.index]
cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip
cluster_domain_suffix = var.cluster_domain_suffix
ssh_authorized_key = var.ssh_authorized_key
}
}

View File

@ -0,0 +1,158 @@
variable "cluster_name" {
type = string
description = "Unique cluster name"
}
# bare-metal
variable "matchbox_http_endpoint" {
type = string
description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)"
}
variable "os_stream" {
type = string
description = "Fedora CoreOS release stream (e.g. testing, stable)"
default = "testing"
}
variable "os_version" {
type = string
description = "Fedora CoreOS version to PXE and install (e.g. 30.20190712.0)"
}
# machines
# Terraform's crude "type system" does not properly support lists of maps so we do this.
variable "controller_names" {
type = list(string)
description = "Ordered list of controller names (e.g. [node1])"
}
variable "controller_macs" {
type = list(string)
description = "Ordered list of controller identifying MAC addresses (e.g. [52:54:00:a1:9c:ae])"
}
variable "controller_domains" {
type = list(string)
description = "Ordered list of controller FQDNs (e.g. [node1.example.com])"
}
variable "worker_names" {
type = list(string)
description = "Ordered list of worker names (e.g. [node2, node3])"
}
variable "worker_macs" {
type = list(string)
description = "Ordered list of worker identifying MAC addresses (e.g. [52:54:00:b2:2f:86, 52:54:00:c3:61:77])"
}
variable "worker_domains" {
type = list(string)
description = "Ordered list of worker FQDNs (e.g. [node2.example.com, node3.example.com])"
}
variable "snippets" {
type = map(list(string))
description = "Map from machine names to lists of Fedora CoreOS Config snippets"
default = {}
}
# configuration
variable "k8s_domain_name" {
description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)"
type = string
}
variable "ssh_authorized_key" {
type = string
description = "SSH public key for user 'core'"
}
variable "asset_dir" {
description = "Path to a directory where generated assets should be placed (contains secrets)"
type = string
}
variable "networking" {
description = "Choice of networking provider (flannel or calico)"
type = string
default = "calico"
}
variable "network_mtu" {
description = "CNI interface MTU (applies to calico only)"
type = string
default = "1480"
}
variable "network_ip_autodetection_method" {
description = "Method to autodetect the host IPv4 address (applies to calico only)"
type = string
default = "first-found"
}
variable "pod_cidr" {
description = "CIDR IPv4 range to assign Kubernetes pods"
type = string
default = "10.2.0.0/16"
}
variable "service_cidr" {
description = <<EOD
CIDR IPv4 range to assign Kubernetes services.
The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
EOD
type = string
default = "10.3.0.0/16"
}
# optional
variable "cluster_domain_suffix" {
description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
type = string
default = "cluster.local"
}
variable "download_protocol" {
type = string
default = "https"
description = "Protocol iPXE should use to download the kernel and initrd. Defaults to https, which requires iPXE compiled with crypto support. Unused if cached_install is true."
}
variable "cached_install" {
type = string
default = "false"
description = "Whether Fedora CoreOS should PXE boot and install from matchbox /assets cache. Note that the admin must have downloaded the os_version into matchbox assets."
}
variable "install_disk" {
type = string
default = "sda"
description = "Disk device to install Fedora CoreOS (e.g. sda)"
}
variable "kernel_args" {
description = "Additional kernel arguments to provide at PXE boot."
type = list(string)
default = []
}
variable "enable_reporting" {
type = string
description = "Enable usage or analytics reporting to upstreams (Calico)"
default = "false"
}
variable "enable_aggregation" {
description = "Enable the Kubernetes Aggregation Layer (defaults to false)"
type = string
default = "false"
}

View File

@ -0,0 +1,11 @@
# Terraform version and plugin versions
terraform {
required_version = "~> 0.12.0"
required_providers {
matchbox = "~> 0.3.0"
ct = "~> 0.4"
template = "~> 2.1"
null = "~> 2.1"
}
}