Start prototype of Fedora CoreOS on bare-metal
* Use terraform-provider-ct v0.4.0 with Fedora CoreOS Config support (not yet released)
This commit is contained in:
Dalton Hubble
parent
dfa6bcfecf
commit
eb92f67125
|
|
@ -0,0 +1,19 @@
|
|||
# Self-hosted Kubernetes assets (kubeconfig, manifests)
|
||||
module "bootkube" {
|
||||
source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6"
|
||||
|
||||
cluster_name = var.cluster_name
|
||||
api_servers = [var.k8s_domain_name]
|
||||
etcd_servers = var.controller_domains
|
||||
asset_dir = var.asset_dir
|
||||
networking = var.networking
|
||||
network_mtu = var.network_mtu
|
||||
network_ip_autodetection_method = var.network_ip_autodetection_method
|
||||
pod_cidr = var.pod_cidr
|
||||
service_cidr = var.service_cidr
|
||||
cluster_domain_suffix = var.cluster_domain_suffix
|
||||
enable_reporting = var.enable_reporting
|
||||
enable_aggregation = var.enable_aggregation
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
---
|
||||
variant: fcos
|
||||
version: 1.0.0
|
||||
storage:
|
||||
files:
|
||||
- path: /etc/kubernetes/kubelet.env
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline: |
|
||||
KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
|
||||
KUBELET_IMAGE_TAG=v1.15.0
|
||||
- path: /etc/hostname
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline:
|
||||
${domain_name}
|
||||
- path: /etc/sysctl.d/max-user-watches.conf
|
||||
filesystem: root
|
||||
contents:
|
||||
inline: |
|
||||
fs.inotify.max_user_watches=16184
|
||||
passwd:
|
||||
users:
|
||||
- name: core
|
||||
ssh_authorized_keys:
|
||||
- ${ssh_authorized_key}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
---
|
||||
variant: fcos
|
||||
version: 1.0.0
|
||||
storage:
|
||||
files:
|
||||
- path: /etc/kubernetes/kubelet.env
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline: |
|
||||
KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
|
||||
KUBELET_IMAGE_TAG=v1.15.0
|
||||
- path: /etc/hostname
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline:
|
||||
${domain_name}
|
||||
- path: /etc/sysctl.d/max-user-watches.conf
|
||||
filesystem: root
|
||||
contents:
|
||||
inline: |
|
||||
fs.inotify.max_user_watches=16184
|
||||
passwd:
|
||||
users:
|
||||
- name: core
|
||||
ssh_authorized_keys:
|
||||
- ${ssh_authorized_key}
|
||||
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Match each controller or worker to a profile
|
||||
|
||||
resource "matchbox_group" "controller" {
|
||||
count = length(var.controller_names)
|
||||
name = format("%s-%s", var.cluster_name, var.controller_names[count.index])
|
||||
profile = matchbox_profile.controllers.*.name[count.index]
|
||||
|
||||
selector = {
|
||||
mac = var.controller_macs[count.index]
|
||||
}
|
||||
}
|
||||
|
||||
resource "matchbox_group" "worker" {
|
||||
count = length(var.worker_names)
|
||||
name = format("%s-%s", var.cluster_name, var.worker_names[count.index])
|
||||
profile = matchbox_profile.workers.*.name[count.index]
|
||||
|
||||
selector = {
|
||||
mac = var.worker_macs[count.index]
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
locals {
|
||||
remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-kernel"
|
||||
remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-initramfs.img"
|
||||
remote_args = [
|
||||
"ip=dhcp",
|
||||
"rd.neednet=1",
|
||||
"coreos.inst=yes",
|
||||
"coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-metal.raw.gz",
|
||||
"coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
|
||||
"coreos.inst.install_dev=${var.install_disk}"
|
||||
]
|
||||
|
||||
cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel"
|
||||
cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.img"
|
||||
cached_args = [
|
||||
"ip=dhcp",
|
||||
"rd.neednet=1",
|
||||
"coreos.inst=yes",
|
||||
"coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal-bios.raw.gz",
|
||||
"coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
|
||||
"coreos.inst.install_dev=${var.install_disk}"
|
||||
]
|
||||
|
||||
kernel = var.cached_install == "true" ? local.cached_kernel : local.remote_kernel
|
||||
initrd = var.cached_install == "true" ? local.cached_initrd : local.remote_initrd
|
||||
args = var.cached_install == "true" ? local.cached_args : local.remote_args
|
||||
}
|
||||
|
||||
|
||||
// Fedora CoreOS controller profile
|
||||
resource "matchbox_profile" "controllers" {
|
||||
count = length(var.controller_names)
|
||||
name = format("%s-controller-%s", var.cluster_name, var.controller_names[count.index])
|
||||
|
||||
kernel = local.kernel
|
||||
initrd = [
|
||||
local.initrd
|
||||
]
|
||||
args = concat(local.args, var.kernel_args)
|
||||
|
||||
raw_ignition = data.ct_config.controller-ignitions.*.rendered[count.index]
|
||||
}
|
||||
|
||||
data "ct_config" "controller-ignitions" {
|
||||
count = length(var.controller_names)
|
||||
|
||||
content = data.template_file.controller-configs.*.rendered[count.index]
|
||||
}
|
||||
|
||||
data "template_file" "controller-configs" {
|
||||
count = length(var.controller_names)
|
||||
|
||||
template = file("${path.module}/fcc/controller.yaml")
|
||||
vars = {
|
||||
domain_name = var.controller_domains[count.index]
|
||||
etcd_name = var.controller_names[count.index]
|
||||
etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))
|
||||
cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip
|
||||
cluster_domain_suffix = var.cluster_domain_suffix
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
}
|
||||
}
|
||||
|
||||
// Fedora CoreOS worker profile
|
||||
resource "matchbox_profile" "workers" {
|
||||
count = length(var.worker_names)
|
||||
name = format("%s-worker-%s", var.cluster_name, var.worker_names[count.index])
|
||||
|
||||
kernel = local.kernel
|
||||
initrd = [
|
||||
local.initrd
|
||||
]
|
||||
args = concat(local.args, var.kernel_args)
|
||||
|
||||
raw_ignition = data.ct_config.worker-ignitions.*.rendered[count.index]
|
||||
}
|
||||
|
||||
data "ct_config" "worker-ignitions" {
|
||||
count = length(var.worker_names)
|
||||
|
||||
content = data.template_file.worker-configs.*.rendered[count.index]
|
||||
}
|
||||
|
||||
data "template_file" "worker-configs" {
|
||||
count = length(var.worker_names)
|
||||
|
||||
template = file("${path.module}/fcc/worker.yaml")
|
||||
vars = {
|
||||
domain_name = var.worker_domains[count.index]
|
||||
cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip
|
||||
cluster_domain_suffix = var.cluster_domain_suffix
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
variable "cluster_name" {
|
||||
type = string
|
||||
description = "Unique cluster name"
|
||||
}
|
||||
|
||||
# bare-metal
|
||||
|
||||
variable "matchbox_http_endpoint" {
|
||||
type = string
|
||||
description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)"
|
||||
}
|
||||
|
||||
variable "os_stream" {
|
||||
type = string
|
||||
description = "Fedora CoreOS release stream (e.g. testing, stable)"
|
||||
default = "testing"
|
||||
}
|
||||
|
||||
variable "os_version" {
|
||||
type = string
|
||||
description = "Fedora CoreOS version to PXE and install (e.g. 30.20190712.0)"
|
||||
}
|
||||
|
||||
# machines
|
||||
# Terraform's crude "type system" does not properly support lists of maps so we do this.
|
||||
|
||||
variable "controller_names" {
|
||||
type = list(string)
|
||||
description = "Ordered list of controller names (e.g. [node1])"
|
||||
}
|
||||
|
||||
variable "controller_macs" {
|
||||
type = list(string)
|
||||
description = "Ordered list of controller identifying MAC addresses (e.g. [52:54:00:a1:9c:ae])"
|
||||
}
|
||||
|
||||
variable "controller_domains" {
|
||||
type = list(string)
|
||||
description = "Ordered list of controller FQDNs (e.g. [node1.example.com])"
|
||||
}
|
||||
|
||||
variable "worker_names" {
|
||||
type = list(string)
|
||||
description = "Ordered list of worker names (e.g. [node2, node3])"
|
||||
}
|
||||
|
||||
variable "worker_macs" {
|
||||
type = list(string)
|
||||
description = "Ordered list of worker identifying MAC addresses (e.g. [52:54:00:b2:2f:86, 52:54:00:c3:61:77])"
|
||||
}
|
||||
|
||||
variable "worker_domains" {
|
||||
type = list(string)
|
||||
description = "Ordered list of worker FQDNs (e.g. [node2.example.com, node3.example.com])"
|
||||
}
|
||||
|
||||
variable "snippets" {
|
||||
type = map(list(string))
|
||||
description = "Map from machine names to lists of Fedora CoreOS Config snippets"
|
||||
default = {}
|
||||
}
|
||||
|
||||
# configuration
|
||||
|
||||
variable "k8s_domain_name" {
|
||||
description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "ssh_authorized_key" {
|
||||
type = string
|
||||
description = "SSH public key for user 'core'"
|
||||
}
|
||||
|
||||
variable "asset_dir" {
|
||||
description = "Path to a directory where generated assets should be placed (contains secrets)"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "networking" {
|
||||
description = "Choice of networking provider (flannel or calico)"
|
||||
type = string
|
||||
default = "calico"
|
||||
}
|
||||
|
||||
variable "network_mtu" {
|
||||
description = "CNI interface MTU (applies to calico only)"
|
||||
type = string
|
||||
default = "1480"
|
||||
}
|
||||
|
||||
variable "network_ip_autodetection_method" {
|
||||
description = "Method to autodetect the host IPv4 address (applies to calico only)"
|
||||
type = string
|
||||
default = "first-found"
|
||||
}
|
||||
|
||||
variable "pod_cidr" {
|
||||
description = "CIDR IPv4 range to assign Kubernetes pods"
|
||||
type = string
|
||||
default = "10.2.0.0/16"
|
||||
}
|
||||
|
||||
variable "service_cidr" {
|
||||
description = <<EOD
|
||||
CIDR IPv4 range to assign Kubernetes services.
|
||||
The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
|
||||
EOD
|
||||
|
||||
|
||||
type = string
|
||||
default = "10.3.0.0/16"
|
||||
}
|
||||
|
||||
# optional
|
||||
|
||||
variable "cluster_domain_suffix" {
|
||||
description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
|
||||
type = string
|
||||
default = "cluster.local"
|
||||
}
|
||||
|
||||
variable "download_protocol" {
|
||||
type = string
|
||||
default = "https"
|
||||
description = "Protocol iPXE should use to download the kernel and initrd. Defaults to https, which requires iPXE compiled with crypto support. Unused if cached_install is true."
|
||||
}
|
||||
|
||||
variable "cached_install" {
|
||||
type = string
|
||||
default = "false"
|
||||
description = "Whether Fedora CoreOS should PXE boot and install from matchbox /assets cache. Note that the admin must have downloaded the os_version into matchbox assets."
|
||||
}
|
||||
|
||||
variable "install_disk" {
|
||||
type = string
|
||||
default = "sda"
|
||||
description = "Disk device to install Fedora CoreOS (e.g. sda)"
|
||||
}
|
||||
|
||||
variable "kernel_args" {
|
||||
description = "Additional kernel arguments to provide at PXE boot."
|
||||
type = list(string)
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "enable_reporting" {
|
||||
type = string
|
||||
description = "Enable usage or analytics reporting to upstreams (Calico)"
|
||||
default = "false"
|
||||
}
|
||||
|
||||
variable "enable_aggregation" {
|
||||
description = "Enable the Kubernetes Aggregation Layer (defaults to false)"
|
||||
type = string
|
||||
default = "false"
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# Terraform version and plugin versions
|
||||
|
||||
terraform {
|
||||
required_version = "~> 0.12.0"
|
||||
required_providers {
|
||||
matchbox = "~> 0.3.0"
|
||||
ct = "~> 0.4"
|
||||
template = "~> 2.1"
|
||||
null = "~> 2.1"
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue