diff --git a/bare-metal/fedora-coreos/kubernetes/bootkube.tf b/bare-metal/fedora-coreos/kubernetes/bootkube.tf new file mode 100644 index 00000000..33df7bd0 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/bootkube.tf @@ -0,0 +1,19 @@ +# Self-hosted Kubernetes assets (kubeconfig, manifests) +module "bootkube" { + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=119cb00fa7b12e0ebd5a70c9c0a4e7eda2e8c3d6" + + cluster_name = var.cluster_name + api_servers = [var.k8s_domain_name] + etcd_servers = var.controller_domains + asset_dir = var.asset_dir + networking = var.networking + network_mtu = var.network_mtu + network_ip_autodetection_method = var.network_ip_autodetection_method + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + cluster_domain_suffix = var.cluster_domain_suffix + enable_reporting = var.enable_reporting + enable_aggregation = var.enable_aggregation +} + + diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml new file mode 100644 index 00000000..e4d8934c --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -0,0 +1,28 @@ +--- +variant: fcos +version: 1.0.0 +storage: + files: + - path: /etc/kubernetes/kubelet.env + filesystem: root + mode: 0644 + contents: + inline: | + KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube + KUBELET_IMAGE_TAG=v1.15.0 + - path: /etc/hostname + filesystem: root + mode: 0644 + contents: + inline: + ${domain_name} + - path: /etc/sysctl.d/max-user-watches.conf + filesystem: root + contents: + inline: | + fs.inotify.max_user_watches=16184 +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml new file mode 100644 index 00000000..8e18b2f1 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -0,0 +1,29 @@ +--- +variant: fcos +version: 1.0.0 +storage: + files: + - path: /etc/kubernetes/kubelet.env + filesystem: root + mode: 0644 + contents: + inline: | + KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube + KUBELET_IMAGE_TAG=v1.15.0 + - path: /etc/hostname + filesystem: root + mode: 0644 + contents: + inline: + ${domain_name} + - path: /etc/sysctl.d/max-user-watches.conf + filesystem: root + contents: + inline: | + fs.inotify.max_user_watches=16184 +passwd: + users: + - name: core + ssh_authorized_keys: + - ${ssh_authorized_key} + diff --git a/bare-metal/fedora-coreos/kubernetes/groups.tf b/bare-metal/fedora-coreos/kubernetes/groups.tf new file mode 100644 index 00000000..fd9f089f --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/groups.tf @@ -0,0 +1,22 @@ +# Match each controller or worker to a profile + +resource "matchbox_group" "controller" { + count = length(var.controller_names) + name = format("%s-%s", var.cluster_name, var.controller_names[count.index]) + profile = matchbox_profile.controllers.*.name[count.index] + + selector = { + mac = var.controller_macs[count.index] + } +} + +resource "matchbox_group" "worker" { + count = length(var.worker_names) + name = format("%s-%s", var.cluster_name, var.worker_names[count.index]) + profile = matchbox_profile.workers.*.name[count.index] + + selector = { + mac = var.worker_macs[count.index] + } +} + diff --git a/bare-metal/fedora-coreos/kubernetes/profiles.tf b/bare-metal/fedora-coreos/kubernetes/profiles.tf new file mode 100644 index 00000000..071b8ec5 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/profiles.tf @@ -0,0 +1,95 @@ +locals { + remote_kernel = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-kernel" + remote_initrd = "https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-installer-initramfs.img" + remote_args = [ + "ip=dhcp", + "rd.neednet=1", + "coreos.inst=yes", + "coreos.inst.image_url=https://builds.coreos.fedoraproject.org/prod/streams/${var.os_stream}/builds/${var.os_version}/fedora-coreos-${var.os_version}-metal.raw.gz", + "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", + "coreos.inst.install_dev=${var.install_disk}" + ] + + cached_kernel = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-kernel" + cached_initrd = "/assets/fedora-coreos/fedora-coreos-${var.os_version}-installer-initramfs.img" + cached_args = [ + "ip=dhcp", + "rd.neednet=1", + "coreos.inst=yes", + "coreos.inst.image_url=${var.matchbox_http_endpoint}/assets/fedora-coreos/fedora-coreos-${var.os_version}-metal-bios.raw.gz", + "coreos.inst.ignition_url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}", + "coreos.inst.install_dev=${var.install_disk}" + ] + + kernel = var.cached_install == "true" ? local.cached_kernel : local.remote_kernel + initrd = var.cached_install == "true" ? local.cached_initrd : local.remote_initrd + args = var.cached_install == "true" ? local.cached_args : local.remote_args +} + + +// Fedora CoreOS controller profile +resource "matchbox_profile" "controllers" { + count = length(var.controller_names) + name = format("%s-controller-%s", var.cluster_name, var.controller_names[count.index]) + + kernel = local.kernel + initrd = [ + local.initrd + ] + args = concat(local.args, var.kernel_args) + + raw_ignition = data.ct_config.controller-ignitions.*.rendered[count.index] +} + +data "ct_config" "controller-ignitions" { + count = length(var.controller_names) + + content = data.template_file.controller-configs.*.rendered[count.index] +} + +data "template_file" "controller-configs" { + count = length(var.controller_names) + + template = file("${path.module}/fcc/controller.yaml") + vars = { + domain_name = var.controller_domains[count.index] + etcd_name = var.controller_names[count.index] + etcd_initial_cluster = join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains)) + cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_domain_suffix = var.cluster_domain_suffix + ssh_authorized_key = var.ssh_authorized_key + } +} + +// Fedora CoreOS worker profile +resource "matchbox_profile" "workers" { + count = length(var.worker_names) + name = format("%s-worker-%s", var.cluster_name, var.worker_names[count.index]) + + kernel = local.kernel + initrd = [ + local.initrd + ] + args = concat(local.args, var.kernel_args) + + raw_ignition = data.ct_config.worker-ignitions.*.rendered[count.index] +} + +data "ct_config" "worker-ignitions" { + count = length(var.worker_names) + + content = data.template_file.worker-configs.*.rendered[count.index] +} + +data "template_file" "worker-configs" { + count = length(var.worker_names) + + template = file("${path.module}/fcc/worker.yaml") + vars = { + domain_name = var.worker_domains[count.index] + cluster_dns_service_ip = module.bootkube.cluster_dns_service_ip + cluster_domain_suffix = var.cluster_domain_suffix + ssh_authorized_key = var.ssh_authorized_key + } +} + diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf new file mode 100644 index 00000000..5cae2d30 --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -0,0 +1,158 @@ +variable "cluster_name" { + type = string + description = "Unique cluster name" +} + +# bare-metal + +variable "matchbox_http_endpoint" { + type = string + description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)" +} + +variable "os_stream" { + type = string + description = "Fedora CoreOS release stream (e.g. testing, stable)" + default = "testing" +} + +variable "os_version" { + type = string + description = "Fedora CoreOS version to PXE and install (e.g. 30.20190712.0)" +} + +# machines +# Terraform's crude "type system" does not properly support lists of maps so we do this. + +variable "controller_names" { + type = list(string) + description = "Ordered list of controller names (e.g. [node1])" +} + +variable "controller_macs" { + type = list(string) + description = "Ordered list of controller identifying MAC addresses (e.g. [52:54:00:a1:9c:ae])" +} + +variable "controller_domains" { + type = list(string) + description = "Ordered list of controller FQDNs (e.g. [node1.example.com])" +} + +variable "worker_names" { + type = list(string) + description = "Ordered list of worker names (e.g. [node2, node3])" +} + +variable "worker_macs" { + type = list(string) + description = "Ordered list of worker identifying MAC addresses (e.g. [52:54:00:b2:2f:86, 52:54:00:c3:61:77])" +} + +variable "worker_domains" { + type = list(string) + description = "Ordered list of worker FQDNs (e.g. [node2.example.com, node3.example.com])" +} + +variable "snippets" { + type = map(list(string)) + description = "Map from machine names to lists of Fedora CoreOS Config snippets" + default = {} +} + +# configuration + +variable "k8s_domain_name" { + description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)" + type = string +} + +variable "ssh_authorized_key" { + type = string + description = "SSH public key for user 'core'" +} + +variable "asset_dir" { + description = "Path to a directory where generated assets should be placed (contains secrets)" + type = string +} + +variable "networking" { + description = "Choice of networking provider (flannel or calico)" + type = string + default = "calico" +} + +variable "network_mtu" { + description = "CNI interface MTU (applies to calico only)" + type = string + default = "1480" +} + +variable "network_ip_autodetection_method" { + description = "Method to autodetect the host IPv4 address (applies to calico only)" + type = string + default = "first-found" +} + +variable "pod_cidr" { + description = "CIDR IPv4 range to assign Kubernetes pods" + type = string + default = "10.2.0.0/16" +} + +variable "service_cidr" { + description = <