From 7b8a51070f14595f7a086078ddaedc74c5ad1029 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 19 May 2024 16:53:47 -0700 Subject: [PATCH] Add Terraform modules for CoreDNS, Cilium, and flannel * With the new component system, these components can be managed independent from the cluster and rolled or edited in advanced ways --- CHANGES.md | 3 +- README.md | 10 +- addons/cilium/cluster-role-binding.tf | 36 +++ addons/cilium/cluster-role.tf | 112 ++++++++ addons/cilium/config.tf | 196 +++++++++++++ addons/cilium/daemonset.tf | 379 +++++++++++++++++++++++++ addons/cilium/deployment.tf | 163 +++++++++++ addons/cilium/service-account.tf | 15 + addons/cilium/variables.tf | 17 ++ addons/cilium/versions.tf | 8 + addons/coredns/cluster-role.tf | 37 +++ addons/coredns/config.tf | 30 ++ addons/coredns/deployment.tf | 151 ++++++++++ addons/coredns/service-account.tf | 24 ++ addons/coredns/service.tf | 31 ++ addons/coredns/variables.tf | 15 + addons/coredns/versions.tf | 9 + addons/flannel/cluster-role-binding.tf | 18 ++ addons/flannel/cluster-role.tf | 24 ++ addons/flannel/config.tf | 44 +++ addons/flannel/daemonset.tf | 167 +++++++++++ addons/flannel/service-account.tf | 7 + addons/flannel/variables.tf | 11 + addons/flannel/versions.tf | 8 + 24 files changed, 1513 insertions(+), 2 deletions(-) create mode 100644 addons/cilium/cluster-role-binding.tf create mode 100644 addons/cilium/cluster-role.tf create mode 100644 addons/cilium/config.tf create mode 100644 addons/cilium/daemonset.tf create mode 100644 addons/cilium/deployment.tf create mode 100644 addons/cilium/service-account.tf create mode 100644 addons/cilium/variables.tf create mode 100644 addons/cilium/versions.tf create mode 100644 addons/coredns/cluster-role.tf create mode 100644 addons/coredns/config.tf create mode 100644 addons/coredns/deployment.tf create mode 100644 addons/coredns/service-account.tf create mode 100644 addons/coredns/service.tf create mode 100644 addons/coredns/variables.tf create mode 100644 addons/coredns/versions.tf create mode 100644 addons/flannel/cluster-role-binding.tf create mode 100644 addons/flannel/cluster-role.tf create mode 100644 addons/flannel/config.tf create mode 100644 addons/flannel/daemonset.tf create mode 100644 addons/flannel/service-account.tf create mode 100644 addons/flannel/variables.tf create mode 100644 addons/flannel/versions.tf diff --git a/CHANGES.md b/CHANGES.md index a151c346..f226ceca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,9 +8,10 @@ Notable changes between versions. * Kubernetes [v1.30.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.30.md#v1301) * Add firewall rules and security group rules for Cilium and Hubble metrics ([#1449](https://github.com/poseidon/typhoon/pull/1449)) -* Introduce `components` variabe to enable/disable/configure pre-installed components ([#1453](https://github.com/poseidon/typhoon/pull/1453)) * Update Cilium from v1.15.3 to [v1.15.5](https://github.com/cilium/cilium/releases/tag/v1.15.5) * Update flannel from v0.24.4 to [v0.25.1](https://github.com/flannel-io/flannel/releases/tag/v0.25.1) +* Introduce `components` variabe to enable/disable/configure pre-installed components ([#1453](https://github.com/poseidon/typhoon/pull/1453)) +* Add Terraform modules for `coredns`, `cilium`, and `flannel` components ### Azure diff --git a/README.md b/README.md index 0ecacaf4..f90f5a5c 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Modules -Typhoon provides a Terraform Module for each supported operating system and platform. +Typhoon provides a Terraform Module for defining a Kubernetes cluster on each supported operating system and platform. Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/). @@ -57,6 +57,14 @@ Typhoon is available for [Flatcar Linux](https://www.flatcar-linux.org/releases/ | AWS | Flatcar Linux (ARM64) | [aws/flatcar-linux/kubernetes](aws/flatcar-linux/kubernetes) | alpha | | Azure | Flatcar Linux (ARM64) | [azure/flatcar-linux/kubernetes](azure/flatcar-linux/kubernetes) | alpha | +Typhoon also provides Terraform Modules for optionally managing individual components applied onto clusters. + +| Name | Terraform Module | Status | +|---------|------------------|--------| +| CoreDNS | [addons/coredns](addons/coredns) | beta | +| Cilium | [addons/cilium](addons/cilium) | beta | +| flannel | [addons/flannel](addons/flannel) | beta | + ## Documentation * [Docs](https://typhoon.psdn.io) diff --git a/addons/cilium/cluster-role-binding.tf b/addons/cilium/cluster-role-binding.tf new file mode 100644 index 00000000..4e52951e --- /dev/null +++ b/addons/cilium/cluster-role-binding.tf @@ -0,0 +1,36 @@ +resource "kubernetes_cluster_role_binding" "operator" { + metadata { + name = "cilium-operator" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cilium-operator" + } + + subject { + kind = "ServiceAccount" + name = "cilium-operator" + namespace = "kube-system" + } +} + +resource "kubernetes_cluster_role_binding" "agent" { + metadata { + name = "cilium-agent" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cilium-agent" + } + + subject { + kind = "ServiceAccount" + name = "cilium-agent" + namespace = "kube-system" + } +} + diff --git a/addons/cilium/cluster-role.tf b/addons/cilium/cluster-role.tf new file mode 100644 index 00000000..a791ab3b --- /dev/null +++ b/addons/cilium/cluster-role.tf @@ -0,0 +1,112 @@ +resource "kubernetes_cluster_role" "operator" { + metadata { + name = "cilium-operator" + } + + # detect and restart [core|kube]dns pods on startup + rule { + verbs = ["get", "list", "watch", "delete"] + api_groups = [""] + resources = ["pods"] + } + + rule { + verbs = ["list", "watch"] + api_groups = [""] + resources = ["nodes"] + } + + rule { + verbs = ["patch"] + api_groups = [""] + resources = ["nodes", "nodes/status"] + } + + rule { + verbs = ["get", "list", "watch"] + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + } + + rule { + verbs = ["get", "list", "watch"] + api_groups = [""] + resources = ["services"] + } + + # Perform LB IP allocation for BGP + rule { + verbs = ["update"] + api_groups = [""] + resources = ["services/status"] + } + + # Perform the translation of a CNP that contains `ToGroup` to its endpoints + rule { + verbs = ["get", "list", "watch"] + api_groups = [""] + resources = ["services", "endpoints", "namespaces"] + } + + rule { + verbs = ["*"] + api_groups = ["cilium.io"] + resources = ["ciliumnetworkpolicies", "ciliumnetworkpolicies/status", "ciliumnetworkpolicies/finalizers", "ciliumclusterwidenetworkpolicies", "ciliumclusterwidenetworkpolicies/status", "ciliumclusterwidenetworkpolicies/finalizers", "ciliumendpoints", "ciliumendpoints/status", "ciliumendpoints/finalizers", "ciliumnodes", "ciliumnodes/status", "ciliumnodes/finalizers", "ciliumidentities", "ciliumidentities/status", "ciliumidentities/finalizers", "ciliumlocalredirectpolicies", "ciliumlocalredirectpolicies/status", "ciliumlocalredirectpolicies/finalizers", "ciliumendpointslices", "ciliumloadbalancerippools", "ciliumloadbalancerippools/status", "ciliumcidrgroups", "ciliuml2announcementpolicies", "ciliuml2announcementpolicies/status", "ciliumpodippools"] + } + + rule { + verbs = ["create", "get", "list", "update", "watch"] + api_groups = ["apiextensions.k8s.io"] + resources = ["customresourcedefinitions"] + } + + # Cilium leader elects if among multiple operator replicas + rule { + verbs = ["create", "get", "update"] + api_groups = ["coordination.k8s.io"] + resources = ["leases"] + } +} + +resource "kubernetes_cluster_role" "agent" { + metadata { + name = "cilium-agent" + } + + rule { + verbs = ["get", "list", "watch"] + api_groups = ["networking.k8s.io"] + resources = ["networkpolicies"] + } + + rule { + verbs = ["get", "list", "watch"] + api_groups = ["discovery.k8s.io"] + resources = ["endpointslices"] + } + + rule { + verbs = ["get", "list", "watch"] + api_groups = [""] + resources = ["namespaces", "services", "pods", "endpoints", "nodes"] + } + + rule { + verbs = ["patch"] + api_groups = [""] + resources = ["nodes/status"] + } + + rule { + verbs = ["create", "get", "list", "watch", "update"] + api_groups = ["apiextensions.k8s.io"] + resources = ["customresourcedefinitions"] + } + + rule { + verbs = ["*"] + api_groups = ["cilium.io"] + resources = ["ciliumnetworkpolicies", "ciliumnetworkpolicies/status", "ciliumclusterwidenetworkpolicies", "ciliumclusterwidenetworkpolicies/status", "ciliumendpoints", "ciliumendpoints/status", "ciliumnodes", "ciliumnodes/status", "ciliumidentities", "ciliumidentities/status", "ciliumlocalredirectpolicies", "ciliumlocalredirectpolicies/status", "ciliumegressnatpolicies", "ciliumendpointslices", "ciliumcidrgroups", "ciliuml2announcementpolicies", "ciliuml2announcementpolicies/status", "ciliumpodippools"] + } +} + diff --git a/addons/cilium/config.tf b/addons/cilium/config.tf new file mode 100644 index 00000000..799428af --- /dev/null +++ b/addons/cilium/config.tf @@ -0,0 +1,196 @@ +resource "kubernetes_config_map" "cilium" { + metadata { + name = "cilium" + namespace = "kube-system" + } + data = { + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in a kvstore, etcd or consul, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode = "crd" + cilium-endpoint-gc-interval = "5m0s" + nodes-gc-interval = "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug = "false" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes + enable-policy = "default" + + # Prometheus + enable-metrics = "true" + prometheus-serve-addr = ":9962" + operator-prometheus-serve-addr = ":9963" + proxy-prometheus-port = "9964" # envoy + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4 = "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6 = "false" + + # Enable probing for a more efficient clock source for the BPF datapath + enable-bpf-clock-probe = "true" + + # Enable use of transparent proxying mechanisms (Linux 5.7+) + enable-bpf-tproxy = "false" + + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation = "medium" + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval = "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags = "all" + + # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio = "0.0025" + # bpf-policy-map-max specified the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max = "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max = "65536" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps = "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name = "default" + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id = "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + routing-mode = "tunnel" + tunnel = "vxlan" + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy = "true" + + auto-direct-node-routes = "false" + + # enableXTSocketFallback enables the fallback compatibility solution + # when the xt_socket kernel module is missing and it is needed for + # the datapath L7 redirection to work properly. See documentation + # for details on when this can be disabled: + # http://docs.cilium.io/en/latest/install/system_requirements/#admin-kernel-version. + enable-xt-socket-fallback = "true" + + # installIptablesRules enables installation of iptables rules to allow for + # TPROXY (L7 proxy injection), itpables based masquerading and compatibility + # with kube-proxy. See documentation for details on when this can be + # disabled. + install-iptables-rules = "true" + + # masquerade traffic leaving the node destined for outside + enable-ipv4-masquerade = "true" + enable-ipv6-masquerade = "false" + + # bpfMasquerade enables masquerading with BPF instead of iptables + enable-bpf-masquerade = "true" + + # kube-proxy + kube-proxy-replacement = "false" + kube-proxy-replacement-healthz-bind-address = "" + enable-session-affinity = "true" + + # ClusterIPs from host namespace + bpf-lb-sock = "true" + # ClusterIPs from external nodes + bpf-lb-external-clusterip = "true" + + # NodePort + enable-node-port = "true" + enable-health-check-nodeport = "false" + + # ExternalIPs + enable-external-ips = "true" + + # HostPort + enable-host-port = "true" + + # IPAM + ipam = "cluster-pool" + disable-cnp-status-updates = "true" + cluster-pool-ipv4-cidr = "${var.pod_cidr}" + cluster-pool-ipv4-mask-size = "24" + + # Health + agent-health-port = "9876" + enable-health-checking = "true" + enable-endpoint-health-checking = "true" + + # Identity + enable-well-known-identities = "false" + enable-remote-node-identity = "true" + + # Hubble server + enable-hubble = var.enable_hubble + hubble-disable-tls = "false" + hubble-listen-address = ":4244" + hubble-socket-path = "/var/run/cilium/hubble.sock" + hubble-tls-client-ca-files = "/var/lib/cilium/tls/hubble/client-ca.crt" + hubble-tls-cert-file = "/var/lib/cilium/tls/hubble/server.crt" + hubble-tls-key-file = "/var/lib/cilium/tls/hubble/server.key" + hubble-export-file-max-backups = "5" + hubble-export-file-max-size-mb = "10" + + # Hubble metrics + hubble-metrics-server = ":9965" + hubble-metrics = "dns drop tcp flow port-distribution icmp httpV2" + enable-hubble-open-metrics = "false" + + + # Misc + enable-bandwidth-manager = "false" + enable-local-redirect-policy = "false" + policy-audit-mode = "false" + operator-api-serve-addr = "127.0.0.1:9234" + enable-l2-neigh-discovery = "true" + enable-k8s-terminating-endpoint = "true" + enable-k8s-networkpolicy = "true" + external-envoy-proxy = "false" + write-cni-conf-when-ready = "/host/etc/cni/net.d/05-cilium.conflist" + cni-exclusive = "true" + cni-log-file = "/var/run/cilium/cilium-cni.log" + } +} + diff --git a/addons/cilium/daemonset.tf b/addons/cilium/daemonset.tf new file mode 100644 index 00000000..5c175246 --- /dev/null +++ b/addons/cilium/daemonset.tf @@ -0,0 +1,379 @@ +resource "kubernetes_daemonset" "cilium" { + wait_for_rollout = false + + metadata { + name = "cilium" + namespace = "kube-system" + labels = { + k8s-app = "cilium" + } + } + spec { + strategy { + type = "RollingUpdate" + rolling_update { + max_unavailable = "1" + } + } + selector { + match_labels = { + k8s-app = "cilium-agent" + } + } + template { + metadata { + labels = { + k8s-app = "cilium-agent" + } + annotations = { + "prometheus.io/port" = "9962" + "prometheus.io/scrape" = "true" + } + } + spec { + host_network = true + priority_class_name = "system-node-critical" + service_account_name = "cilium-agent" + security_context { + seccomp_profile { + type = "RuntimeDefault" + } + } + toleration { + key = "node-role.kubernetes.io/controller" + operator = "Exists" + } + toleration { + key = "node.kubernetes.io/not-ready" + operator = "Exists" + } + dynamic "toleration" { + for_each = var.daemonset_tolerations + content { + key = toleration.value + operator = "Exists" + } + } + automount_service_account_token = true + enable_service_links = false + + # Cilium v1.13.1 starts installing CNI plugins in yet another init container + # https://github.com/cilium/cilium/pull/24075 + init_container { + name = "install-cni" + image = "quay.io/cilium/cilium:v1.15.5" + command = ["/install-plugin.sh"] + security_context { + allow_privilege_escalation = true + privileged = true + capabilities { + drop = ["ALL"] + } + } + volume_mount { + name = "cni-bin-dir" + mount_path = "/host/opt/cni/bin" + } + } + + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + init_container { + name = "mount-cgroup" + image = "quay.io/cilium/cilium:v1.15.5" + command = [ + "sh", + "-ec", + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + "cp /usr/bin/cilium-mount /hostbin/cilium-mount && nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt \"$${BIN_PATH}/cilium-mount\" $CGROUP_ROOT; rm /hostbin/cilium-mount" + ] + env { + name = "CGROUP_ROOT" + value = "/run/cilium/cgroupv2" + } + env { + name = "BIN_PATH" + value = "/opt/cni/bin" + } + security_context { + allow_privilege_escalation = true + privileged = true + } + volume_mount { + name = "hostproc" + mount_path = "/hostproc" + } + volume_mount { + name = "cni-bin-dir" + mount_path = "/hostbin" + } + } + + init_container { + name = "clean-cilium-state" + image = "quay.io/cilium/cilium:v1.15.5" + command = ["/init-container.sh"] + security_context { + allow_privilege_escalation = true + privileged = true + } + volume_mount { + name = "sys-fs-bpf" + mount_path = "/sys/fs/bpf" + } + volume_mount { + name = "var-run-cilium" + mount_path = "/var/run/cilium" + } + # Required to mount cgroup filesystem from the host to cilium agent pod + volume_mount { + name = "cilium-cgroup" + mount_path = "/run/cilium/cgroupv2" + mount_propagation = "HostToContainer" + } + } + + container { + name = "cilium-agent" + image = "quay.io/cilium/cilium:v1.15.5" + command = ["cilium-agent"] + args = [ + "--config-dir=/tmp/cilium/config-map" + ] + env { + name = "K8S_NODE_NAME" + value_from { + field_ref { + api_version = "v1" + field_path = "spec.nodeName" + } + } + } + env { + name = "CILIUM_K8S_NAMESPACE" + value_from { + field_ref { + api_version = "v1" + field_path = "metadata.namespace" + } + } + } + env { + name = "KUBERNETES_SERVICE_HOST" + value_from { + config_map_key_ref { + name = "in-cluster" + key = "apiserver-host" + } + } + } + env { + name = "KUBERNETES_SERVICE_PORT" + value_from { + config_map_key_ref { + name = "in-cluster" + key = "apiserver-port" + } + } + } + port { + name = "peer-service" + protocol = "TCP" + container_port = 4244 + } + # Metrics + port { + name = "metrics" + protocol = "TCP" + container_port = 9962 + } + port { + name = "envoy-metrics" + protocol = "TCP" + container_port = 9964 + } + port { + name = "hubble-metrics" + protocol = "TCP" + container_port = 9965 + } + # Not yet used, prefer exec's + port { + name = "health" + protocol = "TCP" + container_port = 9876 + } + lifecycle { + pre_stop { + exec { + command = ["/cni-uninstall.sh"] + } + } + } + security_context { + allow_privilege_escalation = true + privileged = true + } + liveness_probe { + exec { + command = ["cilium", "status", "--brief"] + } + initial_delay_seconds = 120 + timeout_seconds = 5 + period_seconds = 30 + success_threshold = 1 + failure_threshold = 10 + } + readiness_probe { + exec { + command = ["cilium", "status", "--brief"] + } + initial_delay_seconds = 5 + timeout_seconds = 5 + period_seconds = 20 + success_threshold = 1 + failure_threshold = 3 + } + # Load kernel modules + volume_mount { + name = "lib-modules" + read_only = true + mount_path = "/lib/modules" + } + # Access iptables concurrently + volume_mount { + name = "xtables-lock" + mount_path = "/run/xtables.lock" + } + # Keep state between restarts + volume_mount { + name = "var-run-cilium" + mount_path = "/var/run/cilium" + } + volume_mount { + name = "sys-fs-bpf" + mount_path = "/sys/fs/bpf" + mount_propagation = "Bidirectional" + } + # Configuration + volume_mount { + name = "config" + read_only = true + mount_path = "/tmp/cilium/config-map" + } + # Install config on host + volume_mount { + name = "cni-conf-dir" + mount_path = "/host/etc/cni/net.d" + } + # Hubble + volume_mount { + name = "hubble-tls" + mount_path = "/var/lib/cilium/tls/hubble" + read_only = true + } + } + termination_grace_period_seconds = 1 + + # Load kernel modules + volume { + name = "lib-modules" + host_path { + path = "/lib/modules" + } + } + # Access iptables concurrently with other processes (e.g. kube-proxy) + volume { + name = "xtables-lock" + host_path { + path = "/run/xtables.lock" + type = "FileOrCreate" + } + } + # Keep state between restarts + volume { + name = "var-run-cilium" + host_path { + path = "/var/run/cilium" + type = "DirectoryOrCreate" + } + } + # Keep state for bpf maps between restarts + volume { + name = "sys-fs-bpf" + host_path { + path = "/sys/fs/bpf" + type = "DirectoryOrCreate" + } + } + # Mount host cgroup2 filesystem + volume { + name = "hostproc" + host_path { + path = "/proc" + type = "Directory" + } + } + volume { + name = "cilium-cgroup" + host_path { + path = "/run/cilium/cgroupv2" + type = "DirectoryOrCreate" + } + } + # Read configuration + volume { + name = "config" + config_map { + name = "cilium" + } + } + # Install CNI plugin and config on host + volume { + name = "cni-bin-dir" + host_path { + path = "/opt/cni/bin" + type = "DirectoryOrCreate" + } + } + volume { + name = "cni-conf-dir" + host_path { + path = "/etc/cni/net.d" + type = "DirectoryOrCreate" + } + } + # Hubble TLS (optional) + volume { + name = "hubble-tls" + projected { + default_mode = "0400" + sources { + secret { + name = "hubble-server-certs" + optional = true + items { + key = "ca.crt" + path = "client-ca.crt" + } + items { + key = "tls.crt" + path = "server.crt" + } + items { + key = "tls.key" + path = "server.key" + } + } + } + } + } + } + } + } +} + diff --git a/addons/cilium/deployment.tf b/addons/cilium/deployment.tf new file mode 100644 index 00000000..886333d5 --- /dev/null +++ b/addons/cilium/deployment.tf @@ -0,0 +1,163 @@ +resource "kubernetes_deployment" "operator" { + wait_for_rollout = false + metadata { + name = "cilium-operator" + namespace = "kube-system" + } + spec { + replicas = 1 + strategy { + type = "RollingUpdate" + rolling_update { + max_unavailable = "1" + } + } + selector { + match_labels = { + name = "cilium-operator" + } + } + template { + metadata { + labels = { + name = "cilium-operator" + } + annotations = { + "prometheus.io/scrape" = "true" + "prometheus.io/port" = "9963" + } + } + spec { + host_network = true + priority_class_name = "system-cluster-critical" + service_account_name = "cilium-operator" + security_context { + seccomp_profile { + type = "RuntimeDefault" + } + } + toleration { + key = "node-role.kubernetes.io/controller" + operator = "Exists" + } + toleration { + key = "node.kubernetes.io/not-ready" + operator = "Exists" + } + topology_spread_constraint { + max_skew = 1 + topology_key = "kubernetes.io/hostname" + when_unsatisfiable = "DoNotSchedule" + label_selector { + match_labels = { + name = "cilium-operator" + } + } + } + automount_service_account_token = true + enable_service_links = false + container { + name = "cilium-operator" + image = "quay.io/cilium/operator-generic:v1.15.5" + command = ["cilium-operator-generic"] + args = [ + "--config-dir=/tmp/cilium/config-map", + "--debug=$(CILIUM_DEBUG)" + ] + env { + name = "K8S_NODE_NAME" + value_from { + field_ref { + api_version = "v1" + field_path = "spec.nodeName" + } + } + } + env { + name = "CILIUM_K8S_NAMESPACE" + value_from { + field_ref { + api_version = "v1" + field_path = "metadata.namespace" + } + } + } + env { + name = "KUBERNETES_SERVICE_HOST" + value_from { + config_map_key_ref { + name = "in-cluster" + key = "apiserver-host" + } + } + } + env { + name = "KUBERNETES_SERVICE_PORT" + value_from { + config_map_key_ref { + name = "in-cluster" + key = "apiserver-port" + } + } + } + env { + name = "CILIUM_DEBUG" + value_from { + config_map_key_ref { + name = "cilium" + key = "debug" + optional = true + } + } + } + port { + name = "metrics" + protocol = "TCP" + host_port = 9963 + container_port = 9963 + } + port { + name = "health" + container_port = 9234 + protocol = "TCP" + } + liveness_probe { + http_get { + scheme = "HTTP" + host = "127.0.0.1" + port = "9234" + path = "/healthz" + } + initial_delay_seconds = 60 + timeout_seconds = 3 + period_seconds = 10 + } + readiness_probe { + http_get { + scheme = "HTTP" + host = "127.0.0.1" + port = "9234" + path = "/healthz" + } + timeout_seconds = 3 + period_seconds = 15 + failure_threshold = 5 + } + volume_mount { + name = "config" + read_only = true + mount_path = "/tmp/cilium/config-map" + } + } + + volume { + name = "config" + config_map { + name = "cilium" + } + } + } + } + } +} + diff --git a/addons/cilium/service-account.tf b/addons/cilium/service-account.tf new file mode 100644 index 00000000..5729dcd8 --- /dev/null +++ b/addons/cilium/service-account.tf @@ -0,0 +1,15 @@ +resource "kubernetes_service_account" "operator" { + metadata { + name = "cilium-operator" + namespace = "kube-system" + } + automount_service_account_token = false +} + +resource "kubernetes_service_account" "agent" { + metadata { + name = "cilium-agent" + namespace = "kube-system" + } + automount_service_account_token = false +} diff --git a/addons/cilium/variables.tf b/addons/cilium/variables.tf new file mode 100644 index 00000000..6d269fd2 --- /dev/null +++ b/addons/cilium/variables.tf @@ -0,0 +1,17 @@ +variable "pod_cidr" { + type = string + description = "CIDR IP range to assign Kubernetes pods" + default = "10.2.0.0/16" +} + +variable "daemonset_tolerations" { + type = list(string) + description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])" + default = [] +} + +variable "enable_hubble" { + type = bool + description = "Run the embedded Hubble Server and mount hubble-server-certs Secret" + default = true +} diff --git a/addons/cilium/versions.tf b/addons/cilium/versions.tf new file mode 100644 index 00000000..ad7f7415 --- /dev/null +++ b/addons/cilium/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.8" + } + } +} diff --git a/addons/coredns/cluster-role.tf b/addons/coredns/cluster-role.tf new file mode 100644 index 00000000..3cc095e8 --- /dev/null +++ b/addons/coredns/cluster-role.tf @@ -0,0 +1,37 @@ +resource "kubernetes_cluster_role" "coredns" { + metadata { + name = "system:coredns" + } + rule { + api_groups = [""] + resources = [ + "endpoints", + "services", + "pods", + "namespaces", + ] + verbs = [ + "list", + "watch", + ] + } + rule { + api_groups = [""] + resources = [ + "nodes", + ] + verbs = [ + "get", + ] + } + rule { + api_groups = ["discovery.k8s.io"] + resources = [ + "endpointslices", + ] + verbs = [ + "list", + "watch", + ] + } +} diff --git a/addons/coredns/config.tf b/addons/coredns/config.tf new file mode 100644 index 00000000..e1a614cb --- /dev/null +++ b/addons/coredns/config.tf @@ -0,0 +1,30 @@ +resource "kubernetes_config_map" "coredns" { + metadata { + name = "coredns" + namespace = "kube-system" + } + data = { + "Corefile" = <<-EOF + .:53 { + errors + health { + lameduck 5s + } + ready + log . { + class error + } + kubernetes ${var.cluster_domain_suffix} in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } + EOF + } +} diff --git a/addons/coredns/deployment.tf b/addons/coredns/deployment.tf new file mode 100644 index 00000000..f52bc6c7 --- /dev/null +++ b/addons/coredns/deployment.tf @@ -0,0 +1,151 @@ +resource "kubernetes_deployment" "coredns" { + wait_for_rollout = false + metadata { + name = "coredns" + namespace = "kube-system" + labels = { + k8s-app = "coredns" + "kubernetes.io/name" = "CoreDNS" + } + } + spec { + replicas = var.replicas + strategy { + type = "RollingUpdate" + rolling_update { + max_unavailable = "1" + } + } + selector { + match_labels = { + k8s-app = "coredns" + tier = "control-plane" + } + } + template { + metadata { + labels = { + k8s-app = "coredns" + tier = "control-plane" + } + } + spec { + affinity { + node_affinity { + preferred_during_scheduling_ignored_during_execution { + weight = 100 + preference { + match_expressions { + key = "node.kubernetes.io/controller" + operator = "Exists" + } + } + } + } + pod_anti_affinity { + preferred_during_scheduling_ignored_during_execution { + weight = 100 + pod_affinity_term { + label_selector { + match_expressions { + key = "tier" + operator = "In" + values = ["control-plane"] + } + match_expressions { + key = "k8s-app" + operator = "In" + values = ["coredns"] + } + } + topology_key = "kubernetes.io/hostname" + } + } + } + } + dns_policy = "Default" + priority_class_name = "system-cluster-critical" + security_context { + seccomp_profile { + type = "RuntimeDefault" + } + } + service_account_name = "coredns" + toleration { + key = "node-role.kubernetes.io/controller" + effect = "NoSchedule" + } + container { + name = "coredns" + image = "registry.k8s.io/coredns/coredns:v1.11.1" + args = ["-conf", "/etc/coredns/Corefile"] + port { + name = "dns" + container_port = 53 + protocol = "UDP" + } + port { + name = "dns-tcp" + container_port = 53 + protocol = "TCP" + } + port { + name = "metrics" + container_port = 9153 + protocol = "TCP" + } + resources { + requests = { + cpu = "100m" + memory = "70Mi" + } + limits = { + memory = "170Mi" + } + } + security_context { + capabilities { + add = ["NET_BIND_SERVICE"] + drop = ["all"] + } + read_only_root_filesystem = true + } + liveness_probe { + http_get { + path = "/health" + port = "8080" + scheme = "HTTP" + } + initial_delay_seconds = 60 + timeout_seconds = 5 + success_threshold = 1 + failure_threshold = 5 + } + readiness_probe { + http_get { + path = "/ready" + port = "8181" + scheme = "HTTP" + } + } + volume_mount { + name = "config" + mount_path = "/etc/coredns" + read_only = true + } + } + volume { + name = "config" + config_map { + name = "coredns" + items { + key = "Corefile" + path = "Corefile" + } + } + } + } + } + } +} + diff --git a/addons/coredns/service-account.tf b/addons/coredns/service-account.tf new file mode 100644 index 00000000..61a124e1 --- /dev/null +++ b/addons/coredns/service-account.tf @@ -0,0 +1,24 @@ +resource "kubernetes_service_account" "coredns" { + metadata { + name = "coredns" + namespace = "kube-system" + } + automount_service_account_token = false +} + + +resource "kubernetes_cluster_role_binding" "coredns" { + metadata { + name = "system:coredns" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "system:coredns" + } + subject { + kind = "ServiceAccount" + name = "coredns" + namespace = "kube-system" + } +} diff --git a/addons/coredns/service.tf b/addons/coredns/service.tf new file mode 100644 index 00000000..9bcf59d5 --- /dev/null +++ b/addons/coredns/service.tf @@ -0,0 +1,31 @@ +resource "kubernetes_service" "coredns" { + metadata { + name = "coredns" + namespace = "kube-system" + labels = { + "k8s-app" = "coredns" + "kubernetes.io/name" = "CoreDNS" + } + annotations = { + "prometheus.io/scrape" = "true" + "prometheus.io/port" = "9153" + } + } + spec { + type = "ClusterIP" + cluster_ip = var.cluster_dns_service_ip + selector = { + k8s-app = "coredns" + } + port { + name = "dns" + protocol = "UDP" + port = 53 + } + port { + name = "dns-tcp" + protocol = "TCP" + port = 53 + } + } +} diff --git a/addons/coredns/variables.tf b/addons/coredns/variables.tf new file mode 100644 index 00000000..1bd8c4da --- /dev/null +++ b/addons/coredns/variables.tf @@ -0,0 +1,15 @@ +variable "replicas" { + type = number + description = "CoreDNS replica count" + default = 2 +} + +variable "cluster_dns_service_ip" { + description = "Must be set to `cluster_dns_service_ip` output by cluster" + default = "10.3.0.10" +} + +variable "cluster_domain_suffix" { + description = "Must be set to `cluster_domain_suffix` output by cluster" + default = "cluster.local" +} diff --git a/addons/coredns/versions.tf b/addons/coredns/versions.tf new file mode 100644 index 00000000..5d8ee2d7 --- /dev/null +++ b/addons/coredns/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.8" + } + } +} + diff --git a/addons/flannel/cluster-role-binding.tf b/addons/flannel/cluster-role-binding.tf new file mode 100644 index 00000000..3a7aa9e7 --- /dev/null +++ b/addons/flannel/cluster-role-binding.tf @@ -0,0 +1,18 @@ +resource "kubernetes_cluster_role_binding" "flannel" { + metadata { + name = "flannel" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "flannel" + } + + subject { + kind = "ServiceAccount" + name = "flannel" + namespace = "kube-system" + } +} + diff --git a/addons/flannel/cluster-role.tf b/addons/flannel/cluster-role.tf new file mode 100644 index 00000000..9c2d461d --- /dev/null +++ b/addons/flannel/cluster-role.tf @@ -0,0 +1,24 @@ +resource "kubernetes_cluster_role" "flannel" { + metadata { + name = "flannel" + } + + rule { + api_groups = [""] + resources = ["pods"] + verbs = ["get"] + } + + rule { + api_groups = [""] + resources = ["nodes"] + verbs = ["list", "watch"] + } + + rule { + api_groups = [""] + resources = ["nodes/status"] + verbs = ["patch"] + } +} + diff --git a/addons/flannel/config.tf b/addons/flannel/config.tf new file mode 100644 index 00000000..4a0f8cc9 --- /dev/null +++ b/addons/flannel/config.tf @@ -0,0 +1,44 @@ +resource "kubernetes_config_map" "config" { + metadata { + name = "flannel-config" + namespace = "kube-system" + labels = { + k8s-app = "flannel" + tier = "node" + } + } + + data = { + "cni-conf.json" = <<-EOF + { + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + EOF + "net-conf.json" = <<-EOF + { + "Network": "${var.pod_cidr}", + "Backend": { + "Type": "vxlan", + "Port": 4789 + } + } + EOF + } +} + diff --git a/addons/flannel/daemonset.tf b/addons/flannel/daemonset.tf new file mode 100644 index 00000000..f184072f --- /dev/null +++ b/addons/flannel/daemonset.tf @@ -0,0 +1,167 @@ +resource "kubernetes_daemonset" "flannel" { + metadata { + name = "flannel" + namespace = "kube-system" + labels = { + k8s-app = "flannel" + } + } + spec { + strategy { + type = "RollingUpdate" + rolling_update { + max_unavailable = "1" + } + } + selector { + match_labels = { + k8s-app = "flannel" + } + } + template { + metadata { + labels = { + k8s-app = "flannel" + } + } + spec { + host_network = true + priority_class_name = "system-node-critical" + service_account_name = "flannel" + security_context { + seccomp_profile { + type = "RuntimeDefault" + } + } + toleration { + key = "node-role.kubernetes.io/controller" + operator = "Exists" + } + toleration { + key = "node.kubernetes.io/not-ready" + operator = "Exists" + } + dynamic "toleration" { + for_each = var.daemonset_tolerations + content { + key = toleration.value + operator = "Exists" + } + } + init_container { + name = "install-cni" + image = "quay.io/poseidon/flannel-cni:v0.4.2" + command = ["/install-cni.sh"] + env { + name = "CNI_NETWORK_CONFIG" + value_from { + config_map_key_ref { + name = "flannel-config" + key = "cni-conf.json" + } + } + } + volume_mount { + name = "cni-bin-dir" + mount_path = "/host/opt/cni/bin/" + } + volume_mount { + name = "cni-conf-dir" + mount_path = "/host/etc/cni/net.d" + } + } + + container { + name = "flannel" + image = "docker.io/flannel/flannel:v0.25.1" + command = [ + "/opt/bin/flanneld", + "--ip-masq", + "--kube-subnet-mgr", + "--iface=$(POD_IP)" + ] + env { + name = "POD_NAME" + value_from { + field_ref { + field_path = "metadata.name" + } + } + } + env { + name = "POD_NAMESPACE" + value_from { + field_ref { + field_path = "metadata.namespace" + } + } + } + env { + name = "POD_IP" + value_from { + field_ref { + field_path = "status.podIP" + } + } + } + security_context { + privileged = true + } + resources { + requests = { + cpu = "100m" + } + } + volume_mount { + name = "flannel-config" + mount_path = "/etc/kube-flannel/" + } + volume_mount { + name = "run-flannel" + mount_path = "/run/flannel" + } + volume_mount { + name = "xtables-lock" + mount_path = "/run/xtables.lock" + } + } + + volume { + name = "flannel-config" + config_map { + name = "flannel-config" + } + } + volume { + name = "run-flannel" + host_path { + path = "/run/flannel" + } + } + # Used by install-cni + volume { + name = "cni-bin-dir" + host_path { + path = "/opt/cni/bin" + } + } + volume { + name = "cni-conf-dir" + host_path { + path = "/etc/cni/net.d" + type = "DirectoryOrCreate" + } + } + # Acces iptables concurrently + volume { + name = "xtables-lock" + host_path { + path = "/run/xtables.lock" + type = "FileOrCreate" + } + } + } + } + } +} + diff --git a/addons/flannel/service-account.tf b/addons/flannel/service-account.tf new file mode 100644 index 00000000..b3b81c13 --- /dev/null +++ b/addons/flannel/service-account.tf @@ -0,0 +1,7 @@ +resource "kubernetes_service_account" "flannel" { + metadata { + name = "flannel" + namespace = "kube-system" + } +} + diff --git a/addons/flannel/variables.tf b/addons/flannel/variables.tf new file mode 100644 index 00000000..7a6ad2fc --- /dev/null +++ b/addons/flannel/variables.tf @@ -0,0 +1,11 @@ +variable "pod_cidr" { + type = string + description = "CIDR IP range to assign Kubernetes pods" + default = "10.2.0.0/16" +} + +variable "daemonset_tolerations" { + type = list(string) + description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])" + default = [] +} diff --git a/addons/flannel/versions.tf b/addons/flannel/versions.tf new file mode 100644 index 00000000..ad7f7415 --- /dev/null +++ b/addons/flannel/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.8" + } + } +}