mirror of
https://github.com/puppetmaster/typhoon.git
synced 2025-01-06 05:59:35 +01:00
Add Terraform modules for CoreDNS, Cilium, and flannel
* With the new component system, these components can be managed independent from the cluster and rolled or edited in advanced ways
This commit is contained in:
Dalton Hubble
parent
533ace7011
commit
7b8a51070f
@ -8,9 +8,10 @@ Notable changes between versions.
|
||||
|
||||
* Kubernetes [v1.30.1](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.30.md#v1301)
|
||||
* Add firewall rules and security group rules for Cilium and Hubble metrics ([#1449](https://github.com/poseidon/typhoon/pull/1449))
|
||||
* Introduce `components` variabe to enable/disable/configure pre-installed components ([#1453](https://github.com/poseidon/typhoon/pull/1453))
|
||||
* Update Cilium from v1.15.3 to [v1.15.5](https://github.com/cilium/cilium/releases/tag/v1.15.5)
|
||||
* Update flannel from v0.24.4 to [v0.25.1](https://github.com/flannel-io/flannel/releases/tag/v0.25.1)
|
||||
* Introduce `components` variabe to enable/disable/configure pre-installed components ([#1453](https://github.com/poseidon/typhoon/pull/1453))
|
||||
* Add Terraform modules for `coredns`, `cilium`, and `flannel` components
|
||||
|
||||
### Azure
|
||||
|
||||
|
||||
10
README.md
10
README.md
@ -26,7 +26,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
|
||||
|
||||
## Modules
|
||||
|
||||
Typhoon provides a Terraform Module for each supported operating system and platform.
|
||||
Typhoon provides a Terraform Module for defining a Kubernetes cluster on each supported operating system and platform.
|
||||
|
||||
Typhoon is available for [Fedora CoreOS](https://getfedora.org/coreos/).
|
||||
|
||||
@ -57,6 +57,14 @@ Typhoon is available for [Flatcar Linux](https://www.flatcar-linux.org/releases/
|
||||
| AWS | Flatcar Linux (ARM64) | [aws/flatcar-linux/kubernetes](aws/flatcar-linux/kubernetes) | alpha |
|
||||
| Azure | Flatcar Linux (ARM64) | [azure/flatcar-linux/kubernetes](azure/flatcar-linux/kubernetes) | alpha |
|
||||
|
||||
Typhoon also provides Terraform Modules for optionally managing individual components applied onto clusters.
|
||||
|
||||
| Name | Terraform Module | Status |
|
||||
|---------|------------------|--------|
|
||||
| CoreDNS | [addons/coredns](addons/coredns) | beta |
|
||||
| Cilium | [addons/cilium](addons/cilium) | beta |
|
||||
| flannel | [addons/flannel](addons/flannel) | beta |
|
||||
|
||||
## Documentation
|
||||
|
||||
* [Docs](https://typhoon.psdn.io)
|
||||
|
||||
36
addons/cilium/cluster-role-binding.tf
Normal file
36
addons/cilium/cluster-role-binding.tf
Normal file
@ -0,0 +1,36 @@
|
||||
resource "kubernetes_cluster_role_binding" "operator" {
|
||||
metadata {
|
||||
name = "cilium-operator"
|
||||
}
|
||||
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "ClusterRole"
|
||||
name = "cilium-operator"
|
||||
}
|
||||
|
||||
subject {
|
||||
kind = "ServiceAccount"
|
||||
name = "cilium-operator"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cluster_role_binding" "agent" {
|
||||
metadata {
|
||||
name = "cilium-agent"
|
||||
}
|
||||
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "ClusterRole"
|
||||
name = "cilium-agent"
|
||||
}
|
||||
|
||||
subject {
|
||||
kind = "ServiceAccount"
|
||||
name = "cilium-agent"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
112
addons/cilium/cluster-role.tf
Normal file
112
addons/cilium/cluster-role.tf
Normal file
@ -0,0 +1,112 @@
|
||||
resource "kubernetes_cluster_role" "operator" {
|
||||
metadata {
|
||||
name = "cilium-operator"
|
||||
}
|
||||
|
||||
# detect and restart [core|kube]dns pods on startup
|
||||
rule {
|
||||
verbs = ["get", "list", "watch", "delete"]
|
||||
api_groups = [""]
|
||||
resources = ["pods"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["list", "watch"]
|
||||
api_groups = [""]
|
||||
resources = ["nodes"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["patch"]
|
||||
api_groups = [""]
|
||||
resources = ["nodes", "nodes/status"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["get", "list", "watch"]
|
||||
api_groups = ["discovery.k8s.io"]
|
||||
resources = ["endpointslices"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["get", "list", "watch"]
|
||||
api_groups = [""]
|
||||
resources = ["services"]
|
||||
}
|
||||
|
||||
# Perform LB IP allocation for BGP
|
||||
rule {
|
||||
verbs = ["update"]
|
||||
api_groups = [""]
|
||||
resources = ["services/status"]
|
||||
}
|
||||
|
||||
# Perform the translation of a CNP that contains `ToGroup` to its endpoints
|
||||
rule {
|
||||
verbs = ["get", "list", "watch"]
|
||||
api_groups = [""]
|
||||
resources = ["services", "endpoints", "namespaces"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["*"]
|
||||
api_groups = ["cilium.io"]
|
||||
resources = ["ciliumnetworkpolicies", "ciliumnetworkpolicies/status", "ciliumnetworkpolicies/finalizers", "ciliumclusterwidenetworkpolicies", "ciliumclusterwidenetworkpolicies/status", "ciliumclusterwidenetworkpolicies/finalizers", "ciliumendpoints", "ciliumendpoints/status", "ciliumendpoints/finalizers", "ciliumnodes", "ciliumnodes/status", "ciliumnodes/finalizers", "ciliumidentities", "ciliumidentities/status", "ciliumidentities/finalizers", "ciliumlocalredirectpolicies", "ciliumlocalredirectpolicies/status", "ciliumlocalredirectpolicies/finalizers", "ciliumendpointslices", "ciliumloadbalancerippools", "ciliumloadbalancerippools/status", "ciliumcidrgroups", "ciliuml2announcementpolicies", "ciliuml2announcementpolicies/status", "ciliumpodippools"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["create", "get", "list", "update", "watch"]
|
||||
api_groups = ["apiextensions.k8s.io"]
|
||||
resources = ["customresourcedefinitions"]
|
||||
}
|
||||
|
||||
# Cilium leader elects if among multiple operator replicas
|
||||
rule {
|
||||
verbs = ["create", "get", "update"]
|
||||
api_groups = ["coordination.k8s.io"]
|
||||
resources = ["leases"]
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cluster_role" "agent" {
|
||||
metadata {
|
||||
name = "cilium-agent"
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["get", "list", "watch"]
|
||||
api_groups = ["networking.k8s.io"]
|
||||
resources = ["networkpolicies"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["get", "list", "watch"]
|
||||
api_groups = ["discovery.k8s.io"]
|
||||
resources = ["endpointslices"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["get", "list", "watch"]
|
||||
api_groups = [""]
|
||||
resources = ["namespaces", "services", "pods", "endpoints", "nodes"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["patch"]
|
||||
api_groups = [""]
|
||||
resources = ["nodes/status"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["create", "get", "list", "watch", "update"]
|
||||
api_groups = ["apiextensions.k8s.io"]
|
||||
resources = ["customresourcedefinitions"]
|
||||
}
|
||||
|
||||
rule {
|
||||
verbs = ["*"]
|
||||
api_groups = ["cilium.io"]
|
||||
resources = ["ciliumnetworkpolicies", "ciliumnetworkpolicies/status", "ciliumclusterwidenetworkpolicies", "ciliumclusterwidenetworkpolicies/status", "ciliumendpoints", "ciliumendpoints/status", "ciliumnodes", "ciliumnodes/status", "ciliumidentities", "ciliumidentities/status", "ciliumlocalredirectpolicies", "ciliumlocalredirectpolicies/status", "ciliumegressnatpolicies", "ciliumendpointslices", "ciliumcidrgroups", "ciliuml2announcementpolicies", "ciliuml2announcementpolicies/status", "ciliumpodippools"]
|
||||
}
|
||||
}
|
||||
|
||||
196
addons/cilium/config.tf
Normal file
196
addons/cilium/config.tf
Normal file
@ -0,0 +1,196 @@
|
||||
resource "kubernetes_config_map" "cilium" {
|
||||
metadata {
|
||||
name = "cilium"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
data = {
|
||||
# Identity allocation mode selects how identities are shared between cilium
|
||||
# nodes by setting how they are stored. The options are "crd" or "kvstore".
|
||||
# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
|
||||
# These can be queried with:
|
||||
# kubectl get ciliumid
|
||||
# - "kvstore" stores identities in a kvstore, etcd or consul, that is
|
||||
# configured below. Cilium versions before 1.6 supported only the kvstore
|
||||
# backend. Upgrades from these older cilium versions should continue using
|
||||
# the kvstore by commenting out the identity-allocation-mode below, or
|
||||
# setting it to "kvstore".
|
||||
identity-allocation-mode = "crd"
|
||||
cilium-endpoint-gc-interval = "5m0s"
|
||||
nodes-gc-interval = "5m0s"
|
||||
|
||||
# If you want to run cilium in debug mode change this value to true
|
||||
debug = "false"
|
||||
# The agent can be put into the following three policy enforcement modes
|
||||
# default, always and never.
|
||||
# https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes
|
||||
enable-policy = "default"
|
||||
|
||||
# Prometheus
|
||||
enable-metrics = "true"
|
||||
prometheus-serve-addr = ":9962"
|
||||
operator-prometheus-serve-addr = ":9963"
|
||||
proxy-prometheus-port = "9964" # envoy
|
||||
|
||||
# Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
|
||||
# address.
|
||||
enable-ipv4 = "true"
|
||||
|
||||
# Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
|
||||
# address.
|
||||
enable-ipv6 = "false"
|
||||
|
||||
# Enable probing for a more efficient clock source for the BPF datapath
|
||||
enable-bpf-clock-probe = "true"
|
||||
|
||||
# Enable use of transparent proxying mechanisms (Linux 5.7+)
|
||||
enable-bpf-tproxy = "false"
|
||||
|
||||
# If you want cilium monitor to aggregate tracing for packets, set this level
|
||||
# to "low", "medium", or "maximum". The higher the level, the less packets
|
||||
# that will be seen in monitor output.
|
||||
monitor-aggregation = "medium"
|
||||
|
||||
# The monitor aggregation interval governs the typical time between monitor
|
||||
# notification events for each allowed connection.
|
||||
#
|
||||
# Only effective when monitor aggregation is set to "medium" or higher.
|
||||
monitor-aggregation-interval = "5s"
|
||||
|
||||
# The monitor aggregation flags determine which TCP flags which, upon the
|
||||
# first observation, cause monitor notifications to be generated.
|
||||
#
|
||||
# Only effective when monitor aggregation is set to "medium" or higher.
|
||||
monitor-aggregation-flags = "all"
|
||||
|
||||
# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
|
||||
# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
|
||||
bpf-map-dynamic-size-ratio = "0.0025"
|
||||
# bpf-policy-map-max specified the maximum number of entries in endpoint
|
||||
# policy map (per endpoint)
|
||||
bpf-policy-map-max = "16384"
|
||||
# bpf-lb-map-max specifies the maximum number of entries in bpf lb service,
|
||||
# backend and affinity maps.
|
||||
bpf-lb-map-max = "65536"
|
||||
|
||||
# Pre-allocation of map entries allows per-packet latency to be reduced, at
|
||||
# the expense of up-front memory allocation for the entries in the maps. The
|
||||
# default value below will minimize memory usage in the default installation;
|
||||
# users who are sensitive to latency may consider setting this to "true".
|
||||
#
|
||||
# This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
|
||||
# this option and behave as though it is set to "true".
|
||||
#
|
||||
# If this value is modified, then during the next Cilium startup the restore
|
||||
# of existing endpoints and tracking of ongoing connections may be disrupted.
|
||||
# As a result, reply packets may be dropped and the load-balancing decisions
|
||||
# for established connections may change.
|
||||
#
|
||||
# If this option is set to "false" during an upgrade from 1.3 or earlier to
|
||||
# 1.4 or later, then it may cause one-time disruptions during the upgrade.
|
||||
preallocate-bpf-maps = "false"
|
||||
|
||||
# Name of the cluster. Only relevant when building a mesh of clusters.
|
||||
cluster-name = "default"
|
||||
# Unique ID of the cluster. Must be unique across all conneted clusters and
|
||||
# in the range of 1 and 255. Only relevant when building a mesh of clusters.
|
||||
cluster-id = "0"
|
||||
|
||||
# Encapsulation mode for communication between nodes
|
||||
# Possible values:
|
||||
# - disabled
|
||||
# - vxlan (default)
|
||||
# - geneve
|
||||
routing-mode = "tunnel"
|
||||
tunnel = "vxlan"
|
||||
# Enables L7 proxy for L7 policy enforcement and visibility
|
||||
enable-l7-proxy = "true"
|
||||
|
||||
auto-direct-node-routes = "false"
|
||||
|
||||
# enableXTSocketFallback enables the fallback compatibility solution
|
||||
# when the xt_socket kernel module is missing and it is needed for
|
||||
# the datapath L7 redirection to work properly. See documentation
|
||||
# for details on when this can be disabled:
|
||||
# http://docs.cilium.io/en/latest/install/system_requirements/#admin-kernel-version.
|
||||
enable-xt-socket-fallback = "true"
|
||||
|
||||
# installIptablesRules enables installation of iptables rules to allow for
|
||||
# TPROXY (L7 proxy injection), itpables based masquerading and compatibility
|
||||
# with kube-proxy. See documentation for details on when this can be
|
||||
# disabled.
|
||||
install-iptables-rules = "true"
|
||||
|
||||
# masquerade traffic leaving the node destined for outside
|
||||
enable-ipv4-masquerade = "true"
|
||||
enable-ipv6-masquerade = "false"
|
||||
|
||||
# bpfMasquerade enables masquerading with BPF instead of iptables
|
||||
enable-bpf-masquerade = "true"
|
||||
|
||||
# kube-proxy
|
||||
kube-proxy-replacement = "false"
|
||||
kube-proxy-replacement-healthz-bind-address = ""
|
||||
enable-session-affinity = "true"
|
||||
|
||||
# ClusterIPs from host namespace
|
||||
bpf-lb-sock = "true"
|
||||
# ClusterIPs from external nodes
|
||||
bpf-lb-external-clusterip = "true"
|
||||
|
||||
# NodePort
|
||||
enable-node-port = "true"
|
||||
enable-health-check-nodeport = "false"
|
||||
|
||||
# ExternalIPs
|
||||
enable-external-ips = "true"
|
||||
|
||||
# HostPort
|
||||
enable-host-port = "true"
|
||||
|
||||
# IPAM
|
||||
ipam = "cluster-pool"
|
||||
disable-cnp-status-updates = "true"
|
||||
cluster-pool-ipv4-cidr = "${var.pod_cidr}"
|
||||
cluster-pool-ipv4-mask-size = "24"
|
||||
|
||||
# Health
|
||||
agent-health-port = "9876"
|
||||
enable-health-checking = "true"
|
||||
enable-endpoint-health-checking = "true"
|
||||
|
||||
# Identity
|
||||
enable-well-known-identities = "false"
|
||||
enable-remote-node-identity = "true"
|
||||
|
||||
# Hubble server
|
||||
enable-hubble = var.enable_hubble
|
||||
hubble-disable-tls = "false"
|
||||
hubble-listen-address = ":4244"
|
||||
hubble-socket-path = "/var/run/cilium/hubble.sock"
|
||||
hubble-tls-client-ca-files = "/var/lib/cilium/tls/hubble/client-ca.crt"
|
||||
hubble-tls-cert-file = "/var/lib/cilium/tls/hubble/server.crt"
|
||||
hubble-tls-key-file = "/var/lib/cilium/tls/hubble/server.key"
|
||||
hubble-export-file-max-backups = "5"
|
||||
hubble-export-file-max-size-mb = "10"
|
||||
|
||||
# Hubble metrics
|
||||
hubble-metrics-server = ":9965"
|
||||
hubble-metrics = "dns drop tcp flow port-distribution icmp httpV2"
|
||||
enable-hubble-open-metrics = "false"
|
||||
|
||||
|
||||
# Misc
|
||||
enable-bandwidth-manager = "false"
|
||||
enable-local-redirect-policy = "false"
|
||||
policy-audit-mode = "false"
|
||||
operator-api-serve-addr = "127.0.0.1:9234"
|
||||
enable-l2-neigh-discovery = "true"
|
||||
enable-k8s-terminating-endpoint = "true"
|
||||
enable-k8s-networkpolicy = "true"
|
||||
external-envoy-proxy = "false"
|
||||
write-cni-conf-when-ready = "/host/etc/cni/net.d/05-cilium.conflist"
|
||||
cni-exclusive = "true"
|
||||
cni-log-file = "/var/run/cilium/cilium-cni.log"
|
||||
}
|
||||
}
|
||||
|
||||
379
addons/cilium/daemonset.tf
Normal file
379
addons/cilium/daemonset.tf
Normal file
@ -0,0 +1,379 @@
|
||||
resource "kubernetes_daemonset" "cilium" {
|
||||
wait_for_rollout = false
|
||||
|
||||
metadata {
|
||||
name = "cilium"
|
||||
namespace = "kube-system"
|
||||
labels = {
|
||||
k8s-app = "cilium"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
strategy {
|
||||
type = "RollingUpdate"
|
||||
rolling_update {
|
||||
max_unavailable = "1"
|
||||
}
|
||||
}
|
||||
selector {
|
||||
match_labels = {
|
||||
k8s-app = "cilium-agent"
|
||||
}
|
||||
}
|
||||
template {
|
||||
metadata {
|
||||
labels = {
|
||||
k8s-app = "cilium-agent"
|
||||
}
|
||||
annotations = {
|
||||
"prometheus.io/port" = "9962"
|
||||
"prometheus.io/scrape" = "true"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
host_network = true
|
||||
priority_class_name = "system-node-critical"
|
||||
service_account_name = "cilium-agent"
|
||||
security_context {
|
||||
seccomp_profile {
|
||||
type = "RuntimeDefault"
|
||||
}
|
||||
}
|
||||
toleration {
|
||||
key = "node-role.kubernetes.io/controller"
|
||||
operator = "Exists"
|
||||
}
|
||||
toleration {
|
||||
key = "node.kubernetes.io/not-ready"
|
||||
operator = "Exists"
|
||||
}
|
||||
dynamic "toleration" {
|
||||
for_each = var.daemonset_tolerations
|
||||
content {
|
||||
key = toleration.value
|
||||
operator = "Exists"
|
||||
}
|
||||
}
|
||||
automount_service_account_token = true
|
||||
enable_service_links = false
|
||||
|
||||
# Cilium v1.13.1 starts installing CNI plugins in yet another init container
|
||||
# https://github.com/cilium/cilium/pull/24075
|
||||
init_container {
|
||||
name = "install-cni"
|
||||
image = "quay.io/cilium/cilium:v1.15.5"
|
||||
command = ["/install-plugin.sh"]
|
||||
security_context {
|
||||
allow_privilege_escalation = true
|
||||
privileged = true
|
||||
capabilities {
|
||||
drop = ["ALL"]
|
||||
}
|
||||
}
|
||||
volume_mount {
|
||||
name = "cni-bin-dir"
|
||||
mount_path = "/host/opt/cni/bin"
|
||||
}
|
||||
}
|
||||
|
||||
# Required to mount cgroup2 filesystem on the underlying Kubernetes node.
|
||||
# We use nsenter command with host's cgroup and mount namespaces enabled.
|
||||
init_container {
|
||||
name = "mount-cgroup"
|
||||
image = "quay.io/cilium/cilium:v1.15.5"
|
||||
command = [
|
||||
"sh",
|
||||
"-ec",
|
||||
# The statically linked Go program binary is invoked to avoid any
|
||||
# dependency on utilities like sh and mount that can be missing on certain
|
||||
# distros installed on the underlying host. Copy the binary to the
|
||||
# same directory where we install cilium cni plugin so that exec permissions
|
||||
# are available.
|
||||
"cp /usr/bin/cilium-mount /hostbin/cilium-mount && nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt \"$${BIN_PATH}/cilium-mount\" $CGROUP_ROOT; rm /hostbin/cilium-mount"
|
||||
]
|
||||
env {
|
||||
name = "CGROUP_ROOT"
|
||||
value = "/run/cilium/cgroupv2"
|
||||
}
|
||||
env {
|
||||
name = "BIN_PATH"
|
||||
value = "/opt/cni/bin"
|
||||
}
|
||||
security_context {
|
||||
allow_privilege_escalation = true
|
||||
privileged = true
|
||||
}
|
||||
volume_mount {
|
||||
name = "hostproc"
|
||||
mount_path = "/hostproc"
|
||||
}
|
||||
volume_mount {
|
||||
name = "cni-bin-dir"
|
||||
mount_path = "/hostbin"
|
||||
}
|
||||
}
|
||||
|
||||
init_container {
|
||||
name = "clean-cilium-state"
|
||||
image = "quay.io/cilium/cilium:v1.15.5"
|
||||
command = ["/init-container.sh"]
|
||||
security_context {
|
||||
allow_privilege_escalation = true
|
||||
privileged = true
|
||||
}
|
||||
volume_mount {
|
||||
name = "sys-fs-bpf"
|
||||
mount_path = "/sys/fs/bpf"
|
||||
}
|
||||
volume_mount {
|
||||
name = "var-run-cilium"
|
||||
mount_path = "/var/run/cilium"
|
||||
}
|
||||
# Required to mount cgroup filesystem from the host to cilium agent pod
|
||||
volume_mount {
|
||||
name = "cilium-cgroup"
|
||||
mount_path = "/run/cilium/cgroupv2"
|
||||
mount_propagation = "HostToContainer"
|
||||
}
|
||||
}
|
||||
|
||||
container {
|
||||
name = "cilium-agent"
|
||||
image = "quay.io/cilium/cilium:v1.15.5"
|
||||
command = ["cilium-agent"]
|
||||
args = [
|
||||
"--config-dir=/tmp/cilium/config-map"
|
||||
]
|
||||
env {
|
||||
name = "K8S_NODE_NAME"
|
||||
value_from {
|
||||
field_ref {
|
||||
api_version = "v1"
|
||||
field_path = "spec.nodeName"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "CILIUM_K8S_NAMESPACE"
|
||||
value_from {
|
||||
field_ref {
|
||||
api_version = "v1"
|
||||
field_path = "metadata.namespace"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "KUBERNETES_SERVICE_HOST"
|
||||
value_from {
|
||||
config_map_key_ref {
|
||||
name = "in-cluster"
|
||||
key = "apiserver-host"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "KUBERNETES_SERVICE_PORT"
|
||||
value_from {
|
||||
config_map_key_ref {
|
||||
name = "in-cluster"
|
||||
key = "apiserver-port"
|
||||
}
|
||||
}
|
||||
}
|
||||
port {
|
||||
name = "peer-service"
|
||||
protocol = "TCP"
|
||||
container_port = 4244
|
||||
}
|
||||
# Metrics
|
||||
port {
|
||||
name = "metrics"
|
||||
protocol = "TCP"
|
||||
container_port = 9962
|
||||
}
|
||||
port {
|
||||
name = "envoy-metrics"
|
||||
protocol = "TCP"
|
||||
container_port = 9964
|
||||
}
|
||||
port {
|
||||
name = "hubble-metrics"
|
||||
protocol = "TCP"
|
||||
container_port = 9965
|
||||
}
|
||||
# Not yet used, prefer exec's
|
||||
port {
|
||||
name = "health"
|
||||
protocol = "TCP"
|
||||
container_port = 9876
|
||||
}
|
||||
lifecycle {
|
||||
pre_stop {
|
||||
exec {
|
||||
command = ["/cni-uninstall.sh"]
|
||||
}
|
||||
}
|
||||
}
|
||||
security_context {
|
||||
allow_privilege_escalation = true
|
||||
privileged = true
|
||||
}
|
||||
liveness_probe {
|
||||
exec {
|
||||
command = ["cilium", "status", "--brief"]
|
||||
}
|
||||
initial_delay_seconds = 120
|
||||
timeout_seconds = 5
|
||||
period_seconds = 30
|
||||
success_threshold = 1
|
||||
failure_threshold = 10
|
||||
}
|
||||
readiness_probe {
|
||||
exec {
|
||||
command = ["cilium", "status", "--brief"]
|
||||
}
|
||||
initial_delay_seconds = 5
|
||||
timeout_seconds = 5
|
||||
period_seconds = 20
|
||||
success_threshold = 1
|
||||
failure_threshold = 3
|
||||
}
|
||||
# Load kernel modules
|
||||
volume_mount {
|
||||
name = "lib-modules"
|
||||
read_only = true
|
||||
mount_path = "/lib/modules"
|
||||
}
|
||||
# Access iptables concurrently
|
||||
volume_mount {
|
||||
name = "xtables-lock"
|
||||
mount_path = "/run/xtables.lock"
|
||||
}
|
||||
# Keep state between restarts
|
||||
volume_mount {
|
||||
name = "var-run-cilium"
|
||||
mount_path = "/var/run/cilium"
|
||||
}
|
||||
volume_mount {
|
||||
name = "sys-fs-bpf"
|
||||
mount_path = "/sys/fs/bpf"
|
||||
mount_propagation = "Bidirectional"
|
||||
}
|
||||
# Configuration
|
||||
volume_mount {
|
||||
name = "config"
|
||||
read_only = true
|
||||
mount_path = "/tmp/cilium/config-map"
|
||||
}
|
||||
# Install config on host
|
||||
volume_mount {
|
||||
name = "cni-conf-dir"
|
||||
mount_path = "/host/etc/cni/net.d"
|
||||
}
|
||||
# Hubble
|
||||
volume_mount {
|
||||
name = "hubble-tls"
|
||||
mount_path = "/var/lib/cilium/tls/hubble"
|
||||
read_only = true
|
||||
}
|
||||
}
|
||||
termination_grace_period_seconds = 1
|
||||
|
||||
# Load kernel modules
|
||||
volume {
|
||||
name = "lib-modules"
|
||||
host_path {
|
||||
path = "/lib/modules"
|
||||
}
|
||||
}
|
||||
# Access iptables concurrently with other processes (e.g. kube-proxy)
|
||||
volume {
|
||||
name = "xtables-lock"
|
||||
host_path {
|
||||
path = "/run/xtables.lock"
|
||||
type = "FileOrCreate"
|
||||
}
|
||||
}
|
||||
# Keep state between restarts
|
||||
volume {
|
||||
name = "var-run-cilium"
|
||||
host_path {
|
||||
path = "/var/run/cilium"
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
# Keep state for bpf maps between restarts
|
||||
volume {
|
||||
name = "sys-fs-bpf"
|
||||
host_path {
|
||||
path = "/sys/fs/bpf"
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
# Mount host cgroup2 filesystem
|
||||
volume {
|
||||
name = "hostproc"
|
||||
host_path {
|
||||
path = "/proc"
|
||||
type = "Directory"
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "cilium-cgroup"
|
||||
host_path {
|
||||
path = "/run/cilium/cgroupv2"
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
# Read configuration
|
||||
volume {
|
||||
name = "config"
|
||||
config_map {
|
||||
name = "cilium"
|
||||
}
|
||||
}
|
||||
# Install CNI plugin and config on host
|
||||
volume {
|
||||
name = "cni-bin-dir"
|
||||
host_path {
|
||||
path = "/opt/cni/bin"
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "cni-conf-dir"
|
||||
host_path {
|
||||
path = "/etc/cni/net.d"
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
# Hubble TLS (optional)
|
||||
volume {
|
||||
name = "hubble-tls"
|
||||
projected {
|
||||
default_mode = "0400"
|
||||
sources {
|
||||
secret {
|
||||
name = "hubble-server-certs"
|
||||
optional = true
|
||||
items {
|
||||
key = "ca.crt"
|
||||
path = "client-ca.crt"
|
||||
}
|
||||
items {
|
||||
key = "tls.crt"
|
||||
path = "server.crt"
|
||||
}
|
||||
items {
|
||||
key = "tls.key"
|
||||
path = "server.key"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
163
addons/cilium/deployment.tf
Normal file
163
addons/cilium/deployment.tf
Normal file
@ -0,0 +1,163 @@
|
||||
resource "kubernetes_deployment" "operator" {
|
||||
wait_for_rollout = false
|
||||
metadata {
|
||||
name = "cilium-operator"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
spec {
|
||||
replicas = 1
|
||||
strategy {
|
||||
type = "RollingUpdate"
|
||||
rolling_update {
|
||||
max_unavailable = "1"
|
||||
}
|
||||
}
|
||||
selector {
|
||||
match_labels = {
|
||||
name = "cilium-operator"
|
||||
}
|
||||
}
|
||||
template {
|
||||
metadata {
|
||||
labels = {
|
||||
name = "cilium-operator"
|
||||
}
|
||||
annotations = {
|
||||
"prometheus.io/scrape" = "true"
|
||||
"prometheus.io/port" = "9963"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
host_network = true
|
||||
priority_class_name = "system-cluster-critical"
|
||||
service_account_name = "cilium-operator"
|
||||
security_context {
|
||||
seccomp_profile {
|
||||
type = "RuntimeDefault"
|
||||
}
|
||||
}
|
||||
toleration {
|
||||
key = "node-role.kubernetes.io/controller"
|
||||
operator = "Exists"
|
||||
}
|
||||
toleration {
|
||||
key = "node.kubernetes.io/not-ready"
|
||||
operator = "Exists"
|
||||
}
|
||||
topology_spread_constraint {
|
||||
max_skew = 1
|
||||
topology_key = "kubernetes.io/hostname"
|
||||
when_unsatisfiable = "DoNotSchedule"
|
||||
label_selector {
|
||||
match_labels = {
|
||||
name = "cilium-operator"
|
||||
}
|
||||
}
|
||||
}
|
||||
automount_service_account_token = true
|
||||
enable_service_links = false
|
||||
container {
|
||||
name = "cilium-operator"
|
||||
image = "quay.io/cilium/operator-generic:v1.15.5"
|
||||
command = ["cilium-operator-generic"]
|
||||
args = [
|
||||
"--config-dir=/tmp/cilium/config-map",
|
||||
"--debug=$(CILIUM_DEBUG)"
|
||||
]
|
||||
env {
|
||||
name = "K8S_NODE_NAME"
|
||||
value_from {
|
||||
field_ref {
|
||||
api_version = "v1"
|
||||
field_path = "spec.nodeName"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "CILIUM_K8S_NAMESPACE"
|
||||
value_from {
|
||||
field_ref {
|
||||
api_version = "v1"
|
||||
field_path = "metadata.namespace"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "KUBERNETES_SERVICE_HOST"
|
||||
value_from {
|
||||
config_map_key_ref {
|
||||
name = "in-cluster"
|
||||
key = "apiserver-host"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "KUBERNETES_SERVICE_PORT"
|
||||
value_from {
|
||||
config_map_key_ref {
|
||||
name = "in-cluster"
|
||||
key = "apiserver-port"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "CILIUM_DEBUG"
|
||||
value_from {
|
||||
config_map_key_ref {
|
||||
name = "cilium"
|
||||
key = "debug"
|
||||
optional = true
|
||||
}
|
||||
}
|
||||
}
|
||||
port {
|
||||
name = "metrics"
|
||||
protocol = "TCP"
|
||||
host_port = 9963
|
||||
container_port = 9963
|
||||
}
|
||||
port {
|
||||
name = "health"
|
||||
container_port = 9234
|
||||
protocol = "TCP"
|
||||
}
|
||||
liveness_probe {
|
||||
http_get {
|
||||
scheme = "HTTP"
|
||||
host = "127.0.0.1"
|
||||
port = "9234"
|
||||
path = "/healthz"
|
||||
}
|
||||
initial_delay_seconds = 60
|
||||
timeout_seconds = 3
|
||||
period_seconds = 10
|
||||
}
|
||||
readiness_probe {
|
||||
http_get {
|
||||
scheme = "HTTP"
|
||||
host = "127.0.0.1"
|
||||
port = "9234"
|
||||
path = "/healthz"
|
||||
}
|
||||
timeout_seconds = 3
|
||||
period_seconds = 15
|
||||
failure_threshold = 5
|
||||
}
|
||||
volume_mount {
|
||||
name = "config"
|
||||
read_only = true
|
||||
mount_path = "/tmp/cilium/config-map"
|
||||
}
|
||||
}
|
||||
|
||||
volume {
|
||||
name = "config"
|
||||
config_map {
|
||||
name = "cilium"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
15
addons/cilium/service-account.tf
Normal file
15
addons/cilium/service-account.tf
Normal file
@ -0,0 +1,15 @@
|
||||
resource "kubernetes_service_account" "operator" {
|
||||
metadata {
|
||||
name = "cilium-operator"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
automount_service_account_token = false
|
||||
}
|
||||
|
||||
resource "kubernetes_service_account" "agent" {
|
||||
metadata {
|
||||
name = "cilium-agent"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
automount_service_account_token = false
|
||||
}
|
||||
17
addons/cilium/variables.tf
Normal file
17
addons/cilium/variables.tf
Normal file
@ -0,0 +1,17 @@
|
||||
variable "pod_cidr" {
|
||||
type = string
|
||||
description = "CIDR IP range to assign Kubernetes pods"
|
||||
default = "10.2.0.0/16"
|
||||
}
|
||||
|
||||
variable "daemonset_tolerations" {
|
||||
type = list(string)
|
||||
description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "enable_hubble" {
|
||||
type = bool
|
||||
description = "Run the embedded Hubble Server and mount hubble-server-certs Secret"
|
||||
default = true
|
||||
}
|
||||
8
addons/cilium/versions.tf
Normal file
8
addons/cilium/versions.tf
Normal file
@ -0,0 +1,8 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = "~> 2.8"
|
||||
}
|
||||
}
|
||||
}
|
||||
37
addons/coredns/cluster-role.tf
Normal file
37
addons/coredns/cluster-role.tf
Normal file
@ -0,0 +1,37 @@
|
||||
resource "kubernetes_cluster_role" "coredns" {
|
||||
metadata {
|
||||
name = "system:coredns"
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = [
|
||||
"endpoints",
|
||||
"services",
|
||||
"pods",
|
||||
"namespaces",
|
||||
]
|
||||
verbs = [
|
||||
"list",
|
||||
"watch",
|
||||
]
|
||||
}
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = [
|
||||
"nodes",
|
||||
]
|
||||
verbs = [
|
||||
"get",
|
||||
]
|
||||
}
|
||||
rule {
|
||||
api_groups = ["discovery.k8s.io"]
|
||||
resources = [
|
||||
"endpointslices",
|
||||
]
|
||||
verbs = [
|
||||
"list",
|
||||
"watch",
|
||||
]
|
||||
}
|
||||
}
|
||||
30
addons/coredns/config.tf
Normal file
30
addons/coredns/config.tf
Normal file
@ -0,0 +1,30 @@
|
||||
resource "kubernetes_config_map" "coredns" {
|
||||
metadata {
|
||||
name = "coredns"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
data = {
|
||||
"Corefile" = <<-EOF
|
||||
.:53 {
|
||||
errors
|
||||
health {
|
||||
lameduck 5s
|
||||
}
|
||||
ready
|
||||
log . {
|
||||
class error
|
||||
}
|
||||
kubernetes ${var.cluster_domain_suffix} in-addr.arpa ip6.arpa {
|
||||
pods insecure
|
||||
fallthrough in-addr.arpa ip6.arpa
|
||||
}
|
||||
prometheus :9153
|
||||
forward . /etc/resolv.conf
|
||||
cache 30
|
||||
loop
|
||||
reload
|
||||
loadbalance
|
||||
}
|
||||
EOF
|
||||
}
|
||||
}
|
||||
151
addons/coredns/deployment.tf
Normal file
151
addons/coredns/deployment.tf
Normal file
@ -0,0 +1,151 @@
|
||||
resource "kubernetes_deployment" "coredns" {
|
||||
wait_for_rollout = false
|
||||
metadata {
|
||||
name = "coredns"
|
||||
namespace = "kube-system"
|
||||
labels = {
|
||||
k8s-app = "coredns"
|
||||
"kubernetes.io/name" = "CoreDNS"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
replicas = var.replicas
|
||||
strategy {
|
||||
type = "RollingUpdate"
|
||||
rolling_update {
|
||||
max_unavailable = "1"
|
||||
}
|
||||
}
|
||||
selector {
|
||||
match_labels = {
|
||||
k8s-app = "coredns"
|
||||
tier = "control-plane"
|
||||
}
|
||||
}
|
||||
template {
|
||||
metadata {
|
||||
labels = {
|
||||
k8s-app = "coredns"
|
||||
tier = "control-plane"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
affinity {
|
||||
node_affinity {
|
||||
preferred_during_scheduling_ignored_during_execution {
|
||||
weight = 100
|
||||
preference {
|
||||
match_expressions {
|
||||
key = "node.kubernetes.io/controller"
|
||||
operator = "Exists"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pod_anti_affinity {
|
||||
preferred_during_scheduling_ignored_during_execution {
|
||||
weight = 100
|
||||
pod_affinity_term {
|
||||
label_selector {
|
||||
match_expressions {
|
||||
key = "tier"
|
||||
operator = "In"
|
||||
values = ["control-plane"]
|
||||
}
|
||||
match_expressions {
|
||||
key = "k8s-app"
|
||||
operator = "In"
|
||||
values = ["coredns"]
|
||||
}
|
||||
}
|
||||
topology_key = "kubernetes.io/hostname"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
dns_policy = "Default"
|
||||
priority_class_name = "system-cluster-critical"
|
||||
security_context {
|
||||
seccomp_profile {
|
||||
type = "RuntimeDefault"
|
||||
}
|
||||
}
|
||||
service_account_name = "coredns"
|
||||
toleration {
|
||||
key = "node-role.kubernetes.io/controller"
|
||||
effect = "NoSchedule"
|
||||
}
|
||||
container {
|
||||
name = "coredns"
|
||||
image = "registry.k8s.io/coredns/coredns:v1.11.1"
|
||||
args = ["-conf", "/etc/coredns/Corefile"]
|
||||
port {
|
||||
name = "dns"
|
||||
container_port = 53
|
||||
protocol = "UDP"
|
||||
}
|
||||
port {
|
||||
name = "dns-tcp"
|
||||
container_port = 53
|
||||
protocol = "TCP"
|
||||
}
|
||||
port {
|
||||
name = "metrics"
|
||||
container_port = 9153
|
||||
protocol = "TCP"
|
||||
}
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "100m"
|
||||
memory = "70Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "170Mi"
|
||||
}
|
||||
}
|
||||
security_context {
|
||||
capabilities {
|
||||
add = ["NET_BIND_SERVICE"]
|
||||
drop = ["all"]
|
||||
}
|
||||
read_only_root_filesystem = true
|
||||
}
|
||||
liveness_probe {
|
||||
http_get {
|
||||
path = "/health"
|
||||
port = "8080"
|
||||
scheme = "HTTP"
|
||||
}
|
||||
initial_delay_seconds = 60
|
||||
timeout_seconds = 5
|
||||
success_threshold = 1
|
||||
failure_threshold = 5
|
||||
}
|
||||
readiness_probe {
|
||||
http_get {
|
||||
path = "/ready"
|
||||
port = "8181"
|
||||
scheme = "HTTP"
|
||||
}
|
||||
}
|
||||
volume_mount {
|
||||
name = "config"
|
||||
mount_path = "/etc/coredns"
|
||||
read_only = true
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "config"
|
||||
config_map {
|
||||
name = "coredns"
|
||||
items {
|
||||
key = "Corefile"
|
||||
path = "Corefile"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
24
addons/coredns/service-account.tf
Normal file
24
addons/coredns/service-account.tf
Normal file
@ -0,0 +1,24 @@
|
||||
resource "kubernetes_service_account" "coredns" {
|
||||
metadata {
|
||||
name = "coredns"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
automount_service_account_token = false
|
||||
}
|
||||
|
||||
|
||||
resource "kubernetes_cluster_role_binding" "coredns" {
|
||||
metadata {
|
||||
name = "system:coredns"
|
||||
}
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "ClusterRole"
|
||||
name = "system:coredns"
|
||||
}
|
||||
subject {
|
||||
kind = "ServiceAccount"
|
||||
name = "coredns"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
31
addons/coredns/service.tf
Normal file
31
addons/coredns/service.tf
Normal file
@ -0,0 +1,31 @@
|
||||
resource "kubernetes_service" "coredns" {
|
||||
metadata {
|
||||
name = "coredns"
|
||||
namespace = "kube-system"
|
||||
labels = {
|
||||
"k8s-app" = "coredns"
|
||||
"kubernetes.io/name" = "CoreDNS"
|
||||
}
|
||||
annotations = {
|
||||
"prometheus.io/scrape" = "true"
|
||||
"prometheus.io/port" = "9153"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
type = "ClusterIP"
|
||||
cluster_ip = var.cluster_dns_service_ip
|
||||
selector = {
|
||||
k8s-app = "coredns"
|
||||
}
|
||||
port {
|
||||
name = "dns"
|
||||
protocol = "UDP"
|
||||
port = 53
|
||||
}
|
||||
port {
|
||||
name = "dns-tcp"
|
||||
protocol = "TCP"
|
||||
port = 53
|
||||
}
|
||||
}
|
||||
}
|
||||
15
addons/coredns/variables.tf
Normal file
15
addons/coredns/variables.tf
Normal file
@ -0,0 +1,15 @@
|
||||
variable "replicas" {
|
||||
type = number
|
||||
description = "CoreDNS replica count"
|
||||
default = 2
|
||||
}
|
||||
|
||||
variable "cluster_dns_service_ip" {
|
||||
description = "Must be set to `cluster_dns_service_ip` output by cluster"
|
||||
default = "10.3.0.10"
|
||||
}
|
||||
|
||||
variable "cluster_domain_suffix" {
|
||||
description = "Must be set to `cluster_domain_suffix` output by cluster"
|
||||
default = "cluster.local"
|
||||
}
|
||||
9
addons/coredns/versions.tf
Normal file
9
addons/coredns/versions.tf
Normal file
@ -0,0 +1,9 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = "~> 2.8"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
18
addons/flannel/cluster-role-binding.tf
Normal file
18
addons/flannel/cluster-role-binding.tf
Normal file
@ -0,0 +1,18 @@
|
||||
resource "kubernetes_cluster_role_binding" "flannel" {
|
||||
metadata {
|
||||
name = "flannel"
|
||||
}
|
||||
|
||||
role_ref {
|
||||
api_group = "rbac.authorization.k8s.io"
|
||||
kind = "ClusterRole"
|
||||
name = "flannel"
|
||||
}
|
||||
|
||||
subject {
|
||||
kind = "ServiceAccount"
|
||||
name = "flannel"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
24
addons/flannel/cluster-role.tf
Normal file
24
addons/flannel/cluster-role.tf
Normal file
@ -0,0 +1,24 @@
|
||||
resource "kubernetes_cluster_role" "flannel" {
|
||||
metadata {
|
||||
name = "flannel"
|
||||
}
|
||||
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["pods"]
|
||||
verbs = ["get"]
|
||||
}
|
||||
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["nodes"]
|
||||
verbs = ["list", "watch"]
|
||||
}
|
||||
|
||||
rule {
|
||||
api_groups = [""]
|
||||
resources = ["nodes/status"]
|
||||
verbs = ["patch"]
|
||||
}
|
||||
}
|
||||
|
||||
44
addons/flannel/config.tf
Normal file
44
addons/flannel/config.tf
Normal file
@ -0,0 +1,44 @@
|
||||
resource "kubernetes_config_map" "config" {
|
||||
metadata {
|
||||
name = "flannel-config"
|
||||
namespace = "kube-system"
|
||||
labels = {
|
||||
k8s-app = "flannel"
|
||||
tier = "node"
|
||||
}
|
||||
}
|
||||
|
||||
data = {
|
||||
"cni-conf.json" = <<-EOF
|
||||
{
|
||||
"name": "cbr0",
|
||||
"cniVersion": "0.3.1",
|
||||
"plugins": [
|
||||
{
|
||||
"type": "flannel",
|
||||
"delegate": {
|
||||
"hairpinMode": true,
|
||||
"isDefaultGateway": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "portmap",
|
||||
"capabilities": {
|
||||
"portMappings": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
"net-conf.json" = <<-EOF
|
||||
{
|
||||
"Network": "${var.pod_cidr}",
|
||||
"Backend": {
|
||||
"Type": "vxlan",
|
||||
"Port": 4789
|
||||
}
|
||||
}
|
||||
EOF
|
||||
}
|
||||
}
|
||||
|
||||
167
addons/flannel/daemonset.tf
Normal file
167
addons/flannel/daemonset.tf
Normal file
@ -0,0 +1,167 @@
|
||||
resource "kubernetes_daemonset" "flannel" {
|
||||
metadata {
|
||||
name = "flannel"
|
||||
namespace = "kube-system"
|
||||
labels = {
|
||||
k8s-app = "flannel"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
strategy {
|
||||
type = "RollingUpdate"
|
||||
rolling_update {
|
||||
max_unavailable = "1"
|
||||
}
|
||||
}
|
||||
selector {
|
||||
match_labels = {
|
||||
k8s-app = "flannel"
|
||||
}
|
||||
}
|
||||
template {
|
||||
metadata {
|
||||
labels = {
|
||||
k8s-app = "flannel"
|
||||
}
|
||||
}
|
||||
spec {
|
||||
host_network = true
|
||||
priority_class_name = "system-node-critical"
|
||||
service_account_name = "flannel"
|
||||
security_context {
|
||||
seccomp_profile {
|
||||
type = "RuntimeDefault"
|
||||
}
|
||||
}
|
||||
toleration {
|
||||
key = "node-role.kubernetes.io/controller"
|
||||
operator = "Exists"
|
||||
}
|
||||
toleration {
|
||||
key = "node.kubernetes.io/not-ready"
|
||||
operator = "Exists"
|
||||
}
|
||||
dynamic "toleration" {
|
||||
for_each = var.daemonset_tolerations
|
||||
content {
|
||||
key = toleration.value
|
||||
operator = "Exists"
|
||||
}
|
||||
}
|
||||
init_container {
|
||||
name = "install-cni"
|
||||
image = "quay.io/poseidon/flannel-cni:v0.4.2"
|
||||
command = ["/install-cni.sh"]
|
||||
env {
|
||||
name = "CNI_NETWORK_CONFIG"
|
||||
value_from {
|
||||
config_map_key_ref {
|
||||
name = "flannel-config"
|
||||
key = "cni-conf.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
volume_mount {
|
||||
name = "cni-bin-dir"
|
||||
mount_path = "/host/opt/cni/bin/"
|
||||
}
|
||||
volume_mount {
|
||||
name = "cni-conf-dir"
|
||||
mount_path = "/host/etc/cni/net.d"
|
||||
}
|
||||
}
|
||||
|
||||
container {
|
||||
name = "flannel"
|
||||
image = "docker.io/flannel/flannel:v0.25.1"
|
||||
command = [
|
||||
"/opt/bin/flanneld",
|
||||
"--ip-masq",
|
||||
"--kube-subnet-mgr",
|
||||
"--iface=$(POD_IP)"
|
||||
]
|
||||
env {
|
||||
name = "POD_NAME"
|
||||
value_from {
|
||||
field_ref {
|
||||
field_path = "metadata.name"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "POD_NAMESPACE"
|
||||
value_from {
|
||||
field_ref {
|
||||
field_path = "metadata.namespace"
|
||||
}
|
||||
}
|
||||
}
|
||||
env {
|
||||
name = "POD_IP"
|
||||
value_from {
|
||||
field_ref {
|
||||
field_path = "status.podIP"
|
||||
}
|
||||
}
|
||||
}
|
||||
security_context {
|
||||
privileged = true
|
||||
}
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "100m"
|
||||
}
|
||||
}
|
||||
volume_mount {
|
||||
name = "flannel-config"
|
||||
mount_path = "/etc/kube-flannel/"
|
||||
}
|
||||
volume_mount {
|
||||
name = "run-flannel"
|
||||
mount_path = "/run/flannel"
|
||||
}
|
||||
volume_mount {
|
||||
name = "xtables-lock"
|
||||
mount_path = "/run/xtables.lock"
|
||||
}
|
||||
}
|
||||
|
||||
volume {
|
||||
name = "flannel-config"
|
||||
config_map {
|
||||
name = "flannel-config"
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "run-flannel"
|
||||
host_path {
|
||||
path = "/run/flannel"
|
||||
}
|
||||
}
|
||||
# Used by install-cni
|
||||
volume {
|
||||
name = "cni-bin-dir"
|
||||
host_path {
|
||||
path = "/opt/cni/bin"
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "cni-conf-dir"
|
||||
host_path {
|
||||
path = "/etc/cni/net.d"
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
# Acces iptables concurrently
|
||||
volume {
|
||||
name = "xtables-lock"
|
||||
host_path {
|
||||
path = "/run/xtables.lock"
|
||||
type = "FileOrCreate"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
7
addons/flannel/service-account.tf
Normal file
7
addons/flannel/service-account.tf
Normal file
@ -0,0 +1,7 @@
|
||||
resource "kubernetes_service_account" "flannel" {
|
||||
metadata {
|
||||
name = "flannel"
|
||||
namespace = "kube-system"
|
||||
}
|
||||
}
|
||||
|
||||
11
addons/flannel/variables.tf
Normal file
11
addons/flannel/variables.tf
Normal file
@ -0,0 +1,11 @@
|
||||
variable "pod_cidr" {
|
||||
type = string
|
||||
description = "CIDR IP range to assign Kubernetes pods"
|
||||
default = "10.2.0.0/16"
|
||||
}
|
||||
|
||||
variable "daemonset_tolerations" {
|
||||
type = list(string)
|
||||
description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
|
||||
default = []
|
||||
}
|
||||
8
addons/flannel/versions.tf
Normal file
8
addons/flannel/versions.tf
Normal file
@ -0,0 +1,8 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = "~> 2.8"
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user