mirror of
https://github.com/puppetmaster/typhoon.git
synced 2025-07-23 08:21:34 +02:00
Rename container-linux modules to flatcar-linux
* CoreOS Container Linux was deprecated in v1.18.3 * Continue transitioning docs and modules from supporting both CoreOS and Flatcar "variants" of Container Linux to now supporting Flatcar Linux and equivalents Action Required: Update the Flatcar Linux modules `source` to replace `s/container-linux/flatcar-linux`. See docs for examples
This commit is contained in:
Dalton Hubble
23
google-cloud/flatcar-linux/kubernetes/LICENSE
Normal file
23
google-cloud/flatcar-linux/kubernetes/LICENSE
Normal file
@ -0,0 +1,23 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017 Typhoon Authors
|
||||
Copyright (c) 2017 Dalton Hubble
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
23
google-cloud/flatcar-linux/kubernetes/README.md
Normal file
23
google-cloud/flatcar-linux/kubernetes/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Typhoon <img align="right" src="https://storage.googleapis.com/poseidon/typhoon-logo.png">
|
||||
|
||||
Typhoon is a minimal and free Kubernetes distribution.
|
||||
|
||||
* Minimal, stable base Kubernetes distribution
|
||||
* Declarative infrastructure and configuration
|
||||
* Free (freedom and cost) and privacy-respecting
|
||||
* Practical for labs, datacenters, and clouds
|
||||
|
||||
Typhoon distributes upstream Kubernetes, architectural conventions, and cluster addons, much like a GNU/Linux distribution provides the Linux kernel and userspace components.
|
||||
|
||||
## Features <a href="https://www.cncf.io/certification/software-conformance/"><img align="right" src="https://storage.googleapis.com/poseidon/certified-kubernetes.png"></a>
|
||||
|
||||
* Kubernetes v1.19.3 (upstream)
|
||||
* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking
|
||||
* On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
|
||||
* Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/cl/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization
|
||||
* Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/)
|
||||
|
||||
## Docs
|
||||
|
||||
Please see the [official docs](https://typhoon.psdn.io) and the Google Cloud [tutorial](https://typhoon.psdn.io/cl/google-cloud/).
|
||||
|
||||
93
google-cloud/flatcar-linux/kubernetes/apiserver.tf
Normal file
93
google-cloud/flatcar-linux/kubernetes/apiserver.tf
Normal file
@ -0,0 +1,93 @@
|
||||
# TCP Proxy load balancer DNS record
|
||||
resource "google_dns_record_set" "apiserver" {
|
||||
# DNS Zone name where record should be created
|
||||
managed_zone = var.dns_zone_name
|
||||
|
||||
# DNS record
|
||||
name = format("%s.%s.", var.cluster_name, var.dns_zone)
|
||||
type = "A"
|
||||
ttl = 300
|
||||
|
||||
# IPv4 address of apiserver TCP Proxy load balancer
|
||||
rrdatas = [google_compute_global_address.apiserver-ipv4.address]
|
||||
}
|
||||
|
||||
# Static IPv4 address for the TCP Proxy Load Balancer
|
||||
resource "google_compute_global_address" "apiserver-ipv4" {
|
||||
name = "${var.cluster_name}-apiserver-ip"
|
||||
ip_version = "IPV4"
|
||||
}
|
||||
|
||||
# Forward IPv4 TCP traffic to the TCP proxy load balancer
|
||||
resource "google_compute_global_forwarding_rule" "apiserver" {
|
||||
name = "${var.cluster_name}-apiserver"
|
||||
ip_address = google_compute_global_address.apiserver-ipv4.address
|
||||
ip_protocol = "TCP"
|
||||
port_range = "443"
|
||||
target = google_compute_target_tcp_proxy.apiserver.self_link
|
||||
}
|
||||
|
||||
# Global TCP Proxy Load Balancer for apiservers
|
||||
resource "google_compute_target_tcp_proxy" "apiserver" {
|
||||
name = "${var.cluster_name}-apiserver"
|
||||
description = "Distribute TCP load across ${var.cluster_name} controllers"
|
||||
backend_service = google_compute_backend_service.apiserver.self_link
|
||||
}
|
||||
|
||||
# Global backend service backed by unmanaged instance groups
|
||||
resource "google_compute_backend_service" "apiserver" {
|
||||
name = "${var.cluster_name}-apiserver"
|
||||
description = "${var.cluster_name} apiserver service"
|
||||
|
||||
protocol = "TCP"
|
||||
port_name = "apiserver"
|
||||
session_affinity = "NONE"
|
||||
timeout_sec = "300"
|
||||
|
||||
# controller(s) spread across zonal instance groups
|
||||
dynamic "backend" {
|
||||
for_each = google_compute_instance_group.controllers
|
||||
content {
|
||||
group = backend.value.self_link
|
||||
}
|
||||
}
|
||||
|
||||
health_checks = [google_compute_health_check.apiserver.self_link]
|
||||
}
|
||||
|
||||
# Instance group of heterogeneous (unmanged) controller instances
|
||||
resource "google_compute_instance_group" "controllers" {
|
||||
count = min(var.controller_count, length(local.zones))
|
||||
|
||||
name = format("%s-controllers-%s", var.cluster_name, element(local.zones, count.index))
|
||||
zone = element(local.zones, count.index)
|
||||
|
||||
named_port {
|
||||
name = "apiserver"
|
||||
port = "6443"
|
||||
}
|
||||
|
||||
# add instances in the zone into the instance group
|
||||
instances = matchkeys(
|
||||
google_compute_instance.controllers.*.self_link,
|
||||
google_compute_instance.controllers.*.zone,
|
||||
[element(local.zones, count.index)],
|
||||
)
|
||||
}
|
||||
|
||||
# TCP health check for apiserver
|
||||
resource "google_compute_health_check" "apiserver" {
|
||||
name = "${var.cluster_name}-apiserver-tcp-health"
|
||||
description = "TCP health check for kube-apiserver"
|
||||
|
||||
timeout_sec = 5
|
||||
check_interval_sec = 5
|
||||
|
||||
healthy_threshold = 1
|
||||
unhealthy_threshold = 3
|
||||
|
||||
tcp_health_check {
|
||||
port = "6443"
|
||||
}
|
||||
}
|
||||
|
||||
19
google-cloud/flatcar-linux/kubernetes/bootstrap.tf
Normal file
19
google-cloud/flatcar-linux/kubernetes/bootstrap.tf
Normal file
@ -0,0 +1,19 @@
|
||||
# Kubernetes assets (kubeconfig, manifests)
|
||||
module "bootstrap" {
|
||||
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=9037d7311b949439b217cd9c657d4500eab3e16b"
|
||||
|
||||
cluster_name = var.cluster_name
|
||||
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]
|
||||
etcd_servers = google_dns_record_set.etcds.*.name
|
||||
networking = var.networking
|
||||
network_mtu = 1440
|
||||
pod_cidr = var.pod_cidr
|
||||
service_cidr = var.service_cidr
|
||||
cluster_domain_suffix = var.cluster_domain_suffix
|
||||
enable_reporting = var.enable_reporting
|
||||
enable_aggregation = var.enable_aggregation
|
||||
|
||||
// temporary
|
||||
external_apiserver_port = 443
|
||||
}
|
||||
|
||||
192
google-cloud/flatcar-linux/kubernetes/cl/controller.yaml
Normal file
192
google-cloud/flatcar-linux/kubernetes/cl/controller.yaml
Normal file
@ -0,0 +1,192 @@
|
||||
---
|
||||
systemd:
|
||||
units:
|
||||
- name: etcd-member.service
|
||||
enabled: true
|
||||
dropins:
|
||||
- name: 40-etcd-cluster.conf
|
||||
contents: |
|
||||
[Service]
|
||||
Environment="ETCD_IMAGE_TAG=v3.4.12"
|
||||
Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
|
||||
Environment="RKT_RUN_ARGS=--insecure-options=image"
|
||||
Environment="ETCD_NAME=${etcd_name}"
|
||||
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
|
||||
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
|
||||
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
|
||||
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
|
||||
Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
|
||||
Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
|
||||
Environment="ETCD_STRICT_RECONFIG_CHECK=true"
|
||||
Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
|
||||
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
|
||||
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
|
||||
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
|
||||
Environment="ETCD_CLIENT_CERT_AUTH=true"
|
||||
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
|
||||
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
|
||||
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
|
||||
Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
|
||||
- name: docker.service
|
||||
enabled: true
|
||||
- name: locksmithd.service
|
||||
mask: true
|
||||
- name: wait-for-dns.service
|
||||
enabled: true
|
||||
contents: |
|
||||
[Unit]
|
||||
Description=Wait for DNS entries
|
||||
Wants=systemd-resolved.service
|
||||
Before=kubelet.service
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=true
|
||||
ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
|
||||
[Install]
|
||||
RequiredBy=kubelet.service
|
||||
RequiredBy=etcd-member.service
|
||||
- name: kubelet.service
|
||||
enabled: true
|
||||
contents: |
|
||||
[Unit]
|
||||
Description=Kubelet
|
||||
Requires=docker.service
|
||||
After=docker.service
|
||||
Wants=rpc-statd.service
|
||||
[Service]
|
||||
Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
|
||||
ExecStartPre=/bin/mkdir -p /opt/cni/bin
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/calico
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
|
||||
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
|
||||
ExecStartPre=/usr/bin/docker run -d \
|
||||
--name kubelet \
|
||||
--privileged \
|
||||
--pid host \
|
||||
--network host \
|
||||
-v /etc/kubernetes:/etc/kubernetes:ro \
|
||||
-v /etc/machine-id:/etc/machine-id:ro \
|
||||
-v /usr/lib/os-release:/etc/os-release:ro \
|
||||
-v /lib/modules:/lib/modules:ro \
|
||||
-v /run:/run \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:ro \
|
||||
-v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
|
||||
-v /var/lib/calico:/var/lib/calico:ro \
|
||||
-v /var/lib/docker:/var/lib/docker \
|
||||
-v /var/lib/kubelet:/var/lib/kubelet:rshared \
|
||||
-v /var/log:/var/log \
|
||||
-v /opt/cni/bin:/opt/cni/bin \
|
||||
$${KUBELET_IMAGE} \
|
||||
--anonymous-auth=false \
|
||||
--authentication-token-webhook \
|
||||
--authorization-mode=Webhook \
|
||||
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
|
||||
--client-ca-file=/etc/kubernetes/ca.crt \
|
||||
--cluster_dns=${cluster_dns_service_ip} \
|
||||
--cluster_domain=${cluster_domain_suffix} \
|
||||
--cni-conf-dir=/etc/kubernetes/cni/net.d \
|
||||
--healthz-port=0 \
|
||||
--kubeconfig=/var/lib/kubelet/kubeconfig \
|
||||
--network-plugin=cni \
|
||||
--node-labels=node.kubernetes.io/controller="true" \
|
||||
--pod-manifest-path=/etc/kubernetes/manifests \
|
||||
--register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \
|
||||
--read-only-port=0 \
|
||||
--rotate-certificates \
|
||||
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
|
||||
ExecStart=docker logs -f kubelet
|
||||
ExecStop=docker stop kubelet
|
||||
ExecStopPost=docker rm kubelet
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: bootstrap.service
|
||||
contents: |
|
||||
[Unit]
|
||||
Description=Kubernetes control plane
|
||||
ConditionPathExists=!/opt/bootstrap/bootstrap.done
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=true
|
||||
WorkingDirectory=/opt/bootstrap
|
||||
ExecStart=/usr/bin/rkt run \
|
||||
--trust-keys-from-https \
|
||||
--volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \
|
||||
--mount volume=config,target=/etc/kubernetes/secrets \
|
||||
--volume assets,kind=host,source=/opt/bootstrap/assets \
|
||||
--mount volume=assets,target=/assets \
|
||||
--volume script,kind=host,source=/opt/bootstrap/apply \
|
||||
--mount volume=script,target=/apply \
|
||||
--insecure-options=image \
|
||||
docker://quay.io/poseidon/kubelet:v1.19.3 \
|
||||
--net=host \
|
||||
--dns=host \
|
||||
--exec=/apply
|
||||
ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
storage:
|
||||
directories:
|
||||
- path: /var/lib/etcd
|
||||
filesystem: root
|
||||
mode: 0700
|
||||
overwrite: true
|
||||
files:
|
||||
- path: /etc/kubernetes/kubeconfig
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline: |
|
||||
${kubeconfig}
|
||||
- path: /opt/bootstrap/layout
|
||||
filesystem: root
|
||||
mode: 0544
|
||||
contents:
|
||||
inline: |
|
||||
#!/bin/bash -e
|
||||
mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
|
||||
awk '/#####/ {filename=$2; next} {print > filename}' assets
|
||||
mkdir -p /etc/ssl/etcd/etcd
|
||||
mkdir -p /etc/kubernetes/bootstrap-secrets
|
||||
mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
|
||||
mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/
|
||||
chown -R etcd:etcd /etc/ssl/etcd
|
||||
chmod -R 500 /etc/ssl/etcd
|
||||
chmod -R 700 /var/lib/etcd
|
||||
mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/
|
||||
mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/
|
||||
mkdir -p /etc/kubernetes/manifests
|
||||
mv static-manifests/* /etc/kubernetes/manifests/
|
||||
mkdir -p /opt/bootstrap/assets
|
||||
mv manifests /opt/bootstrap/assets/manifests
|
||||
mv manifests-networking/* /opt/bootstrap/assets/manifests/
|
||||
rm -rf assets auth static-manifests tls manifests-networking
|
||||
- path: /opt/bootstrap/apply
|
||||
filesystem: root
|
||||
mode: 0544
|
||||
contents:
|
||||
inline: |
|
||||
#!/bin/bash -e
|
||||
export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig
|
||||
until kubectl version; do
|
||||
echo "Waiting for static pod control plane"
|
||||
sleep 5
|
||||
done
|
||||
until kubectl apply -f /assets/manifests -R; do
|
||||
echo "Retry applying manifests"
|
||||
sleep 5
|
||||
done
|
||||
- path: /etc/sysctl.d/max-user-watches.conf
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline: |
|
||||
fs.inotify.max_user_watches=16184
|
||||
passwd:
|
||||
users:
|
||||
- name: core
|
||||
ssh_authorized_keys:
|
||||
- "${ssh_authorized_key}"
|
||||
103
google-cloud/flatcar-linux/kubernetes/controllers.tf
Normal file
103
google-cloud/flatcar-linux/kubernetes/controllers.tf
Normal file
@ -0,0 +1,103 @@
|
||||
# Discrete DNS records for each controller's private IPv4 for etcd usage
|
||||
resource "google_dns_record_set" "etcds" {
|
||||
count = var.controller_count
|
||||
|
||||
# DNS Zone name where record should be created
|
||||
managed_zone = var.dns_zone_name
|
||||
|
||||
# DNS record
|
||||
name = format("%s-etcd%d.%s.", var.cluster_name, count.index, var.dns_zone)
|
||||
type = "A"
|
||||
ttl = 300
|
||||
|
||||
# private IPv4 address for etcd
|
||||
rrdatas = [google_compute_instance.controllers.*.network_interface.0.network_ip[count.index]]
|
||||
}
|
||||
|
||||
# Zones in the region
|
||||
data "google_compute_zones" "all" {
|
||||
region = var.region
|
||||
}
|
||||
|
||||
locals {
|
||||
zones = data.google_compute_zones.all.names
|
||||
|
||||
controllers_ipv4_public = google_compute_instance.controllers.*.network_interface.0.access_config.0.nat_ip
|
||||
}
|
||||
|
||||
# Controller instances
|
||||
resource "google_compute_instance" "controllers" {
|
||||
count = var.controller_count
|
||||
|
||||
name = "${var.cluster_name}-controller-${count.index}"
|
||||
# use a zone in the region and wrap around (e.g. controllers > zones)
|
||||
zone = element(local.zones, count.index)
|
||||
machine_type = var.controller_type
|
||||
|
||||
metadata = {
|
||||
user-data = data.ct_config.controller-ignitions.*.rendered[count.index]
|
||||
}
|
||||
|
||||
boot_disk {
|
||||
auto_delete = true
|
||||
|
||||
initialize_params {
|
||||
image = var.os_image
|
||||
size = var.disk_size
|
||||
}
|
||||
}
|
||||
|
||||
network_interface {
|
||||
network = google_compute_network.network.name
|
||||
|
||||
# Ephemeral external IP
|
||||
access_config {
|
||||
}
|
||||
}
|
||||
|
||||
can_ip_forward = true
|
||||
tags = ["${var.cluster_name}-controller"]
|
||||
|
||||
lifecycle {
|
||||
ignore_changes = [metadata]
|
||||
}
|
||||
}
|
||||
|
||||
# Controller Ignition configs
|
||||
data "ct_config" "controller-ignitions" {
|
||||
count = var.controller_count
|
||||
content = data.template_file.controller-configs.*.rendered[count.index]
|
||||
strict = true
|
||||
snippets = var.controller_snippets
|
||||
}
|
||||
|
||||
# Controller Container Linux configs
|
||||
data "template_file" "controller-configs" {
|
||||
count = var.controller_count
|
||||
|
||||
template = file("${path.module}/cl/controller.yaml")
|
||||
|
||||
vars = {
|
||||
# Cannot use cyclic dependencies on controllers or their DNS records
|
||||
etcd_name = "etcd${count.index}"
|
||||
etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
|
||||
# etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
|
||||
etcd_initial_cluster = join(",", data.template_file.etcds.*.rendered)
|
||||
kubeconfig = indent(10, module.bootstrap.kubeconfig-kubelet)
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
|
||||
cluster_domain_suffix = var.cluster_domain_suffix
|
||||
}
|
||||
}
|
||||
|
||||
data "template_file" "etcds" {
|
||||
count = var.controller_count
|
||||
template = "etcd$${index}=https://$${cluster_name}-etcd$${index}.$${dns_zone}:2380"
|
||||
|
||||
vars = {
|
||||
index = count.index
|
||||
cluster_name = var.cluster_name
|
||||
dns_zone = var.dns_zone
|
||||
}
|
||||
}
|
||||
|
||||
123
google-cloud/flatcar-linux/kubernetes/ingress.tf
Normal file
123
google-cloud/flatcar-linux/kubernetes/ingress.tf
Normal file
@ -0,0 +1,123 @@
|
||||
# Static IPv4 address for Ingress Load Balancing
|
||||
resource "google_compute_global_address" "ingress-ipv4" {
|
||||
name = "${var.cluster_name}-ingress-ipv4"
|
||||
ip_version = "IPV4"
|
||||
}
|
||||
|
||||
# Static IPv6 address for Ingress Load Balancing
|
||||
resource "google_compute_global_address" "ingress-ipv6" {
|
||||
name = "${var.cluster_name}-ingress-ipv6"
|
||||
ip_version = "IPV6"
|
||||
}
|
||||
|
||||
# Forward IPv4 TCP traffic to the HTTP proxy load balancer
|
||||
# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy.
|
||||
resource "google_compute_global_forwarding_rule" "ingress-http-ipv4" {
|
||||
name = "${var.cluster_name}-ingress-http-ipv4"
|
||||
ip_address = google_compute_global_address.ingress-ipv4.address
|
||||
ip_protocol = "TCP"
|
||||
port_range = "80"
|
||||
target = google_compute_target_http_proxy.ingress-http.self_link
|
||||
}
|
||||
|
||||
# Forward IPv4 TCP traffic to the TCP proxy load balancer
|
||||
resource "google_compute_global_forwarding_rule" "ingress-https-ipv4" {
|
||||
name = "${var.cluster_name}-ingress-https-ipv4"
|
||||
ip_address = google_compute_global_address.ingress-ipv4.address
|
||||
ip_protocol = "TCP"
|
||||
port_range = "443"
|
||||
target = google_compute_target_tcp_proxy.ingress-https.self_link
|
||||
}
|
||||
|
||||
# Forward IPv6 TCP traffic to the HTTP proxy load balancer
|
||||
# Google Cloud does not allow TCP proxies for port 80. Must use HTTP proxy.
|
||||
resource "google_compute_global_forwarding_rule" "ingress-http-ipv6" {
|
||||
name = "${var.cluster_name}-ingress-http-ipv6"
|
||||
ip_address = google_compute_global_address.ingress-ipv6.address
|
||||
ip_protocol = "TCP"
|
||||
port_range = "80"
|
||||
target = google_compute_target_http_proxy.ingress-http.self_link
|
||||
}
|
||||
|
||||
# Forward IPv6 TCP traffic to the TCP proxy load balancer
|
||||
resource "google_compute_global_forwarding_rule" "ingress-https-ipv6" {
|
||||
name = "${var.cluster_name}-ingress-https-ipv6"
|
||||
ip_address = google_compute_global_address.ingress-ipv6.address
|
||||
ip_protocol = "TCP"
|
||||
port_range = "443"
|
||||
target = google_compute_target_tcp_proxy.ingress-https.self_link
|
||||
}
|
||||
|
||||
# HTTP proxy load balancer for ingress controllers
|
||||
resource "google_compute_target_http_proxy" "ingress-http" {
|
||||
name = "${var.cluster_name}-ingress-http"
|
||||
description = "Distribute HTTP load across ${var.cluster_name} workers"
|
||||
url_map = google_compute_url_map.ingress-http.self_link
|
||||
}
|
||||
|
||||
# TCP proxy load balancer for ingress controllers
|
||||
resource "google_compute_target_tcp_proxy" "ingress-https" {
|
||||
name = "${var.cluster_name}-ingress-https"
|
||||
description = "Distribute HTTPS load across ${var.cluster_name} workers"
|
||||
backend_service = google_compute_backend_service.ingress-https.self_link
|
||||
}
|
||||
|
||||
# HTTP URL Map (required)
|
||||
resource "google_compute_url_map" "ingress-http" {
|
||||
name = "${var.cluster_name}-ingress-http"
|
||||
|
||||
# Do not add host/path rules for applications here. Use Ingress resources.
|
||||
default_service = google_compute_backend_service.ingress-http.self_link
|
||||
}
|
||||
|
||||
# Backend service backed by managed instance group of workers
|
||||
resource "google_compute_backend_service" "ingress-http" {
|
||||
name = "${var.cluster_name}-ingress-http"
|
||||
description = "${var.cluster_name} ingress service"
|
||||
|
||||
protocol = "HTTP"
|
||||
port_name = "http"
|
||||
session_affinity = "NONE"
|
||||
timeout_sec = "60"
|
||||
|
||||
backend {
|
||||
group = module.workers.instance_group
|
||||
}
|
||||
|
||||
health_checks = [google_compute_health_check.ingress.self_link]
|
||||
}
|
||||
|
||||
# Backend service backed by managed instance group of workers
|
||||
resource "google_compute_backend_service" "ingress-https" {
|
||||
name = "${var.cluster_name}-ingress-https"
|
||||
description = "${var.cluster_name} ingress service"
|
||||
|
||||
protocol = "TCP"
|
||||
port_name = "https"
|
||||
session_affinity = "NONE"
|
||||
timeout_sec = "60"
|
||||
|
||||
backend {
|
||||
group = module.workers.instance_group
|
||||
}
|
||||
|
||||
health_checks = [google_compute_health_check.ingress.self_link]
|
||||
}
|
||||
|
||||
# Ingress HTTP Health Check
|
||||
resource "google_compute_health_check" "ingress" {
|
||||
name = "${var.cluster_name}-ingress-health"
|
||||
description = "Health check for Ingress controller"
|
||||
|
||||
timeout_sec = 5
|
||||
check_interval_sec = 5
|
||||
|
||||
healthy_threshold = 2
|
||||
unhealthy_threshold = 4
|
||||
|
||||
http_health_check {
|
||||
port = 10254
|
||||
request_path = "/healthz"
|
||||
}
|
||||
}
|
||||
|
||||
219
google-cloud/flatcar-linux/kubernetes/network.tf
Normal file
219
google-cloud/flatcar-linux/kubernetes/network.tf
Normal file
@ -0,0 +1,219 @@
|
||||
resource "google_compute_network" "network" {
|
||||
name = var.cluster_name
|
||||
description = "Network for the ${var.cluster_name} cluster"
|
||||
auto_create_subnetworks = true
|
||||
|
||||
timeouts {
|
||||
delete = "6m"
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "allow-ssh" {
|
||||
name = "${var.cluster_name}-allow-ssh"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [22]
|
||||
}
|
||||
|
||||
source_ranges = ["0.0.0.0/0"]
|
||||
target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "internal-etcd" {
|
||||
name = "${var.cluster_name}-internal-etcd"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [2379, 2380]
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-controller"]
|
||||
target_tags = ["${var.cluster_name}-controller"]
|
||||
}
|
||||
|
||||
# Allow Prometheus to scrape etcd metrics
|
||||
resource "google_compute_firewall" "internal-etcd-metrics" {
|
||||
name = "${var.cluster_name}-internal-etcd-metrics"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [2381]
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller"]
|
||||
}
|
||||
|
||||
# Allow Prometheus to scrape kube-scheduler and kube-controller-manager metrics
|
||||
resource "google_compute_firewall" "internal-kube-metrics" {
|
||||
name = "${var.cluster_name}-internal-kube-metrics"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [10251, 10252]
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller"]
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "allow-apiserver" {
|
||||
name = "${var.cluster_name}-allow-apiserver"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [6443]
|
||||
}
|
||||
|
||||
source_ranges = ["0.0.0.0/0"]
|
||||
target_tags = ["${var.cluster_name}-controller"]
|
||||
}
|
||||
|
||||
# BGP and IPIP
|
||||
# https://docs.projectcalico.org/latest/reference/public-cloud/gce
|
||||
resource "google_compute_firewall" "internal-bgp" {
|
||||
count = var.networking != "flannel" ? 1 : 0
|
||||
|
||||
name = "${var.cluster_name}-internal-bgp"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = ["179"]
|
||||
}
|
||||
|
||||
allow {
|
||||
protocol = "ipip"
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
# flannel VXLAN
|
||||
resource "google_compute_firewall" "internal-vxlan" {
|
||||
count = var.networking == "flannel" ? 1 : 0
|
||||
|
||||
name = "${var.cluster_name}-internal-vxlan"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "udp"
|
||||
ports = [4789]
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
# Cilium VXLAN
|
||||
resource "google_compute_firewall" "internal-linux-vxlan" {
|
||||
count = var.networking == "cilium" ? 1 : 0
|
||||
|
||||
name = "${var.cluster_name}-linux-vxlan"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "udp"
|
||||
ports = [8472]
|
||||
}
|
||||
|
||||
# Cilium health
|
||||
allow {
|
||||
protocol = "icmp"
|
||||
}
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [4240]
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
# Allow Prometheus to scrape node-exporter daemonset
|
||||
resource "google_compute_firewall" "internal-node-exporter" {
|
||||
name = "${var.cluster_name}-internal-node-exporter"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [9100]
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
# Allow Prometheus to scrape kube-proxy metrics
|
||||
resource "google_compute_firewall" "internal-kube-proxy" {
|
||||
name = "${var.cluster_name}-internal-kube-proxy"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [10249]
|
||||
}
|
||||
|
||||
source_tags = ["${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
# Allow apiserver to access kubelets for exec, log, port-forward
|
||||
resource "google_compute_firewall" "internal-kubelet" {
|
||||
name = "${var.cluster_name}-internal-kubelet"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [10250]
|
||||
}
|
||||
|
||||
# allow Prometheus to scrape kubelet metrics too
|
||||
source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
# Workers
|
||||
|
||||
resource "google_compute_firewall" "allow-ingress" {
|
||||
name = "${var.cluster_name}-allow-ingress"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [80, 443]
|
||||
}
|
||||
|
||||
source_ranges = ["0.0.0.0/0"]
|
||||
target_tags = ["${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
resource "google_compute_firewall" "google-ingress-health-checks" {
|
||||
name = "${var.cluster_name}-ingress-health"
|
||||
network = google_compute_network.network.name
|
||||
|
||||
allow {
|
||||
protocol = "tcp"
|
||||
ports = [10254]
|
||||
}
|
||||
|
||||
# https://cloud.google.com/load-balancing/docs/health-check-concepts#method
|
||||
source_ranges = [
|
||||
"35.191.0.0/16",
|
||||
"130.211.0.0/22",
|
||||
"35.191.0.0/16",
|
||||
"209.85.152.0/22",
|
||||
"209.85.204.0/22",
|
||||
]
|
||||
|
||||
target_tags = ["${var.cluster_name}-worker"]
|
||||
}
|
||||
|
||||
50
google-cloud/flatcar-linux/kubernetes/outputs.tf
Normal file
50
google-cloud/flatcar-linux/kubernetes/outputs.tf
Normal file
@ -0,0 +1,50 @@
|
||||
output "kubeconfig-admin" {
|
||||
value = module.bootstrap.kubeconfig-admin
|
||||
}
|
||||
|
||||
# Outputs for Kubernetes Ingress
|
||||
|
||||
output "ingress_static_ipv4" {
|
||||
description = "Global IPv4 address for proxy load balancing to the nearest Ingress controller"
|
||||
value = google_compute_global_address.ingress-ipv4.address
|
||||
}
|
||||
|
||||
output "ingress_static_ipv6" {
|
||||
description = "Global IPv6 address for proxy load balancing to the nearest Ingress controller"
|
||||
value = google_compute_global_address.ingress-ipv6.address
|
||||
}
|
||||
|
||||
# Outputs for worker pools
|
||||
|
||||
output "network_name" {
|
||||
value = google_compute_network.network.name
|
||||
}
|
||||
|
||||
output "kubeconfig" {
|
||||
value = module.bootstrap.kubeconfig-kubelet
|
||||
}
|
||||
|
||||
# Outputs for custom firewalling
|
||||
|
||||
output "network_self_link" {
|
||||
value = google_compute_network.network.self_link
|
||||
}
|
||||
|
||||
# Outputs for custom load balancing
|
||||
|
||||
output "worker_instance_group" {
|
||||
description = "Worker managed instance group full URL"
|
||||
value = module.workers.instance_group
|
||||
}
|
||||
|
||||
output "worker_target_pool" {
|
||||
description = "Worker target pool self link"
|
||||
value = module.workers.target_pool
|
||||
}
|
||||
|
||||
# Outputs for debug
|
||||
|
||||
output "assets_dist" {
|
||||
value = module.bootstrap.assets_dist
|
||||
}
|
||||
|
||||
58
google-cloud/flatcar-linux/kubernetes/ssh.tf
Normal file
58
google-cloud/flatcar-linux/kubernetes/ssh.tf
Normal file
@ -0,0 +1,58 @@
|
||||
locals {
|
||||
# format assets for distribution
|
||||
assets_bundle = [
|
||||
# header with the unpack location
|
||||
for key, value in module.bootstrap.assets_dist :
|
||||
format("##### %s\n%s", key, value)
|
||||
]
|
||||
}
|
||||
|
||||
# Secure copy assets to controllers.
|
||||
resource "null_resource" "copy-controller-secrets" {
|
||||
count = var.controller_count
|
||||
|
||||
depends_on = [
|
||||
module.bootstrap,
|
||||
]
|
||||
|
||||
connection {
|
||||
type = "ssh"
|
||||
host = local.controllers_ipv4_public[count.index]
|
||||
user = "core"
|
||||
timeout = "15m"
|
||||
}
|
||||
|
||||
provisioner "file" {
|
||||
content = join("\n", local.assets_bundle)
|
||||
destination = "$HOME/assets"
|
||||
}
|
||||
|
||||
provisioner "remote-exec" {
|
||||
inline = [
|
||||
"sudo /opt/bootstrap/layout",
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
# Connect to a controller to perform one-time cluster bootstrap.
|
||||
resource "null_resource" "bootstrap" {
|
||||
depends_on = [
|
||||
null_resource.copy-controller-secrets,
|
||||
module.workers,
|
||||
google_dns_record_set.apiserver,
|
||||
]
|
||||
|
||||
connection {
|
||||
type = "ssh"
|
||||
host = local.controllers_ipv4_public[0]
|
||||
user = "core"
|
||||
timeout = "15m"
|
||||
}
|
||||
|
||||
provisioner "remote-exec" {
|
||||
inline = [
|
||||
"sudo systemctl start bootstrap",
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
132
google-cloud/flatcar-linux/kubernetes/variables.tf
Normal file
132
google-cloud/flatcar-linux/kubernetes/variables.tf
Normal file
@ -0,0 +1,132 @@
|
||||
variable "cluster_name" {
|
||||
type = string
|
||||
description = "Unique cluster name (prepended to dns_zone)"
|
||||
}
|
||||
|
||||
# Google Cloud
|
||||
|
||||
variable "region" {
|
||||
type = string
|
||||
description = "Google Cloud Region (e.g. us-central1, see `gcloud compute regions list`)"
|
||||
}
|
||||
|
||||
variable "dns_zone" {
|
||||
type = string
|
||||
description = "Google Cloud DNS Zone (e.g. google-cloud.example.com)"
|
||||
}
|
||||
|
||||
variable "dns_zone_name" {
|
||||
type = string
|
||||
description = "Google Cloud DNS Zone name (e.g. example-zone)"
|
||||
}
|
||||
|
||||
# instances
|
||||
|
||||
variable "controller_count" {
|
||||
type = number
|
||||
description = "Number of controllers (i.e. masters)"
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "worker_count" {
|
||||
type = number
|
||||
description = "Number of workers"
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "controller_type" {
|
||||
type = string
|
||||
description = "Machine type for controllers (see `gcloud compute machine-types list`)"
|
||||
default = "n1-standard-1"
|
||||
}
|
||||
|
||||
variable "worker_type" {
|
||||
type = string
|
||||
description = "Machine type for controllers (see `gcloud compute machine-types list`)"
|
||||
default = "n1-standard-1"
|
||||
}
|
||||
|
||||
variable "os_image" {
|
||||
type = string
|
||||
description = "Flatcar Linux image for compute instances (e.g. custom-image)"
|
||||
}
|
||||
|
||||
variable "disk_size" {
|
||||
type = number
|
||||
description = "Size of the disk in GB"
|
||||
default = 40
|
||||
}
|
||||
|
||||
variable "worker_preemptible" {
|
||||
type = bool
|
||||
description = "If enabled, Compute Engine will terminate workers randomly within 24 hours"
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "controller_snippets" {
|
||||
type = list(string)
|
||||
description = "Controller Container Linux Config snippets"
|
||||
default = []
|
||||
}
|
||||
|
||||
variable "worker_snippets" {
|
||||
type = list(string)
|
||||
description = "Worker Container Linux Config snippets"
|
||||
default = []
|
||||
}
|
||||
|
||||
# configuration
|
||||
|
||||
variable "ssh_authorized_key" {
|
||||
type = string
|
||||
description = "SSH public key for user 'core'"
|
||||
}
|
||||
|
||||
variable "networking" {
|
||||
type = string
|
||||
description = "Choice of networking provider (flannel or calico)"
|
||||
default = "calico"
|
||||
}
|
||||
|
||||
variable "pod_cidr" {
|
||||
type = string
|
||||
description = "CIDR IPv4 range to assign Kubernetes pods"
|
||||
default = "10.2.0.0/16"
|
||||
}
|
||||
|
||||
variable "service_cidr" {
|
||||
type = string
|
||||
description = <<EOD
|
||||
CIDR IPv4 range to assign Kubernetes services.
|
||||
The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
|
||||
EOD
|
||||
default = "10.3.0.0/16"
|
||||
}
|
||||
|
||||
|
||||
variable "enable_reporting" {
|
||||
type = bool
|
||||
description = "Enable usage or analytics reporting to upstreams (Calico)"
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "enable_aggregation" {
|
||||
type = bool
|
||||
description = "Enable the Kubernetes Aggregation Layer (defaults to false)"
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "worker_node_labels" {
|
||||
type = list(string)
|
||||
description = "List of initial worker node labels"
|
||||
default = []
|
||||
}
|
||||
|
||||
# unofficial, undocumented, unsupported
|
||||
|
||||
variable "cluster_domain_suffix" {
|
||||
type = string
|
||||
description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
|
||||
default = "cluster.local"
|
||||
}
|
||||
|
||||
15
google-cloud/flatcar-linux/kubernetes/versions.tf
Normal file
15
google-cloud/flatcar-linux/kubernetes/versions.tf
Normal file
@ -0,0 +1,15 @@
|
||||
# Terraform version and plugin versions
|
||||
|
||||
terraform {
|
||||
required_version = ">= 0.12.26, < 0.14.0"
|
||||
required_providers {
|
||||
google = ">= 2.19, < 4.0"
|
||||
template = "~> 2.1"
|
||||
null = "~> 2.1"
|
||||
|
||||
ct = {
|
||||
source = "poseidon/ct"
|
||||
version = "~> 0.6.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
23
google-cloud/flatcar-linux/kubernetes/workers.tf
Normal file
23
google-cloud/flatcar-linux/kubernetes/workers.tf
Normal file
@ -0,0 +1,23 @@
|
||||
module "workers" {
|
||||
source = "./workers"
|
||||
name = var.cluster_name
|
||||
cluster_name = var.cluster_name
|
||||
|
||||
# GCE
|
||||
region = var.region
|
||||
network = google_compute_network.network.name
|
||||
worker_count = var.worker_count
|
||||
machine_type = var.worker_type
|
||||
os_image = var.os_image
|
||||
disk_size = var.disk_size
|
||||
preemptible = var.worker_preemptible
|
||||
|
||||
# configuration
|
||||
kubeconfig = module.bootstrap.kubeconfig-kubelet
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
service_cidr = var.service_cidr
|
||||
cluster_domain_suffix = var.cluster_domain_suffix
|
||||
snippets = var.worker_snippets
|
||||
node_labels = var.worker_node_labels
|
||||
}
|
||||
|
||||
115
google-cloud/flatcar-linux/kubernetes/workers/cl/worker.yaml
Normal file
115
google-cloud/flatcar-linux/kubernetes/workers/cl/worker.yaml
Normal file
@ -0,0 +1,115 @@
|
||||
---
|
||||
systemd:
|
||||
units:
|
||||
- name: docker.service
|
||||
enabled: true
|
||||
- name: locksmithd.service
|
||||
mask: true
|
||||
- name: wait-for-dns.service
|
||||
enabled: true
|
||||
contents: |
|
||||
[Unit]
|
||||
Description=Wait for DNS entries
|
||||
Wants=systemd-resolved.service
|
||||
Before=kubelet.service
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=true
|
||||
ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
|
||||
[Install]
|
||||
RequiredBy=kubelet.service
|
||||
- name: kubelet.service
|
||||
enabled: true
|
||||
contents: |
|
||||
[Unit]
|
||||
Description=Kubelet
|
||||
Requires=docker.service
|
||||
After=docker.service
|
||||
Wants=rpc-statd.service
|
||||
[Service]
|
||||
Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
|
||||
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
|
||||
ExecStartPre=/bin/mkdir -p /opt/cni/bin
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/calico
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
|
||||
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
|
||||
# Podman, rkt, or runc run container processes, whereas docker run
|
||||
# is a client to a daemon and requires workarounds to use within a
|
||||
# systemd unit. https://github.com/moby/moby/issues/6791
|
||||
ExecStartPre=/usr/bin/docker run -d \
|
||||
--name kubelet \
|
||||
--privileged \
|
||||
--pid host \
|
||||
--network host \
|
||||
-v /etc/kubernetes:/etc/kubernetes:ro \
|
||||
-v /etc/machine-id:/etc/machine-id:ro \
|
||||
-v /usr/lib/os-release:/etc/os-release:ro \
|
||||
-v /lib/modules:/lib/modules:ro \
|
||||
-v /run:/run \
|
||||
-v /sys/fs/cgroup:/sys/fs/cgroup:ro \
|
||||
-v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \
|
||||
-v /var/lib/calico:/var/lib/calico:ro \
|
||||
-v /var/lib/docker:/var/lib/docker \
|
||||
-v /var/lib/kubelet:/var/lib/kubelet:rshared \
|
||||
-v /var/log:/var/log \
|
||||
-v /opt/cni/bin:/opt/cni/bin \
|
||||
$${KUBELET_IMAGE} \
|
||||
--anonymous-auth=false \
|
||||
--authentication-token-webhook \
|
||||
--authorization-mode=Webhook \
|
||||
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
|
||||
--client-ca-file=/etc/kubernetes/ca.crt \
|
||||
--cluster_dns=${cluster_dns_service_ip} \
|
||||
--cluster_domain=${cluster_domain_suffix} \
|
||||
--cni-conf-dir=/etc/kubernetes/cni/net.d \
|
||||
--healthz-port=0 \
|
||||
--kubeconfig=/var/lib/kubelet/kubeconfig \
|
||||
--network-plugin=cni \
|
||||
--node-labels=node.kubernetes.io/node \
|
||||
%{~ for label in split(",", node_labels) ~}
|
||||
--node-labels=${label} \
|
||||
%{~ endfor ~}
|
||||
--pod-manifest-path=/etc/kubernetes/manifests \
|
||||
--read-only-port=0 \
|
||||
--rotate-certificates \
|
||||
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
|
||||
ExecStart=docker logs -f kubelet
|
||||
ExecStop=docker stop kubelet
|
||||
ExecStopPost=docker rm kubelet
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: delete-node.service
|
||||
enabled: true
|
||||
contents: |
|
||||
[Unit]
|
||||
Description=Delete Kubernetes node on shutdown
|
||||
[Service]
|
||||
Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3
|
||||
Type=oneshot
|
||||
RemainAfterExit=true
|
||||
ExecStart=/bin/true
|
||||
ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME'
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
storage:
|
||||
files:
|
||||
- path: /etc/kubernetes/kubeconfig
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline: |
|
||||
${kubeconfig}
|
||||
- path: /etc/sysctl.d/max-user-watches.conf
|
||||
filesystem: root
|
||||
mode: 0644
|
||||
contents:
|
||||
inline: |
|
||||
fs.inotify.max_user_watches=16184
|
||||
passwd:
|
||||
users:
|
||||
- name: core
|
||||
ssh_authorized_keys:
|
||||
- "${ssh_authorized_key}"
|
||||
14
google-cloud/flatcar-linux/kubernetes/workers/outputs.tf
Normal file
14
google-cloud/flatcar-linux/kubernetes/workers/outputs.tf
Normal file
@ -0,0 +1,14 @@
|
||||
# Outputs for global load balancing
|
||||
|
||||
output "instance_group" {
|
||||
description = "Worker managed instance group full URL"
|
||||
value = google_compute_region_instance_group_manager.workers.instance_group
|
||||
}
|
||||
|
||||
# Outputs for regional load balancing
|
||||
|
||||
output "target_pool" {
|
||||
description = "Worker target pool self link"
|
||||
value = google_compute_target_pool.workers.self_link
|
||||
}
|
||||
|
||||
23
google-cloud/flatcar-linux/kubernetes/workers/target_pool.tf
Normal file
23
google-cloud/flatcar-linux/kubernetes/workers/target_pool.tf
Normal file
@ -0,0 +1,23 @@
|
||||
# Target pool for TCP/UDP load balancing
|
||||
resource "google_compute_target_pool" "workers" {
|
||||
name = "${var.name}-worker-pool"
|
||||
region = var.region
|
||||
session_affinity = "NONE"
|
||||
|
||||
health_checks = [
|
||||
google_compute_http_health_check.workers.name,
|
||||
]
|
||||
}
|
||||
|
||||
# HTTP Health Check (for TCP/UDP load balancing)
|
||||
# Forward rules (regional) to target pools don't support different external
|
||||
# and internal ports. Health check for nodes with Ingress controllers that
|
||||
# may support proxying or otherwise satisfy the check.
|
||||
resource "google_compute_http_health_check" "workers" {
|
||||
name = "${var.name}-target-pool-health"
|
||||
description = "Health check for the worker target pool"
|
||||
|
||||
port = 10254
|
||||
request_path = "/healthz"
|
||||
}
|
||||
|
||||
106
google-cloud/flatcar-linux/kubernetes/workers/variables.tf
Normal file
106
google-cloud/flatcar-linux/kubernetes/workers/variables.tf
Normal file
@ -0,0 +1,106 @@
|
||||
variable "name" {
|
||||
type = string
|
||||
description = "Unique name for the worker pool"
|
||||
}
|
||||
|
||||
variable "cluster_name" {
|
||||
type = string
|
||||
description = "Must be set to `cluster_name of cluster`"
|
||||
}
|
||||
|
||||
# Google Cloud
|
||||
|
||||
variable "region" {
|
||||
type = string
|
||||
description = "Must be set to `region` of cluster"
|
||||
}
|
||||
|
||||
variable "network" {
|
||||
type = string
|
||||
description = "Must be set to `network_name` output by cluster"
|
||||
}
|
||||
|
||||
# instances
|
||||
|
||||
variable "worker_count" {
|
||||
type = number
|
||||
description = "Number of worker compute instances the instance group should manage"
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "machine_type" {
|
||||
type = string
|
||||
description = "Machine type for compute instances (e.g. gcloud compute machine-types list)"
|
||||
default = "n1-standard-1"
|
||||
}
|
||||
|
||||
variable "os_image" {
|
||||
type = string
|
||||
description = "Flatcar Linux image for compute instanges (e.g. gcloud compute images list)"
|
||||
}
|
||||
|
||||
variable "disk_size" {
|
||||
type = number
|
||||
description = "Size of the disk in GB"
|
||||
default = 40
|
||||
}
|
||||
|
||||
variable "preemptible" {
|
||||
type = bool
|
||||
description = "If enabled, Compute Engine will terminate instances randomly within 24 hours"
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "snippets" {
|
||||
type = list(string)
|
||||
description = "Container Linux Config snippets"
|
||||
default = []
|
||||
}
|
||||
|
||||
# configuration
|
||||
|
||||
variable "kubeconfig" {
|
||||
type = string
|
||||
description = "Must be set to `kubeconfig` output by cluster"
|
||||
}
|
||||
|
||||
variable "ssh_authorized_key" {
|
||||
type = string
|
||||
description = "SSH public key for user 'core'"
|
||||
}
|
||||
|
||||
variable "service_cidr" {
|
||||
type = string
|
||||
description = <<EOD
|
||||
CIDR IPv4 range to assign Kubernetes services.
|
||||
The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for coredns.
|
||||
EOD
|
||||
default = "10.3.0.0/16"
|
||||
}
|
||||
|
||||
variable "node_labels" {
|
||||
type = list(string)
|
||||
description = "List of initial node labels"
|
||||
default = []
|
||||
}
|
||||
|
||||
# unofficial, undocumented, unsupported, temporary
|
||||
|
||||
variable "cluster_domain_suffix" {
|
||||
type = string
|
||||
description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
|
||||
default = "cluster.local"
|
||||
}
|
||||
|
||||
variable "accelerator_type" {
|
||||
type = string
|
||||
default = ""
|
||||
description = "Google Compute Engine accelerator type (e.g. nvidia-tesla-k80, see gcloud compute accelerator-types list)"
|
||||
}
|
||||
|
||||
variable "accelerator_count" {
|
||||
type = string
|
||||
default = "0"
|
||||
description = "Number of compute engine accelerators"
|
||||
}
|
||||
|
||||
14
google-cloud/flatcar-linux/kubernetes/workers/versions.tf
Normal file
14
google-cloud/flatcar-linux/kubernetes/workers/versions.tf
Normal file
@ -0,0 +1,14 @@
|
||||
# Terraform version and plugin versions
|
||||
|
||||
terraform {
|
||||
required_version = ">= 0.12.26, < 0.14.0"
|
||||
required_providers {
|
||||
google = ">= 2.19, < 4.0"
|
||||
template = "~> 2.1"
|
||||
|
||||
ct = {
|
||||
source = "poseidon/ct"
|
||||
version = "~> 0.6.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
91
google-cloud/flatcar-linux/kubernetes/workers/workers.tf
Normal file
91
google-cloud/flatcar-linux/kubernetes/workers/workers.tf
Normal file
@ -0,0 +1,91 @@
|
||||
# Managed instance group of workers
|
||||
resource "google_compute_region_instance_group_manager" "workers" {
|
||||
name = "${var.name}-worker-group"
|
||||
description = "Compute instance group of ${var.name} workers"
|
||||
|
||||
# instance name prefix for instances in the group
|
||||
base_instance_name = "${var.name}-worker"
|
||||
region = var.region
|
||||
version {
|
||||
name = "default"
|
||||
instance_template = google_compute_instance_template.worker.self_link
|
||||
}
|
||||
|
||||
target_size = var.worker_count
|
||||
target_pools = [google_compute_target_pool.workers.self_link]
|
||||
|
||||
named_port {
|
||||
name = "http"
|
||||
port = "80"
|
||||
}
|
||||
|
||||
named_port {
|
||||
name = "https"
|
||||
port = "443"
|
||||
}
|
||||
}
|
||||
|
||||
# Worker instance template
|
||||
resource "google_compute_instance_template" "worker" {
|
||||
name_prefix = "${var.name}-worker-"
|
||||
description = "Worker Instance template"
|
||||
machine_type = var.machine_type
|
||||
|
||||
metadata = {
|
||||
user-data = data.ct_config.worker-ignition.rendered
|
||||
}
|
||||
|
||||
scheduling {
|
||||
automatic_restart = var.preemptible ? false : true
|
||||
preemptible = var.preemptible
|
||||
}
|
||||
|
||||
disk {
|
||||
auto_delete = true
|
||||
boot = true
|
||||
source_image = var.os_image
|
||||
disk_size_gb = var.disk_size
|
||||
}
|
||||
|
||||
network_interface {
|
||||
network = var.network
|
||||
|
||||
# Ephemeral external IP
|
||||
access_config {
|
||||
}
|
||||
}
|
||||
|
||||
can_ip_forward = true
|
||||
tags = ["worker", "${var.cluster_name}-worker", "${var.name}-worker"]
|
||||
|
||||
guest_accelerator {
|
||||
count = var.accelerator_count
|
||||
type = var.accelerator_type
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
# To update an Instance Template, Terraform should replace the existing resource
|
||||
create_before_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
# Worker Ignition config
|
||||
data "ct_config" "worker-ignition" {
|
||||
content = data.template_file.worker-config.rendered
|
||||
strict = true
|
||||
snippets = var.snippets
|
||||
}
|
||||
|
||||
# Worker Container Linux config
|
||||
data "template_file" "worker-config" {
|
||||
template = file("${path.module}/cl/worker.yaml")
|
||||
|
||||
vars = {
|
||||
kubeconfig = indent(10, var.kubeconfig)
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
|
||||
cluster_domain_suffix = var.cluster_domain_suffix
|
||||
node_labels = join(",", var.node_labels)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user