mirror of
https://github.com/puppetmaster/typhoon.git
synced 2024-12-26 21:09:34 +01:00
413585681b
* Kubelet `--lock-file` and `--exit-on-lock-contention` date back to usage of bootkube and at one point running Kubelet in a "self-hosted" style whereby an on-host Kubelet (rkt) started pods, but then a Kubelet DaemonSet was scheduled and able to take over (hence self-hosted). `lock-file` and `exit-on-lock-contention` flags supported this pivot. The pattern has been out of favor (in bootkube too) for years because of dueling Kubelet complexity * Typhoon runs Kubelet as a container via an on-host systemd unit using podman (Fedora CoreOS) or rkt (Flatcar Linux). In fact, Typhoon no longer uses bootkube or control plane pivot (let alone Kubelet pivot) and uses static pods since v1.16.0 * https://github.com/poseidon/typhoon/pull/536
199 lines
8.7 KiB
YAML
199 lines
8.7 KiB
YAML
---
|
|
systemd:
|
|
units:
|
|
- name: etcd-member.service
|
|
enabled: true
|
|
dropins:
|
|
- name: 40-etcd-cluster.conf
|
|
contents: |
|
|
[Service]
|
|
Environment="ETCD_IMAGE_TAG=v3.4.9"
|
|
Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
|
|
Environment="RKT_RUN_ARGS=--insecure-options=image"
|
|
Environment="ETCD_NAME=${etcd_name}"
|
|
Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
|
|
Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
|
|
Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
|
|
Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
|
|
Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
|
|
Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
|
|
Environment="ETCD_STRICT_RECONFIG_CHECK=true"
|
|
Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
|
|
Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
|
|
Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
|
|
Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
|
|
Environment="ETCD_CLIENT_CERT_AUTH=true"
|
|
Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
|
|
Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
|
|
Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
|
|
Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
|
|
- name: docker.service
|
|
enabled: true
|
|
- name: locksmithd.service
|
|
mask: true
|
|
- name: wait-for-dns.service
|
|
enabled: true
|
|
contents: |
|
|
[Unit]
|
|
Description=Wait for DNS entries
|
|
Wants=systemd-resolved.service
|
|
Before=kubelet.service
|
|
[Service]
|
|
Type=oneshot
|
|
RemainAfterExit=true
|
|
ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done'
|
|
[Install]
|
|
RequiredBy=kubelet.service
|
|
RequiredBy=etcd-member.service
|
|
- name: kubelet.service
|
|
enabled: true
|
|
contents: |
|
|
[Unit]
|
|
Description=Kubelet
|
|
Wants=rpc-statd.service
|
|
[Service]
|
|
Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a
|
|
ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
|
|
ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
|
|
ExecStartPre=/bin/mkdir -p /opt/cni/bin
|
|
ExecStartPre=/bin/mkdir -p /var/lib/calico
|
|
ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
|
|
ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
|
|
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
|
|
ExecStart=/usr/bin/rkt run \
|
|
--uuid-file-save=/var/cache/kubelet-pod.uuid \
|
|
--stage1-from-dir=stage1-fly.aci \
|
|
--hosts-entry host \
|
|
--insecure-options=image \
|
|
--volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \
|
|
--mount volume=etc-kubernetes,target=/etc/kubernetes \
|
|
--volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \
|
|
--mount volume=etc-machine-id,target=/etc/machine-id \
|
|
--volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \
|
|
--mount volume=etc-os-release,target=/etc/os-release \
|
|
--volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
|
|
--mount volume=etc-resolv,target=/etc/resolv.conf \
|
|
--volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
|
|
--mount volume=etc-ssl-certs,target=/etc/ssl/certs \
|
|
--volume lib-modules,kind=host,source=/lib/modules,readOnly=true \
|
|
--mount volume=lib-modules,target=/lib/modules \
|
|
--volume run,kind=host,source=/run \
|
|
--mount volume=run,target=/run \
|
|
--volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
|
|
--mount volume=usr-share-certs,target=/usr/share/ca-certificates \
|
|
--volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \
|
|
--mount volume=var-lib-calico,target=/var/lib/calico \
|
|
--volume var-lib-docker,kind=host,source=/var/lib/docker \
|
|
--mount volume=var-lib-docker,target=/var/lib/docker \
|
|
--volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \
|
|
--mount volume=var-lib-kubelet,target=/var/lib/kubelet \
|
|
--volume var-log,kind=host,source=/var/log \
|
|
--mount volume=var-log,target=/var/log \
|
|
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
|
|
--mount volume=opt-cni-bin,target=/opt/cni/bin \
|
|
$${KUBELET_IMAGE} -- \
|
|
--anonymous-auth=false \
|
|
--authentication-token-webhook \
|
|
--authorization-mode=Webhook \
|
|
--bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
|
|
--client-ca-file=/etc/kubernetes/ca.crt \
|
|
--cluster_dns=${cluster_dns_service_ip} \
|
|
--cluster_domain=${cluster_domain_suffix} \
|
|
--cni-conf-dir=/etc/kubernetes/cni/net.d \
|
|
--healthz-port=0 \
|
|
--kubeconfig=/var/lib/kubelet/kubeconfig \
|
|
--network-plugin=cni \
|
|
--node-labels=node.kubernetes.io/master \
|
|
--node-labels=node.kubernetes.io/controller="true" \
|
|
--pod-manifest-path=/etc/kubernetes/manifests \
|
|
--register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
|
|
--read-only-port=0 \
|
|
--rotate-certificates \
|
|
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
|
|
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
|
|
Restart=always
|
|
RestartSec=10
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
- name: bootstrap.service
|
|
contents: |
|
|
[Unit]
|
|
Description=Kubernetes control plane
|
|
ConditionPathExists=!/opt/bootstrap/bootstrap.done
|
|
[Service]
|
|
Type=oneshot
|
|
RemainAfterExit=true
|
|
WorkingDirectory=/opt/bootstrap
|
|
ExecStart=/usr/bin/rkt run \
|
|
--trust-keys-from-https \
|
|
--volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \
|
|
--mount volume=config,target=/etc/kubernetes/secrets \
|
|
--volume assets,kind=host,source=/opt/bootstrap/assets \
|
|
--mount volume=assets,target=/assets \
|
|
--volume script,kind=host,source=/opt/bootstrap/apply \
|
|
--mount volume=script,target=/apply \
|
|
--insecure-options=image \
|
|
docker://quay.io/poseidon/kubelet:v1.18.3-4-g57fa88a \
|
|
--net=host \
|
|
--dns=host \
|
|
--exec=/apply
|
|
ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
storage:
|
|
files:
|
|
- path: /etc/kubernetes/kubeconfig
|
|
filesystem: root
|
|
mode: 0644
|
|
contents:
|
|
inline: |
|
|
${kubeconfig}
|
|
- path: /opt/bootstrap/layout
|
|
filesystem: root
|
|
mode: 0544
|
|
contents:
|
|
inline: |
|
|
#!/bin/bash -e
|
|
mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
|
|
awk '/#####/ {filename=$2; next} {print > filename}' assets
|
|
mkdir -p /etc/ssl/etcd/etcd
|
|
mkdir -p /etc/kubernetes/bootstrap-secrets
|
|
mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
|
|
mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/
|
|
chown -R etcd:etcd /etc/ssl/etcd
|
|
chmod -R 500 /etc/ssl/etcd
|
|
mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/
|
|
mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/
|
|
mkdir -p /etc/kubernetes/manifests
|
|
mv static-manifests/* /etc/kubernetes/manifests/
|
|
mkdir -p /opt/bootstrap/assets
|
|
mv manifests /opt/bootstrap/assets/manifests
|
|
mv manifests-networking/* /opt/bootstrap/assets/manifests/
|
|
rm -rf assets auth static-manifests tls manifests-networking
|
|
- path: /opt/bootstrap/apply
|
|
filesystem: root
|
|
mode: 0544
|
|
contents:
|
|
inline: |
|
|
#!/bin/bash -e
|
|
export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig
|
|
until kubectl version; do
|
|
echo "Waiting for static pod control plane"
|
|
sleep 5
|
|
done
|
|
until kubectl apply -f /assets/manifests -R; do
|
|
echo "Retry applying manifests"
|
|
sleep 5
|
|
done
|
|
- path: /etc/sysctl.d/max-user-watches.conf
|
|
filesystem: root
|
|
mode: 0644
|
|
contents:
|
|
inline: |
|
|
fs.inotify.max_user_watches=16184
|
|
passwd:
|
|
users:
|
|
- name: core
|
|
ssh_authorized_keys:
|
|
- "${ssh_authorized_key}"
|