From fd044ee1174b3e9a79582381ac59660b98825628 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Apr 2020 16:50:51 -0700 Subject: [PATCH] Enable Kubelet TLS bootstrap and NodeRestriction * Enable bootstrap token authentication on kube-apiserver * Generate the bootstrap.kubernetes.io/token Secret that may be used as a bootstrap token * Generate a bootstrap kubeconfig (with a bootstrap token) to be securely distributed to nodes. Each Kubelet will use the bootstrap kubeconfig to authenticate to kube-apiserver as `system:bootstrappers` and send a node-unique CSR for kube-controller-manager to automatically approve to issue a Kubelet certificate and kubeconfig (expires in 72 hours) * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the `system:node-bootstrapper` ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr nodeclient ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr selfnodeclient ClusterRole * Enable NodeRestriction admission controller to limit the scope of Node or Pod objects a Kubelet can modify to those of the node itself * Ability for a Kubelet to delete its Node object is retained as preemptible nodes or those in auto-scaling instance groups need to be able to remove themselves on shutdown. This need continues to have precedence over any risk of a node deleting itself maliciously Security notes: 1. Issued Kubelet certificates authenticate as user `system:node:NAME` and group `system:nodes` and are limited in their authorization to perform API operations by Node authorization and NodeRestriction admission. Previously, a Kubelet's authorization was broader. This is the primary security motivation. 2. The bootstrap kubeconfig credential has the same sensitivity as the previous generated TLS client-certificate kubeconfig. It must be distributed securely to nodes. Its compromise still allows an attacker to obtain a Kubelet kubeconfig 3. Bootstrapping Kubelet kubeconfig's with a limited lifetime offers a slight security improvement. * An attacker who obtains the kubeconfig can likely obtain the bootstrap kubeconfig as well, to obtain the ability to renew their access * A compromised bootstrap kubeconfig could plausibly be handled by replacing the bootstrap token Secret, distributing the token to new nodes, and expiration. Whereas a compromised TLS-client certificate kubeconfig can't be revoked (no CRL). However, replacing a bootstrap token can be impractical in real cluster environments, so the limited lifetime is mostly a theoretical benefit. * Cluster CSR objects are visible via kubectl which is nice 4. Bootstrapping node-unique Kubelet kubeconfigs means Kubelet clients have more identity information, which can improve the utility of audits and future features Rel: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/ Rel: https://github.com/poseidon/terraform-render-bootstrap/pull/185 --- CHANGES.md | 3 +++ addons/prometheus/rules.yaml | 8 ++++---- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/container-linux/kubernetes/cl/controller.yaml | 6 ++++-- aws/container-linux/kubernetes/workers/cl/worker.yaml | 6 ++++-- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 6 ++++-- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/cl/controller.yaml | 6 ++++-- azure/container-linux/kubernetes/workers/cl/worker.yaml | 6 ++++-- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml | 6 ++++-- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/cl/controller.yaml | 6 ++++-- bare-metal/container-linux/kubernetes/cl/worker.yaml | 6 ++++-- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml | 6 ++++-- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 6 ++++-- digital-ocean/container-linux/kubernetes/cl/worker.yaml | 6 ++++-- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- .../fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml | 6 ++++-- docs/topics/security.md | 6 ++++-- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- .../container-linux/kubernetes/cl/controller.yaml | 6 ++++-- .../container-linux/kubernetes/workers/cl/worker.yaml | 6 ++++-- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml | 6 ++++-- .../fedora-coreos/kubernetes/workers/fcc/worker.yaml | 6 ++++-- 33 files changed, 101 insertions(+), 56 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6a28683d..96ef5453 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,9 @@ Notable changes between versions. ## Latest +* Use Kubelet [TLS bootstrap](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/) with bootstrap token authentication ([#713](https://github.com/poseidon/typhoon/pull/713)) + * Enable Node [Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) and [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction) to reduce authorization scope + * Renew Kubelet certificates every 72 hours * Update Calico from v3.13.1 to [v3.13.3](https://docs.projectcalico.org/v3.13/release-notes/) #### Addons diff --git a/addons/prometheus/rules.yaml b/addons/prometheus/rules.yaml index 69026c1e..359cad7e 100644 --- a/addons/prometheus/rules.yaml +++ b/addons/prometheus/rules.yaml @@ -882,10 +882,10 @@ data: { "alert": "KubeClientCertificateExpiration", "annotations": { - "message": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.", + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 1.0 hours.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 3600\n", "labels": { "severity": "warning" } @@ -893,10 +893,10 @@ data: { "alert": "KubeClientCertificateExpiration", "annotations": { - "message": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.", + "message": "A client certificate used to authenticate to the apiserver is expiring in less than 0.1 hours.", "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" }, - "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n", + "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 300\n", "labels": { "severity": "critical" } diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 860821ae..9a371c89 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 9beb4a8b..2506dd46 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -49,7 +49,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -95,6 +95,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -102,7 +103,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -110,6 +111,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 63a1e3b6..0924c634 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -22,7 +22,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -68,6 +68,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -75,7 +76,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -84,6 +85,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 1991cb13..64ffb08d 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index 727960d7..c7474e10 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -51,7 +51,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -83,6 +83,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -92,7 +93,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -100,6 +101,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 0501b8de..865fb9d8 100644 --- a/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -53,6 +53,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -62,7 +63,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -71,6 +72,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 4495381d..1ab4a476 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 293e3b45..aa0e6941 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -49,7 +49,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -94,13 +94,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -108,6 +109,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index e070c975..2e1d74b2 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -22,7 +22,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -67,13 +67,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -82,6 +83,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 6b4c90ff..01c74b1f 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/fcc/controller.yaml b/azure/fedora-coreos/kubernetes/fcc/controller.yaml index c74ecf58..035667c0 100644 --- a/azure/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/azure/fedora-coreos/kubernetes/fcc/controller.yaml @@ -51,7 +51,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -83,6 +83,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -92,7 +93,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -100,6 +101,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 664f0c6a..056cbafd 100644 --- a/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -53,6 +53,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -62,7 +63,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -71,6 +72,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 45179f95..6cae7504 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 9cde76e8..9f5c5f56 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -57,7 +57,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -107,6 +107,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -115,7 +116,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -123,6 +124,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index 164323cf..dde9d758 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -30,7 +30,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -80,6 +80,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -88,7 +89,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -100,6 +101,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index e3851ddf..587d4f1e 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index 63021316..14cbfdb8 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -50,7 +50,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -84,6 +84,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -94,7 +95,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -102,6 +103,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml index 5133d36e..f61fe8d6 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/worker.yaml @@ -20,7 +20,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -54,6 +54,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -64,7 +65,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -76,6 +77,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index b256588a..af5e3e46 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index c7a29751..08f65372 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -57,7 +57,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service @@ -105,6 +105,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ @@ -112,7 +113,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -120,6 +121,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 2248e077..c2f9419d 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -30,7 +30,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service @@ -78,6 +78,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ @@ -85,12 +86,13 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 002360c3..3bd95bbe 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml index 99934057..67106af2 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/controller.yaml @@ -50,7 +50,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Requires=afterburn.service After=afterburn.service Wants=rpc-statd.service @@ -85,6 +85,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -95,7 +96,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -103,6 +104,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml index 8209e4e0..a18b5052 100644 --- a/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Requires=afterburn.service After=afterburn.service Wants=rpc-statd.service @@ -56,6 +56,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -66,12 +67,13 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=$${AFTERBURN_DIGITALOCEAN_IPV4_PRIVATE_0} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/docs/topics/security.md b/docs/topics/security.md index fc4cbb42..e8949c83 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -7,8 +7,10 @@ Typhoon aims to be minimal and secure. We're running it ourselves after all. **Kubernetes** * etcd with peer-to-peer and client-auth TLS -* Generated kubelet TLS certificates and `kubeconfig` (365 days) -* [Role-Based Access Control](https://kubernetes.io/docs/admin/authorization/rbac/) is enabled. Apps must define RBAC policies +* Kubelets TLS bootstrap certificates (72 hours) +* Generated TLS certificate (365 days) for admin `kubeconfig` +* [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/node/) is enabled to limit Kubelet authorization +* [Role-Based Access Control](https://kubernetes.io/docs/admin/authorization/rbac/) is enabled. Apps must define RBAC policies for API access * Workloads run on worker nodes only, unless they tolerate the master taint * Kubernetes [Network Policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) and Calico [NetworkPolicy](https://docs.projectcalico.org/latest/reference/calicoctl/resources/networkpolicy) support [^1] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index f95498fb..49af2d10 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index 1c88d409..30fbee4d 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -49,7 +49,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -94,13 +94,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -108,6 +109,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 52a0cfae..0bb704e2 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -22,7 +22,7 @@ systemd: enable: true contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -67,13 +67,14 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ --cluster_domain=${cluster_domain_suffix} \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -82,6 +83,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index ef6937a2..0b3b41d7 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml index 727960d7..c7474e10 100644 --- a/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/fcc/controller.yaml @@ -51,7 +51,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -83,6 +83,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -92,7 +93,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -100,6 +101,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes diff --git a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml index 0501b8de..865fb9d8 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/fcc/worker.yaml @@ -21,7 +21,7 @@ systemd: enabled: true contents: | [Unit] - Description=Kubelet via Hyperkube (System Container) + Description=Kubelet (System Container) Wants=rpc-statd.service [Service] ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d @@ -53,6 +53,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=systemd \ --cgroups-per-qos=true \ --enforce-node-allocatable=pods \ @@ -62,7 +63,7 @@ systemd: --cni-conf-dir=/etc/kubernetes/cni/net.d \ --exit-on-lock-contention \ --healthz-port=0 \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -71,6 +72,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/podman stop kubelet Delegate=yes