From 0d6410505d8d65cef94a1a3f2f921327e88adfd2 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 14 Sep 2017 08:41:17 -0700 Subject: [PATCH] bare-metal: Update kubelet.service unit to match upstream * Mount host /opt/cni/bin in Kubelet to use host's CNI plugins * Switch /var/run/kubelet-pod.uuid to /var/cache/kubelet-pod.uuid to persist between reboots and cleanup old Kubelet pods * Organize Kubelet flags in alphabetical order --- .../kubernetes/cl/controller.yaml.tmpl | 35 ++++++++++--------- .../kubernetes/cl/worker.yaml.tmpl | 33 +++++++++-------- .../pxe-worker/cl/bootkube-worker.yaml.tmpl | 33 +++++++++-------- 3 files changed, 55 insertions(+), 46 deletions(-) diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index 9719a4c6..9e41eb25 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -58,37 +58,40 @@ systemd: Description=Kubelet via Hyperkube ACI [Service] EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ + Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ --volume var-lib-cni,kind=host,source=/var/lib/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log" + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged \ - --hostname-override={{.domain_name}} \ - --node-labels=node-role.kubernetes.io/master \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --anonymous-auth=false \ + --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns={{.k8s_dns_service_ip}} \ - --cluster_domain=cluster.local - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid + --cluster_domain=cluster.local \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --hostname-override={{.domain_name}} \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/master \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --require-kubeconfig + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always RestartSec=10 [Install] diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl index 3ee584bc..60207f3e 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl @@ -33,36 +33,39 @@ systemd: Description=Kubelet via Hyperkube ACI [Service] EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ + Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ --volume var-lib-cni,kind=host,source=/var/lib/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log" + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged \ - --hostname-override={{.domain_name}} \ - --node-labels=node-role.kubernetes.io/node \ + --anonymous-auth=false \ + --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns={{.k8s_dns_service_ip}} \ - --cluster_domain=cluster.local - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid + --cluster_domain=cluster.local \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --hostname-override={{.domain_name}} \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/node \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --require-kubeconfig + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always RestartSec=5 [Install] diff --git a/bare-metal/container-linux/pxe-worker/cl/bootkube-worker.yaml.tmpl b/bare-metal/container-linux/pxe-worker/cl/bootkube-worker.yaml.tmpl index 90a12d43..19256f5e 100644 --- a/bare-metal/container-linux/pxe-worker/cl/bootkube-worker.yaml.tmpl +++ b/bare-metal/container-linux/pxe-worker/cl/bootkube-worker.yaml.tmpl @@ -33,36 +33,39 @@ systemd: Description=Kubelet via Hyperkube ACI [Service] EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ + Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ --volume var-lib-cni,kind=host,source=/var/lib/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log" + ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid + ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged \ - --hostname-override={{.domain_name}} \ - --node-labels=node-role.kubernetes.io/node \ + --anonymous-auth=false \ + --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns={{.k8s_dns_service_ip}} \ - --cluster_domain=cluster.local - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid + --cluster_domain=cluster.local \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --exit-on-lock-contention \ + --hostname-override={{.domain_name}} \ + --kubeconfig=/etc/kubernetes/kubeconfig \ + --lock-file=/var/run/lock/kubelet.lock \ + --network-plugin=cni \ + --node-labels=node-role.kubernetes.io/node \ + --pod-manifest-path=/etc/kubernetes/manifests \ + --require-kubeconfig + ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always RestartSec=5 [Install]