From f1f4cd6fc02b0530cbcc8951ce1edb6e56bdca69 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 29 Dec 2019 11:17:26 -0800 Subject: [PATCH] Inline Container Linux kubelet.service, deprecate kubelet-wrapper * Change kubelet.service on Container Linux nodes to ExecStart Kubelet inline to replace the use of the host OS kubelet-wrapper script * Express rkt run flags and volume mounts in a clear, uniform way to make the Kubelet service easier to audit, manage, and understand * Eliminate reliance on a Container Linux kubelet-wrapper script * Typhoon for Fedora CoreOS developed a kubelet.service that similarly uses an inline ExecStart (except with podman instead of rkt) and a more minimal set of volume mounts. Adopt the volume improvements: * Change Kubelet /etc/kubernetes volume to read-only * Change Kubelet /etc/resolv.conf volume to read-only * Remove unneeded /var/lib/cni volume mount Background: * kubelet-wrapper was added in CoreOS around the time of Kubernetes v1.0 to simplify running a CoreOS-built hyperkube ACI image via rkt-fly. The script defaults are no longer ideal (e.g. rkt's notion of trust dates back to quay.io ACI image serving and signing, which informed the OCI standard images we use today, though they still lack rkt's signing ideas). * Shipping kubelet-wrapper was regretted at CoreOS, but remains in the distro for compatibility. The script is not updated to track hyperkube changes, but it is stable and kubelet.env overrides bridge most gaps * Typhoon Container Linux nodes have used kubelet-wrapper to rkt/rkt-fly run the Kubelet via the official k8s.gcr.io hyperkube image using overrides (new image registry, new image format, restart handling, new mounts, new entrypoint in v1.17). * Observation: Most of what it takes to run a Kubelet container is defined in Typhoon, not in kubelet-wrapper. The wrapper's value is now undermined by having to workaround its dated defaults. Typhoon may be better served defining Kubelet.service explicitly * Typhoon for Fedora CoreOS developed a kubelet.service without the use of a host OS kubelet-wrapper which is both clearer and eliminated some volume mounts --- CHANGES.md | 1 + .../kubernetes/cl/controller.yaml | 56 +++++++++------- .../kubernetes/workers/cl/worker.yaml | 58 ++++++++++------- .../kubernetes/cl/controller.yaml | 58 ++++++++++------- .../kubernetes/workers/cl/worker.yaml | 58 ++++++++++------- .../kubernetes/cl/controller.yaml | 64 +++++++++++-------- .../container-linux/kubernetes/cl/worker.yaml | 64 +++++++++++-------- .../kubernetes/cl/controller.yaml | 56 +++++++++------- .../container-linux/kubernetes/cl/worker.yaml | 56 +++++++++------- .../kubernetes/cl/controller.yaml | 57 ++++++++++------- .../kubernetes/workers/cl/worker.yaml | 57 ++++++++++------- 11 files changed, 342 insertions(+), 243 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 93282a36..e37a0200 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Notable changes between versions. * Update CoreDNS from v1.6.5 to [v1.6.6](https://coredns.io/2019/12/11/coredns-1.6.6-release/) ([#602](https://github.com/poseidon/typhoon/pull/602)) * Update Calico from v3.10.2 to v3.11.1 ([#604](https://github.com/poseidon/typhoon/pull/604)) +* Inline Kubelet service on Container Linux nodes ([#606](https://github.com/poseidon/typhoon/pull/606)) #### Addons diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index 9240653c..95fb29bd 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -50,29 +50,47 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -130,14 +148,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 6a36c8ce..ba27b829 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,29 +25,47 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -92,14 +110,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index 80f53cd3..236f65e9 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -50,28 +50,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -128,14 +146,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index d29ecb36..ab099292 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,28 +25,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -90,14 +108,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index ddc943f9..42e7d396 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -58,33 +58,51 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume iscsiconf,kind=host,source=/etc/iscsi/ \ - --mount volume=iscsiconf,target=/etc/iscsi/ \ - --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=iscsiadm,target=/sbin/iscsiadm \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + --volume etc-iscsi,kind=host,source=/etc/iscsi \ + --mount volume=etc-iscsi,target=/etc/iscsi \ + --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ + --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -137,14 +155,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index efe18ddc..c2b78e7a 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -33,33 +33,51 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume iscsiconf,kind=host,source=/etc/iscsi/ \ - --mount volume=iscsiconf,target=/etc/iscsi/ \ - --volume iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=iscsiadm,target=/sbin/iscsiadm \ - --insecure-options=image" Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + --volume etc-iscsi,kind=host,source=/etc/iscsi \ + --mount volume=etc-iscsi,target=/etc/iscsi \ + --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ + --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -85,14 +103,6 @@ systemd: storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/hostname filesystem: root mode: 0644 diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index d51f24fe..e654fb45 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -60,29 +60,47 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env EnvironmentFile=/run/metadata/coreos - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -134,14 +152,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 4ae4603e..00701403 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -35,29 +35,47 @@ systemd: After=coreos-metadata.service Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env EnvironmentFile=/run/metadata/coreos - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -93,14 +111,6 @@ systemd: WantedBy=multi-user.target storage: files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index b2baaa04..c48e90eb 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -50,29 +50,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --hosts-entry=host \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -129,14 +146,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /opt/bootstrap/layout filesystem: root mode: 0544 diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index 4d4bae02..f6e5bd5d 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -25,29 +25,46 @@ systemd: Description=Kubelet via Hyperkube Wants=rpc-statd.service [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-lib-calico,kind=host,source=/var/lib/calico \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --hosts-entry=host \ - --insecure-options=image" ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/usr/bin/rkt run \ + --uuid-file-save=/var/cache/kubelet-pod.uuid \ + --stage1-from-dir=stage1-fly.aci \ + --hosts-entry host \ + --insecure-options=image \ + --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount volume=etc-kubernetes,target=/etc/kubernetes \ + --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ + --mount volume=etc-machine-id,target=/etc/machine-id \ + --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ + --mount volume=etc-os-release,target=/etc/os-release \ + --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ + --mount volume=etc-resolv,target=/etc/resolv.conf \ + --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ + --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ + --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ + --mount volume=lib-modules,target=/lib/modules \ + --volume run,kind=host,source=/run \ + --mount volume=run,target=/run \ + --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ + --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ + --volume var-lib-calico,kind=host,source=/var/lib/calico \ + --mount volume=var-lib-calico,target=/var/lib/calico \ + --volume var-lib-docker,kind=host,source=/var/lib/docker \ + --mount volume=var-lib-docker,target=/var/lib/docker \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ + --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ + --volume var-log,kind=host,source=/var/log \ + --mount volume=var-log,target=/var/log \ + --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ + --mount volume=opt-cni-bin,target=/opt/cni/bin \ + docker://k8s.gcr.io/hyperkube:v1.17.0 \ + --exec=/usr/local/bin/kubelet -- \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -91,14 +108,6 @@ storage: contents: inline: | ${kubeconfig} - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube - KUBELET_IMAGE_TAG=v1.17.0 - KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet" - path: /etc/sysctl.d/max-user-watches.conf filesystem: root contents: