From 9b88d4bbfd820e1f30e40f9373a5fa746bec4130 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Tue, 17 Apr 2018 23:31:09 -0700 Subject: [PATCH] Use bootkube system container on fedora-atomic * Use the upstream bootkube image packaged with the required metadata to be usable as a system container under systemd * Run bootkube with runc so no host level components use Docker any more. Docker is still the runtime * Remove bootkube script and old systemd unit --- .../kubernetes/cloudinit/controller.yaml.tmpl | 31 ++--------------- aws/fedora-atomic/kubernetes/ssh.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 33 +++---------------- bare-metal/fedora-atomic/kubernetes/ssh.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 31 ++--------------- digital-ocean/fedora-atomic/kubernetes/ssh.tf | 2 +- .../kubernetes/cloudinit/controller.yaml.tmpl | 31 ++--------------- google-cloud/fedora-atomic/kubernetes/ssh.tf | 2 +- 8 files changed, 17 insertions(+), 117 deletions(-) diff --git a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index fe9c7c71..0d7ec241 100644 --- a/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/aws/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -70,33 +70,7 @@ write_files: permissions: '0644' content: | ${kubeconfig} - - path: /etc/systemd/system/bootkube.service - content: | - [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/var/bootkube/init_bootkube.done - [Service] - Type=oneshot - RemainAfterExit=true - WorkingDirectory=/var/bootkube - ExecStartPre=/bin/mkdir -p /var/bootkube - ExecStart=/usr/local/bin/bootkube-start - ExecStartPost=/bin/touch /var/bootkube/init_bootkube.done - [Install] - WantedBy=multi-user.target - - path: /var/bootkube/.keep - - path: /usr/local/bin/bootkube-start - permissions: '0755' - content: | - #!/bin/bash -e - # Wrapper for bootkube start - [ -n "$(ls /var/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /var/bootkube/assets/manifests-*/* /var/bootkube/assets/manifests && rm -rf /var/bootkube/assets/manifests-* - /usr/bin/docker run --rm --name bootkube \ - --net=host \ - --volume /etc/kubernetes:/etc/kubernetes:Z \ - --volume /var/bootkube/assets:/assets:Z \ - --entrypoint=/bootkube \ - quay.io/coreos/bootkube:v0.12.0 start --asset-dir=/assets + - path: /var/lib/bootkube/.keep - path: /etc/selinux/config owner: root:root permissions: '0644' @@ -109,9 +83,10 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/dghubble/etcd:0265e6680d2533f3fbf4512af868d29ff07451ca" + - "atomic install --system --name=kubelet quay.io/dghubble/kubelet:8baca5cbaf7b7ee0710380c7d8897e444ebdcb27" + - "atomic install --system --name=bootkube quay.io/dghubble/bootkube:3cc2345503c60186db5272fa918514259e3c4a9d" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, cloud-metadata.service] - - "atomic install --system --name=kubelet quay.io/dghubble/kubelet:8baca5cbaf7b7ee0710380c7d8897e444ebdcb27" - [systemctl, start, --no-block, kubelet.service] users: - default diff --git a/aws/fedora-atomic/kubernetes/ssh.tf b/aws/fedora-atomic/kubernetes/ssh.tf index ec455481..c72a09e4 100644 --- a/aws/fedora-atomic/kubernetes/ssh.tf +++ b/aws/fedora-atomic/kubernetes/ssh.tf @@ -82,7 +82,7 @@ resource "null_resource" "bootkube-start" { provisioner "remote-exec" { inline = [ "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/bootkube", + "sudo mv $HOME/assets /var/lib/bootkube", "sudo systemctl start bootkube", ] } diff --git a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index c913fe96..cdeb72d9 100644 --- a/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/bare-metal/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -60,48 +60,23 @@ write_files: PathExists=/etc/kubernetes/kubeconfig [Install] WantedBy=multi-user.target - - path: /etc/systemd/system/bootkube.service - content: | - [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/var/bootkube/init_bootkube.done - [Service] - Type=oneshot - RemainAfterExit=true - WorkingDirectory=/var/bootkube - ExecStartPre=/bin/mkdir -p /var/bootkube - ExecStart=/usr/local/bin/bootkube-start - ExecStartPost=/bin/touch /var/bootkube/init_bootkube.done - [Install] - WantedBy=multi-user.target - - path: /var/bootkube/.keep + - path: /var/lib/bootkube/.keep - path: /etc/selinux/config owner: root:root permissions: '0644' content: | SELINUX=permissive SELINUXTYPE=targeted - - path: /usr/local/bin/bootkube-start - permissions: '0755' - content: | - #!/bin/bash -e - # Wrapper for bootkube start - [ -n "$(ls /var/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /var/bootkube/assets/manifests-*/* /var/bootkube/assets/manifests && rm -rf /var/bootkube/assets/manifests-* - /usr/bin/docker run --rm --name bootkube \ - --net=host \ - --volume /etc/kubernetes:/etc/kubernetes:Z \ - --volume /var/bootkube/assets:/assets:Z \ - --entrypoint=/bootkube \ - quay.io/coreos/bootkube:v0.12.0 start --asset-dir=/assets bootcmd: - [setenforce, Permissive] - [systemctl, disable, firewalld, --now] runcmd: - [systemctl, daemon-reload] - - "atomic install --system --name=etcd quay.io/dghubble/etcd:0265e6680d2533f3fbf4512af868d29ff07451ca" - - [systemctl, start, --no-block, etcd.service] - [hostnamectl, set-hostname, ${domain_name}] + - "atomic install --system --name=etcd quay.io/dghubble/etcd:0265e6680d2533f3fbf4512af868d29ff07451ca" - "atomic install --system --name=kubelet quay.io/dghubble/kubelet:8baca5cbaf7b7ee0710380c7d8897e444ebdcb27" + - "atomic install --system --name=bootkube quay.io/dghubble/bootkube:3cc2345503c60186db5272fa918514259e3c4a9d" + - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/bare-metal/fedora-atomic/kubernetes/ssh.tf b/bare-metal/fedora-atomic/kubernetes/ssh.tf index 82fd3a14..52c2231e 100644 --- a/bare-metal/fedora-atomic/kubernetes/ssh.tf +++ b/bare-metal/fedora-atomic/kubernetes/ssh.tf @@ -113,7 +113,7 @@ resource "null_resource" "bootkube-start" { provisioner "remote-exec" { inline = [ "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/bootkube", + "sudo mv $HOME/assets /var/lib/bootkube", "sudo systemctl start bootkube", ] } diff --git a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 5a2941a9..13006c30 100644 --- a/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -74,48 +74,23 @@ write_files: PathExists=/etc/kubernetes/kubeconfig [Install] WantedBy=multi-user.target - - path: /etc/systemd/system/bootkube.service - content: | - [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/var/bootkube/init_bootkube.done - [Service] - Type=oneshot - RemainAfterExit=true - WorkingDirectory=/var/bootkube - ExecStartPre=/bin/mkdir -p /var/bootkube - ExecStart=/usr/local/bin/bootkube-start - ExecStartPost=/bin/touch /var/bootkube/init_bootkube.done - [Install] - WantedBy=multi-user.target - - path: /var/bootkube/.keep + - path: /var/lib/bootkube/.keep - path: /etc/selinux/config owner: root:root permissions: '0644' content: | SELINUX=permissive SELINUXTYPE=targeted - - path: /usr/local/bin/bootkube-start - permissions: '0755' - content: | - #!/bin/bash -e - # Wrapper for bootkube start - [ -n "$(ls /var/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /var/bootkube/assets/manifests-*/* /var/bootkube/assets/manifests && rm -rf /var/bootkube/assets/manifests-* - /usr/bin/docker run --rm --name bootkube \ - --net=host \ - --volume /etc/kubernetes:/etc/kubernetes:Z \ - --volume /var/bootkube/assets:/assets:Z \ - --entrypoint=/bootkube \ - quay.io/coreos/bootkube:v0.12.0 start --asset-dir=/assets bootcmd: - [setenforce, Permissive] - [systemctl, disable, firewalld, --now] runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/dghubble/etcd:0265e6680d2533f3fbf4512af868d29ff07451ca" + - "atomic install --system --name=kubelet quay.io/dghubble/kubelet:8baca5cbaf7b7ee0710380c7d8897e444ebdcb27" + - "atomic install --system --name=bootkube quay.io/dghubble/bootkube:3cc2345503c60186db5272fa918514259e3c4a9d" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, cloud-metadata.service] - - "atomic install --system --name=kubelet quay.io/dghubble/kubelet:8baca5cbaf7b7ee0710380c7d8897e444ebdcb27" - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] users: diff --git a/digital-ocean/fedora-atomic/kubernetes/ssh.tf b/digital-ocean/fedora-atomic/kubernetes/ssh.tf index 582944dd..60bf6ddc 100644 --- a/digital-ocean/fedora-atomic/kubernetes/ssh.tf +++ b/digital-ocean/fedora-atomic/kubernetes/ssh.tf @@ -110,7 +110,7 @@ resource "null_resource" "bootkube-start" { provisioner "remote-exec" { inline = [ "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/bootkube", + "sudo mv $HOME/assets /var/lib/bootkube", "sudo systemctl start bootkube", ] } diff --git a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index 876f055f..8a888dd5 100644 --- a/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl +++ b/google-cloud/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -71,33 +71,7 @@ write_files: permissions: '0644' content: | ${kubeconfig} - - path: /etc/systemd/system/bootkube.service - content: | - [Unit] - Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/var/bootkube/init_bootkube.done - [Service] - Type=oneshot - RemainAfterExit=true - WorkingDirectory=/var/bootkube - ExecStartPre=/bin/mkdir -p /var/bootkube - ExecStart=/usr/local/bin/bootkube-start - ExecStartPost=/bin/touch /var/bootkube/init_bootkube.done - [Install] - WantedBy=multi-user.target - - path: /var/bootkube/.keep - - path: /usr/local/bin/bootkube-start - permissions: '0755' - content: | - #!/bin/bash -e - # Wrapper for bootkube start - [ -n "$(ls /var/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /var/bootkube/assets/manifests-*/* /var/bootkube/assets/manifests && rm -rf /var/bootkube/assets/manifests-* - /usr/bin/docker run --rm --name bootkube \ - --net=host \ - --volume /etc/kubernetes:/etc/kubernetes:Z \ - --volume /var/bootkube/assets:/assets:Z \ - --entrypoint=/bootkube \ - quay.io/coreos/bootkube:v0.12.0 start --asset-dir=/assets + - path: /var/lib/bootkube/.keep - path: /etc/selinux/config owner: root:root permissions: '0644' @@ -110,9 +84,10 @@ bootcmd: runcmd: - [systemctl, daemon-reload] - "atomic install --system --name=etcd quay.io/dghubble/etcd:0265e6680d2533f3fbf4512af868d29ff07451ca" + - "atomic install --system --name=kubelet quay.io/dghubble/kubelet:8baca5cbaf7b7ee0710380c7d8897e444ebdcb27" + - "atomic install --system --name=bootkube quay.io/dghubble/bootkube:3cc2345503c60186db5272fa918514259e3c4a9d" - [systemctl, start, --no-block, etcd.service] - [systemctl, enable, cloud-metadata.service] - - "atomic install --system --name=kubelet quay.io/dghubble/kubelet:8baca5cbaf7b7ee0710380c7d8897e444ebdcb27" - [systemctl, start, --no-block, kubelet.service] users: - default diff --git a/google-cloud/fedora-atomic/kubernetes/ssh.tf b/google-cloud/fedora-atomic/kubernetes/ssh.tf index 57c00a08..54517eaf 100644 --- a/google-cloud/fedora-atomic/kubernetes/ssh.tf +++ b/google-cloud/fedora-atomic/kubernetes/ssh.tf @@ -82,7 +82,7 @@ resource "null_resource" "bootkube-start" { provisioner "remote-exec" { inline = [ "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /var/bootkube", + "sudo mv $HOME/assets /var/lib/bootkube", "sudo systemctl start bootkube", ] }