Switch from upstream hyperkube image to individual images

* Kubernetes plans to stop releasing the hyperkube container image
* Upstream will continue to publish `kube-apiserver`, `kube-controller-manager`,
`kube-scheduler`, and `kube-proxy` container images to `k8s.gcr.io`
* Upstream will publish Kubelet only as a binary for distros to package,
either as a DEB/RPM on traditional distros or a container image on
container-optimized operating systems
* Typhoon will package the upstream Kubelet (checksummed) and its
dependencies as a container image for use on CoreOS Container Linux,
Flatcar Linux, and Fedora CoreOS
* Update the Typhoon container image security policy to list
`quay.io/poseidon/kubelet`as an official distributed artifact

Hyperkube: https://github.com/kubernetes/kubernetes/pull/88676
Kubelet Container Image: https://github.com/poseidon/kubelet
Kubelet Quay Repo: https://quay.io/repository/poseidon/kubelet
This commit is contained in:
Dalton Hubble 2020-03-16 21:21:41 -07:00
parent ddc1ff5348
commit 590d941f50
27 changed files with 66 additions and 62 deletions

View File

@ -5,6 +5,15 @@ Notable changes between versions.
## Latest
* Update etcd from v3.4.4 to [v3.4.5](https://github.com/etcd-io/etcd/releases/tag/v3.4.5)
* Switch from upstream hyperkube image to individual images ([#669](https://github.com/poseidon/typhoon/pull/669))
* Use upstream `k8s.gcr.io` `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, and `kube-proxy` container images
* Use [poseidon/kubelet](https://github.com/poseidon/kubelet) to package the upstream Kubelet binary (checksummed) and
its dependencies as an automated build container image [quay.io/poseidon/kubelet](https://quay.io/repository/poseidon/kubelet)
* Update base images for control plane and Kubelet images used by Typhoon from upstream's debian 9 (stretch) to debian 10
(buster) base
* Update Typhoon container image security policy to list `quay.io/poseidon/kubelet`as an official distributed artifact
* Background: Kubernetes will [stop releasing](https://github.com/kubernetes/kubernetes/pull/88676) the hyperkube container
image and provide the Kubelet as a binary for distros to package
#### Addons

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

View File

@ -91,8 +91,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -136,7 +135,7 @@ systemd:
--volume script,kind=host,source=/opt/bootstrap/apply \
--mount volume=script,target=/apply \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
--exec=/apply

View File

@ -64,8 +64,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -80,9 +79,9 @@ systemd:
--lock-file=/var/run/lock/kubelet.lock \
--network-plugin=cni \
--node-labels=node.kubernetes.io/node \
%{ for label in split(",", node_labels) }
%{~ for label in split(",", node_labels) ~}
--node-labels=${label} \
%{ endfor ~}
%{~ endfor ~}
--pod-manifest-path=/etc/kubernetes/manifests \
--read-only-port=0 \
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
@ -128,11 +127,11 @@ storage:
--volume config,kind=host,source=/etc/kubernetes \
--mount volume=config,target=/etc/kubernetes \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:612b947 \
--net=host \
--dns=host \
-- \
kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
/usr/local/bin/kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
passwd:
users:
- name: core

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

View File

@ -79,7 +79,7 @@ systemd:
--volume /var/log:/var/log \
--volume /var/run/lock:/var/run/lock:z \
--volume /opt/cni/bin:/opt/cni/bin:z \
k8s.gcr.io/hyperkube:v1.17.4 kubelet \
quay.io/poseidon/kubelet:v1.17.4 \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -123,7 +123,7 @@ systemd:
--volume /opt/bootstrap/assets:/assets:ro,Z \
--volume /opt/bootstrap/apply:/apply:ro,Z \
--entrypoint=/apply \
k8s.gcr.io/hyperkube:v1.17.4
quay.io/poseidon/kubelet:v1.17.4
ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
ExecStartPost=-/usr/bin/podman stop bootstrap
storage:

View File

@ -49,7 +49,7 @@ systemd:
--volume /var/log:/var/log \
--volume /var/run/lock:/var/run/lock:z \
--volume /opt/cni/bin:/opt/cni/bin:z \
k8s.gcr.io/hyperkube:v1.17.4 kubelet \
quay.io/poseidon/kubelet:v1.17.4 \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -87,7 +87,7 @@ systemd:
Type=oneshot
RemainAfterExit=true
ExecStart=/bin/true
ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
[Install]
WantedBy=multi-user.target
storage:

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

View File

@ -90,8 +90,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -134,7 +133,7 @@ systemd:
--volume script,kind=host,source=/opt/bootstrap/apply \
--mount volume=script,target=/apply \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
--exec=/apply

View File

@ -63,8 +63,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -78,9 +77,9 @@ systemd:
--lock-file=/var/run/lock/kubelet.lock \
--network-plugin=cni \
--node-labels=node.kubernetes.io/node \
%{ for label in split(",", node_labels) }
%{~ for label in split(",", node_labels) ~}
--node-labels=${label} \
%{ endfor ~}
%{~ endfor ~}
--pod-manifest-path=/etc/kubernetes/manifests \
--read-only-port=0 \
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
@ -126,11 +125,11 @@ storage:
--volume config,kind=host,source=/etc/kubernetes \
--mount volume=config,target=/etc/kubernetes \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
-- \
kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]')
/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]')
passwd:
users:
- name: core

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [var.k8s_domain_name]

View File

@ -103,8 +103,7 @@ systemd:
--mount volume=etc-iscsi,target=/etc/iscsi \
--volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
--mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -149,7 +148,7 @@ systemd:
--volume script,kind=host,source=/opt/bootstrap/apply \
--mount volume=script,target=/apply \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
--exec=/apply

View File

@ -76,8 +76,7 @@ systemd:
--mount volume=etc-iscsi,target=/etc/iscsi \
--volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \
--mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [var.k8s_domain_name]

View File

@ -80,7 +80,7 @@ systemd:
--volume /opt/cni/bin:/opt/cni/bin:z \
--volume /etc/iscsi:/etc/iscsi \
--volume /sbin/iscsiadm:/sbin/iscsiadm \
k8s.gcr.io/hyperkube:v1.17.4 kubelet \
quay.io/poseidon/kubelet:v1.17.4 \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -134,7 +134,7 @@ systemd:
--volume /opt/bootstrap/assets:/assets:ro,Z \
--volume /opt/bootstrap/apply:/apply:ro,Z \
--entrypoint=/apply \
k8s.gcr.io/hyperkube:v1.17.4
quay.io/poseidon/kubelet:v1.17.4
ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
ExecStartPost=-/usr/bin/podman stop bootstrap
storage:

View File

@ -50,7 +50,7 @@ systemd:
--volume /opt/cni/bin:/opt/cni/bin:z \
--volume /etc/iscsi:/etc/iscsi \
--volume /sbin/iscsiadm:/sbin/iscsiadm \
k8s.gcr.io/hyperkube:v1.17.4 kubelet \
quay.io/poseidon/kubelet:v1.17.4 \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

View File

@ -101,8 +101,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -146,7 +145,7 @@ systemd:
--volume script,kind=host,source=/opt/bootstrap/apply \
--mount volume=script,target=/apply \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
--exec=/apply

View File

@ -74,8 +74,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -132,8 +131,8 @@ storage:
--volume config,kind=host,source=/etc/kubernetes \
--mount volume=config,target=/etc/kubernetes \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
-- \
kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

View File

@ -31,8 +31,8 @@ Together, they diversify Typhoon to support a range of container technologies.
| single-master | all platforms | all platforms |
| multi-master | all platforms | all platforms |
| control plane | static pods | static pods |
| kubelet image | upstream hyperkube | upstream hyperkube |
| control plane images | upstream hyperkube | upstream hyperkube |
| kubelet image | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary |
| control plane images | upstream images | upstream images |
| on-host etcd | rkt-fly | podman |
| on-host kubelet | rkt-fly | podman |
| CNI plugins | calico or flannel | calico or flannel |

View File

@ -40,9 +40,14 @@ Typhoon limits exposure to many security threats, but it is not a silver bullet.
* Do not give untrusted users a shell behind your firewall
* Define network policies for your namespaces
## OpenPGP Signing
## Container Images
Typhoon uses upstream container images and binaries. We do not distribute artifacts of our own. If you find artifacts claiming to be from Typhoon, please send a note.
Typhoon uses upstream container images (where possible) and upstream binaries.
!!! note
Kubernetes releases `kubelet` as a binary for distros to package, either as a DEB/RPM on traditional distros or as a container image for container-optimized operating systems.
Typhoon [packages](https://github.com/poseidon/kubelet) the upstream Kubelet and its dependencies as a [container image](https://quay.io/repository/poseidon/kubelet) for use in Typhoon. The upstream Kubelet binary is checksummed and packaged directly. Quay automated builds provide verifiability and confidence in image contents.
## Disclosures

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

View File

@ -90,8 +90,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -134,7 +133,7 @@ systemd:
--volume script,kind=host,source=/opt/bootstrap/apply \
--mount volume=script,target=/apply \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
--exec=/apply

View File

@ -63,8 +63,7 @@ systemd:
--mount volume=var-log,target=/var/log \
--volume opt-cni-bin,kind=host,source=/opt/cni/bin \
--mount volume=opt-cni-bin,target=/opt/cni/bin \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
--exec=/usr/local/bin/kubelet -- \
docker://quay.io/poseidon/kubelet:v1.17.4 -- \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -78,9 +77,9 @@ systemd:
--lock-file=/var/run/lock/kubelet.lock \
--network-plugin=cni \
--node-labels=node.kubernetes.io/node \
%{ for label in split(",", node_labels) }
%{~ for label in split(",", node_labels) ~}
--node-labels=${label} \
%{ endfor ~}
%{~ endfor ~}
--pod-manifest-path=/etc/kubernetes/manifests \
--read-only-port=0 \
--volume-plugin-dir=/var/lib/kubelet/volumeplugins
@ -126,11 +125,11 @@ storage:
--volume config,kind=host,source=/etc/kubernetes \
--mount volume=config,target=/etc/kubernetes \
--insecure-options=image \
docker://k8s.gcr.io/hyperkube:v1.17.4 \
docker://quay.io/poseidon/kubelet:v1.17.4 \
--net=host \
--dns=host \
-- \
kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
passwd:
users:
- name: core

View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=73784c1b2c791d9ba586a1478979ac34dd324dad"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=e76f0a09fa9e6421a9cf697ee03714c6224e2581"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

View File

@ -79,7 +79,7 @@ systemd:
--volume /var/log:/var/log \
--volume /var/run/lock:/var/run/lock:z \
--volume /opt/cni/bin:/opt/cni/bin:z \
k8s.gcr.io/hyperkube:v1.17.4 kubelet \
quay.io/poseidon/kubelet:v1.17.4 \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -123,7 +123,7 @@ systemd:
--volume /opt/bootstrap/assets:/assets:ro,Z \
--volume /opt/bootstrap/apply:/apply:ro,Z \
--entrypoint=/apply \
k8s.gcr.io/hyperkube:v1.17.4
quay.io/poseidon/kubelet:v1.17.4
ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
ExecStartPost=-/usr/bin/podman stop bootstrap
storage:

View File

@ -49,7 +49,7 @@ systemd:
--volume /var/log:/var/log \
--volume /var/run/lock:/var/run/lock:z \
--volume /opt/cni/bin:/opt/cni/bin:z \
k8s.gcr.io/hyperkube:v1.17.4 kubelet \
quay.io/poseidon/kubelet:v1.17.4 \
--anonymous-auth=false \
--authentication-token-webhook \
--authorization-mode=Webhook \
@ -87,7 +87,7 @@ systemd:
Type=oneshot
RemainAfterExit=true
ExecStart=/bin/true
ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z k8s.gcr.io/hyperkube:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
ExecStop=/bin/bash -c '/usr/bin/podman run --volume /etc/kubernetes:/etc/kubernetes:ro,z --entrypoint /usr/local/bin/kubectl quay.io/poseidon/kubelet:v1.17.4 kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $HOSTNAME'
[Install]
WantedBy=multi-user.target
storage: