From 358854e712ad0e94f5a057107815833ee7c60e16 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 9 May 2020 15:58:45 -0700 Subject: [PATCH] Fix Calico install-cni crash loop on Pod restarts * Set a consistent MCS level/range for Calico install-cni * Note: Rebooting a node was a workaround, because Kubelet relabels /etc/kubernetes(/cni/net.d) Background: * On SELinux enforcing systems, the Calico CNI install-cni container ran with default SELinux context and a random MCS pair. install-cni places CNI configs by first creating a temporary file and then moving them into place, which means the file MCS categories depend on the containers SELinux context. * calico-node Pod restarts creates a new install-cni container with a different MCS pair that cannot access the earlier written file (it places configs every time), causing the init container to error and calico-node to crash loop * https://github.com/projectcalico/cni-plugin/issues/874 ``` mv: inter-device move failed: '/calico.conf.tmp' to '/host/etc/cni/net.d/10-calico.conflist'; unable to remove target: Permission denied Failed to mv files. This may be caused by selinux configuration on the host, or something else. ``` Note, this isn't a host SELinux configuration issue. Related: * https://github.com/poseidon/terraform-render-bootstrap/pull/186 --- CHANGES.md | 8 +++++++- aws/container-linux/kubernetes/bootstrap.tf | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 2 +- azure/container-linux/kubernetes/bootstrap.tf | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 2 +- bare-metal/container-linux/kubernetes/bootstrap.tf | 2 +- bare-metal/fedora-coreos/kubernetes/bootstrap.tf | 2 +- digital-ocean/container-linux/kubernetes/bootstrap.tf | 2 +- digital-ocean/fedora-coreos/kubernetes/bootstrap.tf | 2 +- google-cloud/container-linux/kubernetes/bootstrap.tf | 2 +- google-cloud/fedora-coreos/kubernetes/bootstrap.tf | 2 +- 11 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b38e9a89..4e6dbcbe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,11 +9,17 @@ Notable changes between versions. * Renew Kubelet certificates every 72 hours * Update Calico from v3.13.1 to [v3.13.3](https://docs.projectcalico.org/v3.13/release-notes/) -#### AWS +### AWS * Fix Terraform plan error when `controller_count` exceeds AWS zones (e.g. 5 controllers) ([#714](https://github.com/poseidon/typhoon/pull/714)) * Regressed in v1.17.1 ([#605](https://github.com/poseidon/typhoon/pull/605)) +### Fedora CoreOS + +* Fix Calico `install-cni` crashloop on Pod restarts ([#724](https://github.com/poseidon/typhoon/pull/724)) + * SELinux enforcement requires consistent file context MCS level + * Restarting a node resolved the issue as a previous workaround + #### Google * Use new Fedora CoreOS official [image streams](https://docs.fedoraproject.org/en-US/fedora-coreos/update-streams/) ([#723](https://github.com/poseidon/typhoon/pull/722)) diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 9a371c89..92423b98 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 64ffb08d..27f7bc36 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index 1ab4a476..a07e2187 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 01c74b1f..5b842fe3 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index 6cae7504..06d9cfbc 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 587d4f1e..4c3ea223 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index af5e3e46..102f664b 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 3bd95bbe..21e726b2 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 49af2d10..b7486a53 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 0b3b41d7..69c9724b 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=1dc36b58b83b68b09d072ee7548e006c74e56bb1" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]