Fix conflict between Calico and NetworkManager
* Observed frequent kube-scheduler and controller-manager restarts with Calico as the CNI provider. Root cause was unclear since control plane was functional and tests of pod to pod network connectivity passed * Root cause: Calico sets up cali* and tunl* network interfaces for containers on hosts. NetworkManager tries to manage these interfaces. It periodically disconnected veth pairs. Logs did not surface this issue since its not an error per-se, just Calico and NetworkManager dueling for control. Kubernetes correctly restarted pods failing health checks and ensured 2 replicas were running so the control plane functioned mostly normally. Pod to pod connecitivity was only affected occassionally. Pain to debug. * Solution: Configure NetworkManager to ignore the Calico ifaces per Calico's recommendation. Cloud-init writes files after NetworkManager starts, so a restart is required on first boot. On subsequent boots, the file is present so no restart is needed
This commit is contained in:
parent
0a7fab56e2
commit
567e18f015
|
@ -71,6 +71,12 @@ write_files:
|
||||||
content: |
|
content: |
|
||||||
${kubeconfig}
|
${kubeconfig}
|
||||||
- path: /var/lib/bootkube/.keep
|
- path: /var/lib/bootkube/.keep
|
||||||
|
- path: /etc/NetworkManager/conf.d/typhoon.conf
|
||||||
|
content: |
|
||||||
|
[main]
|
||||||
|
plugins=keyfile
|
||||||
|
[keyfile]
|
||||||
|
unmanaged-devices=interface-name:cali*;interface-name:tunl*
|
||||||
- path: /etc/selinux/config
|
- path: /etc/selinux/config
|
||||||
owner: root:root
|
owner: root:root
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
|
@ -84,6 +90,7 @@ bootcmd:
|
||||||
- [modprobe, ip_vs]
|
- [modprobe, ip_vs]
|
||||||
runcmd:
|
runcmd:
|
||||||
- [systemctl, daemon-reload]
|
- [systemctl, daemon-reload]
|
||||||
|
- [systemctl, restart, NetworkManager]
|
||||||
- "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.3"
|
- "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.3"
|
||||||
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
||||||
- "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.12.0"
|
- "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.12.0"
|
||||||
|
|
|
@ -47,6 +47,12 @@ write_files:
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
content: |
|
content: |
|
||||||
${kubeconfig}
|
${kubeconfig}
|
||||||
|
- path: /etc/NetworkManager/conf.d/typhoon.conf
|
||||||
|
content: |
|
||||||
|
[main]
|
||||||
|
plugins=keyfile
|
||||||
|
[keyfile]
|
||||||
|
unmanaged-devices=interface-name:cali*;interface-name:tunl*
|
||||||
- path: /etc/selinux/config
|
- path: /etc/selinux/config
|
||||||
owner: root:root
|
owner: root:root
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
|
@ -60,6 +66,7 @@ bootcmd:
|
||||||
- [modprobe, ip_vs]
|
- [modprobe, ip_vs]
|
||||||
runcmd:
|
runcmd:
|
||||||
- [systemctl, daemon-reload]
|
- [systemctl, daemon-reload]
|
||||||
|
- [systemctl, restart, NetworkManager]
|
||||||
- [systemctl, enable, cloud-metadata.service]
|
- [systemctl, enable, cloud-metadata.service]
|
||||||
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
||||||
- [systemctl, start, --no-block, kubelet.service]
|
- [systemctl, start, --no-block, kubelet.service]
|
||||||
|
|
|
@ -61,6 +61,12 @@ write_files:
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
- path: /var/lib/bootkube/.keep
|
- path: /var/lib/bootkube/.keep
|
||||||
|
- path: /etc/NetworkManager/conf.d/typhoon.conf
|
||||||
|
content: |
|
||||||
|
[main]
|
||||||
|
plugins=keyfile
|
||||||
|
[keyfile]
|
||||||
|
unmanaged-devices=interface-name:cali*;interface-name:tunl*
|
||||||
- path: /etc/selinux/config
|
- path: /etc/selinux/config
|
||||||
owner: root:root
|
owner: root:root
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
|
@ -74,6 +80,7 @@ bootcmd:
|
||||||
- [modprobe, ip_vs]
|
- [modprobe, ip_vs]
|
||||||
runcmd:
|
runcmd:
|
||||||
- [systemctl, daemon-reload]
|
- [systemctl, daemon-reload]
|
||||||
|
- [systemctl, restart, NetworkManager]
|
||||||
- [hostnamectl, set-hostname, ${domain_name}]
|
- [hostnamectl, set-hostname, ${domain_name}]
|
||||||
- "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.3"
|
- "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.3"
|
||||||
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
||||||
|
|
|
@ -37,6 +37,12 @@ write_files:
|
||||||
PathExists=/etc/kubernetes/kubeconfig
|
PathExists=/etc/kubernetes/kubeconfig
|
||||||
[Install]
|
[Install]
|
||||||
WantedBy=multi-user.target
|
WantedBy=multi-user.target
|
||||||
|
- path: /etc/NetworkManager/conf.d/typhoon.conf
|
||||||
|
content: |
|
||||||
|
[main]
|
||||||
|
plugins=keyfile
|
||||||
|
[keyfile]
|
||||||
|
unmanaged-devices=interface-name:cali*;interface-name:tunl*
|
||||||
- path: /etc/selinux/config
|
- path: /etc/selinux/config
|
||||||
owner: root:root
|
owner: root:root
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
|
@ -50,6 +56,7 @@ bootcmd:
|
||||||
- [modprobe, ip_vs]
|
- [modprobe, ip_vs]
|
||||||
runcmd:
|
runcmd:
|
||||||
- [systemctl, daemon-reload]
|
- [systemctl, daemon-reload]
|
||||||
|
- [systemctl, restart, NetworkManager]
|
||||||
- [hostnamectl, set-hostname, ${domain_name}]
|
- [hostnamectl, set-hostname, ${domain_name}]
|
||||||
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
||||||
- [systemctl, enable, kubelet.path]
|
- [systemctl, enable, kubelet.path]
|
||||||
|
|
|
@ -72,6 +72,12 @@ write_files:
|
||||||
content: |
|
content: |
|
||||||
${kubeconfig}
|
${kubeconfig}
|
||||||
- path: /var/lib/bootkube/.keep
|
- path: /var/lib/bootkube/.keep
|
||||||
|
- path: /etc/NetworkManager/conf.d/typhoon.conf
|
||||||
|
content: |
|
||||||
|
[main]
|
||||||
|
plugins=keyfile
|
||||||
|
[keyfile]
|
||||||
|
unmanaged-devices=interface-name:cali*;interface-name:tunl*
|
||||||
- path: /etc/selinux/config
|
- path: /etc/selinux/config
|
||||||
owner: root:root
|
owner: root:root
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
|
@ -85,6 +91,7 @@ bootcmd:
|
||||||
- [modprobe, ip_vs]
|
- [modprobe, ip_vs]
|
||||||
runcmd:
|
runcmd:
|
||||||
- [systemctl, daemon-reload]
|
- [systemctl, daemon-reload]
|
||||||
|
- [systemctl, restart, NetworkManager]
|
||||||
- "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.3"
|
- "atomic install --system --name=etcd quay.io/poseidon/etcd:v3.3.3"
|
||||||
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
||||||
- "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.12.0"
|
- "atomic install --system --name=bootkube quay.io/poseidon/bootkube:v0.12.0"
|
||||||
|
|
|
@ -48,6 +48,12 @@ write_files:
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
content: |
|
content: |
|
||||||
${kubeconfig}
|
${kubeconfig}
|
||||||
|
- path: /etc/NetworkManager/conf.d/typhoon.conf
|
||||||
|
content: |
|
||||||
|
[main]
|
||||||
|
plugins=keyfile
|
||||||
|
[keyfile]
|
||||||
|
unmanaged-devices=interface-name:cali*;interface-name:tunl*
|
||||||
- path: /etc/selinux/config
|
- path: /etc/selinux/config
|
||||||
owner: root:root
|
owner: root:root
|
||||||
permissions: '0644'
|
permissions: '0644'
|
||||||
|
@ -61,6 +67,7 @@ bootcmd:
|
||||||
- [modprobe, ip_vs]
|
- [modprobe, ip_vs]
|
||||||
runcmd:
|
runcmd:
|
||||||
- [systemctl, daemon-reload]
|
- [systemctl, daemon-reload]
|
||||||
|
- [systemctl, restart, NetworkManager]
|
||||||
- [systemctl, enable, cloud-metadata.service]
|
- [systemctl, enable, cloud-metadata.service]
|
||||||
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
- "atomic install --system --name=kubelet quay.io/poseidon/kubelet:v1.10.1"
|
||||||
- [systemctl, start, --no-block, kubelet.service]
|
- [systemctl, start, --no-block, kubelet.service]
|
||||||
|
|
Loading…
Reference in New Issue