diff --git a/CHANGES.md b/CHANGES.md index 33ce781e..ea91d006 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,12 @@ Notable changes between versions. * Kubernetes [v1.25.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.25.md#v1250) * Disable LocalStorageCapacityIsolationFSQuotaMonitoring feature gate ([#1220](https://github.com/poseidon/typhoon/pull/1220)) +* Migrate most Kubelet flags to KubeletConfiguration file ([#1219](https://github.com/poseidon/typhoon/pull/1219)) +* Configure Kubelet Graceful Node Shutdown ([#1222](https://github.com/poseidon/typhoon/pull/1222)) + * Allow up to 30s for critical pods to gracefully shutdown on node shutdown + * Allow up to 15s for regular pods to gracefully shutdown on node shutdown + * Mark node NotReady promptly on node shutdown + * Lengthen systemd inhibitor lock max delay from 5s to 45s ### Fedora CoreOS diff --git a/aws/fedora-coreos/kubernetes/butane/controller.yaml b/aws/fedora-coreos/kubernetes/butane/controller.yaml index 484ac6f6..60b57e12 100644 --- a/aws/fedora-coreos/kubernetes/butane/controller.yaml +++ b/aws/fedora-coreos/kubernetes/butane/controller.yaml @@ -154,6 +154,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -194,6 +196,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml b/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml index 990747bd..72503200 100644 --- a/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -122,10 +122,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/aws/flatcar-linux/kubernetes/butane/controller.yaml b/aws/flatcar-linux/kubernetes/butane/controller.yaml index cd14a80d..bf5eaeae 100644 --- a/aws/flatcar-linux/kubernetes/butane/controller.yaml +++ b/aws/flatcar-linux/kubernetes/butane/controller.yaml @@ -153,6 +153,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -193,6 +195,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml b/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml index 3da2888c..be3141c1 100644 --- a/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -121,10 +121,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/azure/fedora-coreos/kubernetes/butane/controller.yaml b/azure/fedora-coreos/kubernetes/butane/controller.yaml index cc7f255f..a39d667c 100644 --- a/azure/fedora-coreos/kubernetes/butane/controller.yaml +++ b/azure/fedora-coreos/kubernetes/butane/controller.yaml @@ -149,6 +149,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -189,6 +191,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml b/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml index 014d23a6..0f0bdc4a 100644 --- a/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -117,10 +117,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/azure/flatcar-linux/kubernetes/butane/controller.yaml b/azure/flatcar-linux/kubernetes/butane/controller.yaml index 9b4b8c82..0d4a704f 100644 --- a/azure/flatcar-linux/kubernetes/butane/controller.yaml +++ b/azure/flatcar-linux/kubernetes/butane/controller.yaml @@ -149,6 +149,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -189,6 +191,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml b/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml index 1716853f..d2f77d73 100644 --- a/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -117,10 +117,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml b/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml index 6eaca567..091f7a35 100644 --- a/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml @@ -159,6 +159,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -199,6 +201,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml b/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml index 4199c206..c05283b4 100644 --- a/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml @@ -113,10 +113,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml b/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml index 49c6945a..cdf8930d 100644 --- a/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml +++ b/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml @@ -160,6 +160,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -200,6 +202,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml b/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml index 493f1498..de2ef194 100644 --- a/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml +++ b/bare-metal/flatcar-linux/kubernetes/butane/worker.yaml @@ -118,10 +118,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml index e9f60d47..93a7744d 100644 --- a/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml @@ -156,6 +156,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -196,6 +198,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml index bcaeb444..4b822b85 100644 --- a/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml @@ -122,10 +122,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml b/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml index ade78500..799a312b 100644 --- a/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml +++ b/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml @@ -158,6 +158,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -198,6 +200,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml b/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml index f1ffaa5b..e86daadb 100644 --- a/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml +++ b/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml @@ -121,10 +121,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml b/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml index 3e063c7a..f5858785 100644 --- a/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml @@ -148,6 +148,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -188,6 +190,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml index 5f1b5659..19b29470 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -116,10 +116,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf contents: inline: | diff --git a/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml b/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml index 93bc1c29..cc701bdd 100644 --- a/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml +++ b/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml @@ -148,6 +148,8 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf @@ -188,6 +190,11 @@ storage: echo "Retry applying manifests" sleep 5 done + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: diff --git a/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml b/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml index a35ef07b..ac46de25 100644 --- a/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -116,10 +116,17 @@ storage: featureGates: LocalStorageCapacityIsolationFSQuotaMonitoring: false rotateCertificates: true + shutdownGracePeriod: 45s + shutdownGracePeriodCriticalPods: 30s staticPodPath: /etc/kubernetes/manifests readOnlyPort: 0 resolvConf: /run/systemd/resolve/resolv.conf volumePluginDir: /var/lib/kubelet/volumeplugins + - path: /etc/systemd/logind.conf.d/inhibitors.conf + contents: + inline: | + [Login] + InhibitDelayMaxSec=45s - path: /etc/sysctl.d/max-user-watches.conf mode: 0644 contents: