Roll instance template changes to worker managed instance groups

* When a worker managed instance group's (MIG) instance template
changes (including machine type, disk size, or Butane snippets
but excluding new AMIs), use Google Cloud's rolling update features
to ensure instances match declared state
* Ignore new AMIs since Fedora CoreOS and Flatcar Linux nodes
already auto-update and reboot themselves
* Rolling updates will create surge instances, wait for health
checks, then delete old instances (0 unavilable instances)
* Instances are replaced to ensure new Ignition/Butane snippets
are respected
* Add managed instance group autohealing (i.e. health checks) to
ensure new instances' Kubelet is running

Renames

* Name apiserver and kubelet health checks consistently
* Rename MIG from `${var.name}-worker-group` to `${var.name}-worker`

Rel: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups
This commit is contained in:
Dalton Hubble
2022-08-14 12:12:55 -07:00
parent 6facfca4ed
commit 20b76d6e00
7 changed files with 138 additions and 23 deletions

View File

@ -1,6 +1,6 @@
# Managed instance group of workers
resource "google_compute_region_instance_group_manager" "workers" {
name = "${var.name}-worker-group"
name = "${var.name}-worker"
description = "Compute instance group of ${var.name} workers"
# instance name prefix for instances in the group
@ -11,6 +11,16 @@ resource "google_compute_region_instance_group_manager" "workers" {
instance_template = google_compute_instance_template.worker.self_link
}
# Roll out MIG instance template changes by replacing instances.
# - Surge to create new instances, then delete old instances.
# - Replace ensures new Ignition is picked up
update_policy {
type = "PROACTIVE"
max_surge_fixed = 3
max_unavailable_fixed = 0
minimal_action = "REPLACE"
}
target_size = var.worker_count
target_pools = [google_compute_target_pool.workers.self_link]
@ -23,6 +33,27 @@ resource "google_compute_region_instance_group_manager" "workers" {
name = "https"
port = "443"
}
auto_healing_policies {
health_check = google_compute_health_check.worker.id
initial_delay_sec = 120
}
}
# Health check for worker Kubelet
resource "google_compute_health_check" "worker" {
name = "${var.name}-kubelet-health"
description = "Health check for worker Kubelet"
timeout_sec = 20
check_interval_sec = 30
healthy_threshold = 1
unhealthy_threshold = 6
ssl_health_check {
port = "10250"
}
}
# Worker instance template
@ -36,8 +67,11 @@ resource "google_compute_instance_template" "worker" {
}
scheduling {
automatic_restart = var.preemptible ? false : true
preemptible = var.preemptible
provisioning_model = var.preemptible ? "SPOT" : "STANDARD"
preemptible = var.preemptible
automatic_restart = var.preemptible ? false : true
# Spot instances with termination action DELETE cannot be used with MIGs
instance_termination_action = var.preemptible ? "STOP" : null
}
disk {
@ -49,10 +83,8 @@ resource "google_compute_instance_template" "worker" {
network_interface {
network = var.network
# Ephemeral external IP
access_config {
}
access_config {}
}
can_ip_forward = true
@ -64,6 +96,9 @@ resource "google_compute_instance_template" "worker" {
}
lifecycle {
ignore_changes = [
disk[0].source_image
]
# To update an Instance Template, Terraform should replace the existing resource
create_before_destroy = true
}