Roll instance template changes to worker managed instance groups

* When a worker managed instance group's (MIG) instance template
changes (including machine type, disk size, or Butane snippets
but excluding new AMIs), use Google Cloud's rolling update features
to ensure instances match declared state
* Ignore new AMIs since Fedora CoreOS and Flatcar Linux nodes
already auto-update and reboot themselves
* Rolling updates will create surge instances, wait for health
checks, then delete old instances (0 unavilable instances)
* Instances are replaced to ensure new Ignition/Butane snippets
are respected
* Add managed instance group autohealing (i.e. health checks) to
ensure new instances' Kubelet is running

Renames

* Name apiserver and kubelet health checks consistently
* Rename MIG from `${var.name}-worker-group` to `${var.name}-worker`

Rel: https://cloud.google.com/compute/docs/instance-groups/rolling-out-updates-to-managed-instance-groups

google-cloud/flatcar-linux/kubernetes/apiserver.tf

@@ -75,10 +75,10 @@ resource "google_compute_instance_group" "controllers" {
   )
 }
 
-# TCP health check for apiserver
+# Health check for kube-apiserver
 resource "google_compute_health_check" "apiserver" {
-  name        = "${var.cluster_name}-apiserver-tcp-health"
-  description = "TCP health check for kube-apiserver"
+  name        = "${var.cluster_name}-apiserver-health"
+  description = "Health check for kube-apiserver"
 
   timeout_sec        = 5
   check_interval_sec = 5
@@ -86,7 +86,7 @@ resource "google_compute_health_check" "apiserver" {
   healthy_threshold   = 1
   unhealthy_threshold = 3
 
-  tcp_health_check {
+  ssl_health_check {
     port = "6443"
   }
 }

google-cloud/flatcar-linux/kubernetes/network.tf

@@ -196,6 +196,24 @@ resource "google_compute_firewall" "allow-ingress" {
   target_tags   = ["${var.cluster_name}-worker"]
 }
 
+resource "google_compute_firewall" "google-kubelet-health-checks" {
+  name    = "${var.cluster_name}-kubelet-health"
+  network = google_compute_network.network.name
+
+  allow {
+    protocol = "tcp"
+    ports    = [10250]
+  }
+
+  # https://cloud.google.com/compute/docs/instance-groups/autohealing-instances-in-migs
+  source_ranges = [
+    "35.191.0.0/16",
+    "130.211.0.0/22",
+  ]
+
+  target_tags = ["${var.cluster_name}-worker"]
+}
+
 resource "google_compute_firewall" "google-ingress-health-checks" {
   name    = "${var.cluster_name}-ingress-health"
   network = google_compute_network.network.name

google-cloud/flatcar-linux/kubernetes/workers/workers.tf

@@ -1,6 +1,6 @@
 # Managed instance group of workers
 resource "google_compute_region_instance_group_manager" "workers" {
-  name        = "${var.name}-worker-group"
+  name        = "${var.name}-worker"
   description = "Compute instance group of ${var.name} workers"
 
   # instance name prefix for instances in the group
@@ -11,6 +11,16 @@ resource "google_compute_region_instance_group_manager" "workers" {
     instance_template = google_compute_instance_template.worker.self_link
   }
 
+  # Roll out MIG instance template changes by replacing instances.
+  # - Surge to create new instances, then delete old instances.
+  # - Replace ensures new Ignition is picked up
+  update_policy {
+    type                  = "PROACTIVE"
+    max_surge_fixed       = 3
+    max_unavailable_fixed = 0
+    minimal_action        = "REPLACE"
+  }
+
   target_size  = var.worker_count
   target_pools = [google_compute_target_pool.workers.self_link]
 
@@ -23,6 +33,27 @@ resource "google_compute_region_instance_group_manager" "workers" {
     name = "https"
     port = "443"
   }
+
+  auto_healing_policies {
+    health_check      = google_compute_health_check.worker.id
+    initial_delay_sec = 120
+  }
+}
+
+# Health check for worker Kubelet
+resource "google_compute_health_check" "worker" {
+  name        = "${var.name}-kubelet-health"
+  description = "Health check for worker Kubelet"
+
+  timeout_sec        = 20
+  check_interval_sec = 30
+
+  healthy_threshold   = 1
+  unhealthy_threshold = 6
+
+  ssl_health_check {
+    port = "10250"
+  }
 }
 
 # Worker instance template
@@ -36,8 +67,11 @@ resource "google_compute_instance_template" "worker" {
   }
 
   scheduling {
-    automatic_restart = var.preemptible ? false : true
-    preemptible       = var.preemptible
+    provisioning_model = var.preemptible ? "SPOT" : "STANDARD"
+    preemptible        = var.preemptible
+    automatic_restart  = var.preemptible ? false : true
+    # Spot instances with termination action DELETE cannot be used with MIGs
+    instance_termination_action = var.preemptible ? "STOP" : null
   }
 
   disk {
@@ -49,10 +83,8 @@ resource "google_compute_instance_template" "worker" {
 
   network_interface {
     network = var.network
-
     # Ephemeral external IP
-    access_config {
-    }
+    access_config {}
   }
 
   can_ip_forward = true
@@ -64,6 +96,9 @@ resource "google_compute_instance_template" "worker" {
   }
 
   lifecycle {
+    ignore_changes = [
+      disk[0].source_image
+    ]
     # To update an Instance Template, Terraform should replace the existing resource
     create_before_destroy = true
   }