From 8d2c8b8db692b27b89d18e47a40cf7b49562694e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 13 Oct 2022 09:47:45 -0700 Subject: [PATCH] Switch to Flatcar Azure gen2 images and change worker type * Switch from Azure Hypervisor generation 1 to generation 2 * Change default Azure `worker_type` from Standard_DS1_v2 to Standard_D2as_v5 * Get 2 VCPU, 7 GiB, 12500Mbps (vs 1 VCPU, 3.5GiB, 750 Mbps) * Small increase in pay-as-you-go price ($53.29 -> $62.78) * Small increase in spot price ($5.64/mo -> $7.37/mo) * Change from Intel to AMD EPYC (`D2as_v5` cheaper than `D2s_v5`) Notes: Azure makes you accept terms for each plan: ``` az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 ``` Rel: * https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series * https://learn.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series#dsv2-series --- CHANGES.md | 10 ++++++++++ azure/flatcar-linux/kubernetes/controllers.tf | 6 +++--- azure/flatcar-linux/kubernetes/variables.tf | 14 +++++++------- .../flatcar-linux/kubernetes/workers/variables.tf | 2 +- azure/flatcar-linux/kubernetes/workers/workers.tf | 6 +++--- docs/advanced/worker-pools.md | 2 +- docs/flatcar-linux/azure.md | 4 ++-- 7 files changed, 27 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3750b9b2..d2c68ad3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,16 @@ Notable changes between versions. * Kubernetes [v1.25.2](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.25.md#v1252) * Switch Kubernetes registry from `k8s.gcr.io` to `registry.k8s.io` for addons +### Flatcar Linux + +* Switch from Azure Hypervisor gen1 to gen2 (**action required**) ([#1248](https://github.com/poseidon/typhoon/pull/1248)) + * Run `az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2` +* Change default Azure `worker_type` from [`Standard_DS1_v2`](https://learn.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series#dsv2-series) to [`Standard_D2as_v5`](https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series#dasv5-series) ([#1248](https://github.com/poseidon/typhoon/pull/1248)) + * Get 2 VCPU, 7 GiB, 12500Mbps (vs 1 VCPU, 3.5GiB, 750 Mbps) + * Small increase in pay-as-you-go price ($53.29 -> $62.78) + * Small increase in spot price ($5.64/mo -> $7.37/mo) + * Change from Intel to AMD EPYC (`D2as_v5` cheaper than `D2s_v5`) + ### Addons * Update Prometheus from v2.38.0 to [v2.39.1](https://github.com/prometheus/prometheus/releases/tag/v2.39.1) diff --git a/azure/flatcar-linux/kubernetes/controllers.tf b/azure/flatcar-linux/kubernetes/controllers.tf index 7d15bf72..efe80075 100644 --- a/azure/flatcar-linux/kubernetes/controllers.tf +++ b/azure/flatcar-linux/kubernetes/controllers.tf @@ -53,16 +53,16 @@ resource "azurerm_linux_virtual_machine" "controllers" { # Flatcar Container Linux source_image_reference { - publisher = "Kinvolk" + publisher = "kinvolk" offer = "flatcar-container-linux-free" - sku = local.channel + sku = "${local.channel}-gen2" version = "latest" } plan { - name = local.channel publisher = "kinvolk" product = "flatcar-container-linux-free" + name = "${local.channel}-gen2" } # network diff --git a/azure/flatcar-linux/kubernetes/variables.tf b/azure/flatcar-linux/kubernetes/variables.tf index 58cda2f2..ce3baf12 100644 --- a/azure/flatcar-linux/kubernetes/variables.tf +++ b/azure/flatcar-linux/kubernetes/variables.tf @@ -43,7 +43,7 @@ variable "controller_type" { variable "worker_type" { type = string description = "Machine type for workers (see `az vm list-skus --location centralus`)" - default = "Standard_DS1_v2" + default = "Standard_D2as_v5" } variable "os_image" { @@ -133,6 +133,12 @@ variable "worker_node_labels" { default = [] } +variable "daemonset_tolerations" { + type = list(string) + description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])" + default = [] +} + # unofficial, undocumented, unsupported variable "cluster_domain_suffix" { @@ -140,9 +146,3 @@ variable "cluster_domain_suffix" { description = "Queries for domains with the suffix will be answered by coredns. Default is cluster.local (e.g. foo.default.svc.cluster.local) " default = "cluster.local" } - -variable "daemonset_tolerations" { - type = list(string) - description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])" - default = [] -} diff --git a/azure/flatcar-linux/kubernetes/workers/variables.tf b/azure/flatcar-linux/kubernetes/workers/variables.tf index 9d168394..c2537b4b 100644 --- a/azure/flatcar-linux/kubernetes/workers/variables.tf +++ b/azure/flatcar-linux/kubernetes/workers/variables.tf @@ -41,7 +41,7 @@ variable "worker_count" { variable "vm_type" { type = string description = "Machine type for instances (see `az vm list-skus --location centralus`)" - default = "Standard_DS1_v2" + default = "Standard_D2as_v5" } variable "os_image" { diff --git a/azure/flatcar-linux/kubernetes/workers/workers.tf b/azure/flatcar-linux/kubernetes/workers/workers.tf index 6d910c7d..9c9ef059 100644 --- a/azure/flatcar-linux/kubernetes/workers/workers.tf +++ b/azure/flatcar-linux/kubernetes/workers/workers.tf @@ -24,16 +24,16 @@ resource "azurerm_linux_virtual_machine_scale_set" "workers" { # Flatcar Container Linux source_image_reference { - publisher = "Kinvolk" + publisher = "kinvolk" offer = "flatcar-container-linux-free" - sku = local.channel + sku = "${local.channel}-gen2" version = "latest" } plan { - name = local.channel publisher = "kinvolk" product = "flatcar-container-linux-free" + name = "${local.channel}-gen2" } # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too diff --git a/docs/advanced/worker-pools.md b/docs/advanced/worker-pools.md index 559df24a..47afb6a5 100644 --- a/docs/advanced/worker-pools.md +++ b/docs/advanced/worker-pools.md @@ -189,7 +189,7 @@ The Azure internal `workers` module supports a number of [variables](https://git | Name | Description | Default | Example | |:-----|:------------|:--------|:--------| | worker_count | Number of instances | 1 | 3 | -| vm_type | Machine type for instances | "Standard_DS1_v2" | See below | +| vm_type | Machine type for instances | "Standard_D2as_v5" | See below | | os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha | | priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | "Regular" | "Spot" | | snippets | Container Linux Config snippets | [] | [examples](/advanced/customization/) | diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 5a00e707..611aed8a 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -65,8 +65,8 @@ Additional configuration options are described in the `azurerm` provider [docs]( Flatcar Linux publishes images to the Azure Marketplace and requires accepting terms. ``` -az vm image terms show --publish kinvolk --offer flatcar-container-linux-free --plan stable az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable +az vm image terms accept --publish kinvolk --offer flatcar-container-linux-free --plan stable-gen2 ``` ## Cluster @@ -227,7 +227,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | controller_count | Number of controllers (i.e. masters) | 1 | 1 | | worker_count | Number of workers | 1 | 3 | | controller_type | Machine type for controllers | "Standard_B2s" | See below | -| worker_type | Machine type for workers | "Standard_DS1_v2" | See below | +| worker_type | Machine type for workers | "Standard_D2as_v5" | See below | | os_image | Channel for a Container Linux derivative | "flatcar-stable" | flatcar-stable, flatcar-beta, flatcar-alpha | | disk_size | Size of the disk in GB | 30 | 100 | | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot |