Update ARM64 cluster and hybrid cluster docs
* Typhoon now supports arbitrary combinations of controller, worker, and worker pool architectures so we can drop the specific details of full-cluster vs hybrid cluster. Just pick the architecture for each group of nodes accordingly. * However, if a custom node taint is set, continue to configure the cluster's daemonsets accordingly with `daemonset_tolerations`
This commit is contained in:
Dalton Hubble
parent
67e5ecf6f2
commit
83f1bd2373
|
|
@ -14,12 +14,12 @@ Notable changes between versions.
|
|||
|
||||
### AWS
|
||||
|
||||
* Allow configuring controller and worker disks ([#1482](https://github.com/poseidon/typhoon/pull/1482))
|
||||
* Configure controller and worker disks ([#1482](https://github.com/poseidon/typhoon/pull/1482))
|
||||
* Add `controller_disk_type`, `controller_disk_size`, and `controller_disk_iops` variables
|
||||
* Add `worker_disk_type`, `worker_disk_size`, and `worker_disk_iops` variables
|
||||
* Remove `disk_type`, `disk_size`, and `disk_iops` variables
|
||||
* Fix propagating settings to worker disks, previously ignored
|
||||
* Allow configuring CPU pricing model for burstable instance types ([#1482](https://github.com/poseidon/typhoon/pull/1482))
|
||||
* Configure CPU pricing model for burstable instance types ([#1482](https://github.com/poseidon/typhoon/pull/1482))
|
||||
* Add `controller_cpu_credits` and `worker_cpu_credits` variables (`standard` or `unlimited`)
|
||||
* Configure controller or worker instance architecture ([#1485](https://github.com/poseidon/typhoon/pull/1485))
|
||||
* Add `controller_arch` and `worker_arch` variables (`amd64` or `arm64`)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
# ARM64
|
||||
|
||||
Typhoon supports ARM64 Kubernetes clusters with ARM64 controller and worker nodes (full-cluster) or adding worker pools of ARM64 nodes to clusters with an x86/amd64 control plane for a hybdrid (mixed-arch) cluster.
|
||||
|
||||
Typhoon ARM64 clusters (full-cluster or mixed-arch) are available on:
|
||||
Typhoon supports Kubernetes clusters with ARM64 controller or worker nodes on several platforms:
|
||||
|
||||
* AWS with Fedora CoreOS or Flatcar Linux
|
||||
* Azure with Flatcar Linux
|
||||
|
||||
## Cluster
|
||||
## AWS
|
||||
|
||||
Create a cluster on AWS with ARM64 controller and worker nodes. Container workloads must be `arm64` compatible and use `arm64` (or multi-arch) container images.
|
||||
|
||||
|
|
@ -22,17 +20,16 @@ Create a cluster on AWS with ARM64 controller and worker nodes. Container worklo
|
|||
dns_zone = "aws.example.com"
|
||||
dns_zone_id = "Z3PAABBCFAKEC0"
|
||||
|
||||
# instances
|
||||
controller_type = "t4g.small"
|
||||
controller_arch = "arm64"
|
||||
worker_count = 2
|
||||
worker_type = "t4g.small"
|
||||
worker_arch = "arm64"
|
||||
worker_price = "0.0168"
|
||||
|
||||
# configuration
|
||||
ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..."
|
||||
|
||||
# optional
|
||||
arch = "arm64"
|
||||
networking = "cilium"
|
||||
worker_count = 2
|
||||
worker_price = "0.0168"
|
||||
|
||||
controller_type = "t4g.small"
|
||||
worker_type = "t4g.small"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -47,17 +44,16 @@ Create a cluster on AWS with ARM64 controller and worker nodes. Container worklo
|
|||
dns_zone = "aws.example.com"
|
||||
dns_zone_id = "Z3PAABBCFAKEC0"
|
||||
|
||||
# instances
|
||||
controller_type = "t4g.small"
|
||||
controller_arch = "arm64"
|
||||
worker_count = 2
|
||||
worker_type = "t4g.small"
|
||||
worker_arch = "arm64"
|
||||
worker_price = "0.0168"
|
||||
|
||||
# configuration
|
||||
ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..."
|
||||
|
||||
# optional
|
||||
arch = "arm64"
|
||||
networking = "cilium"
|
||||
worker_count = 2
|
||||
worker_price = "0.0168"
|
||||
|
||||
controller_type = "t4g.small"
|
||||
worker_type = "t4g.small"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -71,115 +67,6 @@ ip-10-0-32-166 Ready <none> 80s v1.30.3 10.0.32.166 <none> F
|
|||
ip-10-0-5-79 Ready <none> 77s v1.30.3 10.0.5.79 <none> Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.aarch64 containerd://1.5.8
|
||||
```
|
||||
|
||||
## Hybrid
|
||||
|
||||
Create a hybrid/mixed arch cluster by defining an AWS cluster. Then define a [worker pool](worker-pools.md#aws) with ARM64 workers. Optional taints are added to aid in scheduling.
|
||||
|
||||
=== "FCOS Cluster"
|
||||
|
||||
```tf
|
||||
module "gravitas" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
cluster_name = "gravitas"
|
||||
dns_zone = "aws.example.com"
|
||||
dns_zone_id = "Z3PAABBCFAKEC0"
|
||||
|
||||
# configuration
|
||||
ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..."
|
||||
|
||||
# optional
|
||||
networking = "cilium"
|
||||
worker_count = 2
|
||||
worker_price = "0.021"
|
||||
|
||||
daemonset_tolerations = ["arch"] # important
|
||||
}
|
||||
```
|
||||
|
||||
=== "Flatcar Cluster"
|
||||
|
||||
```tf
|
||||
module "gravitas" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
cluster_name = "gravitas"
|
||||
dns_zone = "aws.example.com"
|
||||
dns_zone_id = "Z3PAABBCFAKEC0"
|
||||
|
||||
# configuration
|
||||
ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..."
|
||||
|
||||
# optional
|
||||
networking = "cilium"
|
||||
worker_count = 2
|
||||
worker_price = "0.021"
|
||||
|
||||
daemonset_tolerations = ["arch"] # important
|
||||
}
|
||||
```
|
||||
|
||||
=== "FCOS ARM64 Workers"
|
||||
|
||||
```tf
|
||||
module "gravitas-arm64" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes/workers?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
vpc_id = module.gravitas.vpc_id
|
||||
subnet_ids = module.gravitas.subnet_ids
|
||||
security_groups = module.gravitas.worker_security_groups
|
||||
|
||||
# configuration
|
||||
name = "gravitas-arm64"
|
||||
kubeconfig = module.gravitas.kubeconfig
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
|
||||
# optional
|
||||
arch = "arm64"
|
||||
instance_type = "t4g.small"
|
||||
spot_price = "0.0168"
|
||||
node_taints = ["arch=arm64:NoSchedule"]
|
||||
}
|
||||
```
|
||||
|
||||
=== "Flatcar ARM64 Workers"
|
||||
|
||||
```tf
|
||||
module "gravitas-arm64" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes/workers?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
vpc_id = module.gravitas.vpc_id
|
||||
subnet_ids = module.gravitas.subnet_ids
|
||||
security_groups = module.gravitas.worker_security_groups
|
||||
|
||||
# configuration
|
||||
name = "gravitas-arm64"
|
||||
kubeconfig = module.gravitas.kubeconfig
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
|
||||
# optional
|
||||
arch = "arm64"
|
||||
instance_type = "t4g.small"
|
||||
spot_price = "0.0168"
|
||||
node_taints = ["arch=arm64:NoSchedule"]
|
||||
}
|
||||
```
|
||||
|
||||
Verify amd64 (x86_64) and arm64 (aarch64) nodes are present.
|
||||
|
||||
```
|
||||
$ kubectl get nodes -o wide
|
||||
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
|
||||
ip-10-0-1-73 Ready <none> 111m v1.30.3 10.0.1.73 <none> Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8
|
||||
ip-10-0-22-79... Ready <none> 111m v1.30.3 10.0.22.79 <none> Flatcar Container Linux by Kinvolk 3033.2.0 (Oklo) 5.10.84-flatcar containerd://1.5.8
|
||||
ip-10-0-24-130 Ready <none> 111m v1.30.3 10.0.24.130 <none> Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8
|
||||
ip-10-0-39-19 Ready <none> 111m v1.30.3 10.0.39.19 <none> Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8
|
||||
```
|
||||
|
||||
## Azure
|
||||
|
||||
Create a cluster on Azure with ARM64 controller and worker nodes. Container workloads must be `arm64` compatible and use `arm64` (or multi-arch) container images.
|
||||
|
|
@ -194,13 +81,128 @@ module "ramius" {
|
|||
dns_zone = "azure.example.com"
|
||||
dns_zone_group = "example-group"
|
||||
|
||||
# instances
|
||||
controller_arch = "arm64"
|
||||
controller_type = "Standard_B2pls_v5"
|
||||
worker_count = 2
|
||||
controller_arch = "arm64"
|
||||
worker_type = "Standard_D2pls_v5"
|
||||
|
||||
# configuration
|
||||
ssh_authorized_key = "ssh-rsa AAAAB3Nz..."
|
||||
|
||||
# optional
|
||||
arch = "arm64"
|
||||
controller_type = "Standard_D2pls_v5"
|
||||
worker_type = "Standard_D2pls_v5"
|
||||
worker_count = 2
|
||||
}
|
||||
```
|
||||
|
||||
## Hybrid
|
||||
|
||||
Create a hybrid/mixed arch cluster by defining a cluster where [worker pool(s)](worker-pools.md#aws) have a different instance type architecture than controllers or other workers. Taints are added to aid in scheduling.
|
||||
|
||||
Here's an AWS example,
|
||||
|
||||
=== "FCOS Cluster"
|
||||
|
||||
```tf
|
||||
module "gravitas" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
cluster_name = "gravitas"
|
||||
dns_zone = "aws.example.com"
|
||||
dns_zone_id = "Z3PAABBCFAKEC0"
|
||||
|
||||
# instances
|
||||
worker_count = 2
|
||||
worker_arch = "arm64"
|
||||
worker_type = "t4g.medium"
|
||||
worker_price = "0.021"
|
||||
|
||||
# configuration
|
||||
daemonset_tolerations = ["arch"] # important
|
||||
networking = "cilium"
|
||||
ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..."
|
||||
}
|
||||
```
|
||||
|
||||
=== "Flatcar Cluster"
|
||||
|
||||
```tf
|
||||
module "gravitas" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
cluster_name = "gravitas"
|
||||
dns_zone = "aws.example.com"
|
||||
dns_zone_id = "Z3PAABBCFAKEC0"
|
||||
|
||||
# instances
|
||||
worker_count = 2
|
||||
worker_arch = "arm64"
|
||||
worker_type = "t4g.medium"
|
||||
worker_price = "0.021"
|
||||
|
||||
# configuration
|
||||
daemonset_tolerations = ["arch"] # important
|
||||
networking = "cilium"
|
||||
ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..."
|
||||
}
|
||||
```
|
||||
|
||||
=== "FCOS ARM64 Workers"
|
||||
|
||||
```tf
|
||||
module "gravitas-arm64" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes/workers?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
vpc_id = module.gravitas.vpc_id
|
||||
subnet_ids = module.gravitas.subnet_ids
|
||||
security_groups = module.gravitas.worker_security_groups
|
||||
|
||||
# instances
|
||||
arch = "arm64"
|
||||
instance_type = "t4g.small"
|
||||
spot_price = "0.0168"
|
||||
|
||||
# configuration
|
||||
name = "gravitas-arm64"
|
||||
kubeconfig = module.gravitas.kubeconfig
|
||||
node_taints = ["arch=arm64:NoSchedule"]
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
}
|
||||
```
|
||||
|
||||
=== "Flatcar ARM64 Workers"
|
||||
|
||||
```tf
|
||||
module "gravitas-arm64" {
|
||||
source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes/workers?ref=v1.30.3"
|
||||
|
||||
# AWS
|
||||
vpc_id = module.gravitas.vpc_id
|
||||
subnet_ids = module.gravitas.subnet_ids
|
||||
security_groups = module.gravitas.worker_security_groups
|
||||
|
||||
# instances
|
||||
arch = "arm64"
|
||||
instance_type = "t4g.small"
|
||||
spot_price = "0.0168"
|
||||
|
||||
# configuration
|
||||
name = "gravitas-arm64"
|
||||
kubeconfig = module.gravitas.kubeconfig
|
||||
node_taints = ["arch=arm64:NoSchedule"]
|
||||
ssh_authorized_key = var.ssh_authorized_key
|
||||
}
|
||||
```
|
||||
|
||||
Verify amd64 (x86_64) and arm64 (aarch64) nodes are present.
|
||||
|
||||
```
|
||||
$ kubectl get nodes -o wide
|
||||
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
|
||||
ip-10-0-1-73 Ready <none> 111m v1.30.3 10.0.1.73 <none> Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8
|
||||
ip-10-0-22-79... Ready <none> 111m v1.30.3 10.0.22.79 <none> Flatcar Container Linux by Kinvolk 3033.2.0 (Oklo) 5.10.84-flatcar containerd://1.5.8
|
||||
ip-10-0-24-130 Ready <none> 111m v1.30.3 10.0.24.130 <none> Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8
|
||||
ip-10-0-39-19 Ready <none> 111m v1.30.3 10.0.39.19 <none> Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8
|
||||
```
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue