first-try

2022-12-01 17:59:08 +01:00 · 2022-12-01 17:59:08 +01:00 · d1607f120f
parent fc2c9a2a5f
commit d1607f120f
9 changed files with 521 additions and 8 deletions
--- a/butane/k8s-leader.yaml
+++ b/butane/k8s-leader.yaml
@ -0,0 +1,209 @@
+variant: flatcar
+version: 1.0.0
+passwd:
+  users:
+    - name: cadoles
+      password_hash: "$y$j9T$7dNoyJBSiQ5dJ6.kLs17I0$G9VVQkBUddW2E6TVQchecSE94jNxLz4F3wguz5QkvD8"
+      groups:
+        - wheel
+        - sudo
+      ssh_authorized_keys: 
+        - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDDph3zh6ojSvH44k13z9B6xj+Hargo3uzkxnYv5s5NI4yagNuBXEc3aS++KdocND+FtVfLK+iVE8qHo2bvmpMmVkqU6WU2apN7DfIP0QGLlSGeo+UOZ9hGeEDlgVO4AOnZKZ5kPGBEPZ84JXuE9CmhKfwEVCK8w3B8XQttA8alFl4A4/4F14x2w4njsSLY1H3b0qah7hgYKU5zHIGLg8Lxx+1BxGIF0l5n5m5rqAskRNaF+aYbs0CcWHv49bPK0sJJ0qPV2r2sq8BlzuZFHExnZRIxpsIXdce4Bm4rdlGi7tBmmurLk4OOtDkwvhD0LMaNJf10k6QLSmRUTVzgsYz/dmGxopbMtwwIXkwi014uSZgi8wAuznXx5I4j2TUGPZHOVf+1iw/yaxWlgTVOSoX7ZxyhDgW5cCgZZGNzU5UWe0vUuVTB+hfSMj50/Q6+Vi92/mDMbPhm4nBoVzD5DT15mB+yGyN45Ej61m0JzVUyZexfvVaffEug1/u5dnwilP0WGKr4i2OXxOXtvSdAs5rlZjvppZk6IxRCwXIcPwEFL97ZrQZAxlVS5Nh+ZnlSwTe3zfQhzHj1ao0AdCAHFPUEdoUPJhSb0OjyCvZ9XZ1KCkXhuhuN/3IUhuoWl4soNCeC3KmU/USx1wda438Exj0hM1mTyBZScDPGyD9nw78DGw== Philippe Caseiro"
+systemd:
+  units:
+    - name: install-cni.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install CNI plugins K8S
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/cni/bin/firewall
+        [Service]
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="CNI_VERSION=v0.8.2"
+        Environment="ARCH=amd64"
+        Environment="URL=https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz"
+        ExecStartPre=/bin/mkdir -p /opt/cni/bin
+        ExecStart=/bin/sh -c "/usr/bin/curl -L ${URL} | /usr/bin/tar -C /opt/cni/bin -xz"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-crictl.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install crictl
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/bin/crictl
+        [Service]
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="DOWNLOAD_DIR=/opt/bin"
+        Environment="CRICTL_VERSION=v1.22.0"
+        Environment="ARCH=amd64"
+        Environment="URL=https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz"
+        ExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}
+        ExecStart=/bin/sh -c "/usr/bin/curl -L ${URL} | /usr/bin/tar -C $DOWNLOAD_DIR -xz"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-kubernetes.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install K8S
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/bin/kubelet
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="KUBE_RELEASE=v1.25.4"
+        Environment="ARCH=amd64"
+        Environment="DOWNLOAD_DIR=/opt/bin"
+        Environment="URL=https://storage.googleapis.com/kubernetes-release/release/${KUBE_RELEASE}/bin/linux/${ARCH}"
+        ExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}
+        ExecStart=/bin/sh -c "/usr/bin/curl -L --remote-name-all ${URL}/{kubeadm,kubelet,kubectl}"
+        ExecStartPost=/bin/sh -c "/bin/chmod +x ${DOWNLOAD_DIR}/{kubeadm,kubelet,kubectl}"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-cilium.service
+      enabled: false
+      contents: |
+        [Unit]
+        Description=Install Cilium client
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/bin/cilium
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="CILIUM_RELEASE=v0.12.8"
+        Environment="ARCH=amd64"
+        Environment="DOWNLOAD_DIR=/opt/bin"
+        Environment="URL=https://github.com/cilium/cilium-cli/releases/download/${CILIUM_RELEASE}/cilium-linux-${ARCH}.tar.gz
+        ExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}
+        ExecStart=/bin/sh -c "/usr/bin/curl -L ${URL} | /usr/bin/tar -C ${DOWNLOAD_DIR} -xz"
+        [Install]
+        WantedBy=multi-user.target
+    - name: kubelet.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=The Kubernetes Node Agent
+        Documentation=https://kubernetes.io/docs/home/
+        Wants=network-online.target
+        After=network-online.target
+        [Service]
+        ExecStart=/opt/bin/kubelet
+        Restart=always
+        StartLimitInterval=0
+        RestartSec=10
+        [Install]
+        WantedBy=multi-user.target
+      dropins:
+        - name: 10-kubeadm.conf
+          contents: |          
+            [Service]
+            Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+            Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
+            EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
+            EnvironmentFile=-/etc/default/kubelet
+            ExecStart=
+            ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+    - name: init-cluster.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubernetes cluster init
+        Documentation=https://kubernetes.io/docs/home/
+        Wants=network-online.target
+        After=network-online.target install-kubernetes.service
+        ConditionPathExists=!/opt/libexec/kubernetes/kubelet-plugins/volume/exec/
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=360
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="caSecretKey=$(date | sha256sum | awk \'{print $1}`')"
+        Environment="SET_HOSTNAME=leader.ks.cadoles.com"
+        Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/bin"
+        Environment="KUBECONFIG=/etc/kubernetes/admin.conf"
+        ExecStartPre=/bin/sh -c "/usr/bin/hostnamectl hostname ${SET_HOSTNAME}"
+        ExecStart=/opt/bin/kubeadm init --config /root/kubeadm-config.yaml --upload-certs
+        ExecStartPost=/bin/sh -c "/bin/mkdir -p /home/cadoles/.kube"
+        ExecStartPost=/bin/sh -c "/bin/cp -i /etc/kubernetes/admin.conf /home/cadoles/.kube/config"
+        ExecStartPost=/bin/sh -c "/bin/chown cadoles:cadoles /home/cadoles/.kube/config"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-network-cni.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install network CNI (calico)
+        Documentation=https://kubernetes.io/docs/home/
+        Wants=network-online.target
+        After=network-online.target init-cluster.service
+        ConditionPathExists=!/opt/cni/bin/calico
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=360
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="KUBECONFIG=/etc/kubernetes/admin.conf"
+        Environment="CIDR=10.244.0.0/16"
+        Environment="URL=https://raw.githubusercontent.com/projectcalico/calico/v3.24.5/manifests/calico.yaml"
+        ExecStart=/opt/bin/kubectl apply -f ${URL}
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  files:
+    - path: /etc/sysctl.d/k8s.conf
+      mode: 0600
+      contents:
+        inline: |
+          net.bridge.bridge-nf-call-ip6tables = 1
+          net.bridge.bridge-nf-call-iptables = 1
+    - path: /root/kubeadm-config.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubeadm.k8s.io/v1beta3
+          kind: InitConfiguration
+          certificateKey: b4edd5fb7aec1b90e06c68e090213621f621e31353a165dabeed0f8617c341b6
+          nodeRegistration:
+            name: leader.ks.cadoles.com
+            kubeletExtraArgs:
+              volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+          ---
+          apiVersion: kubeadm.k8s.io/v1beta3
+          kind: ClusterConfiguration
+          networking:
+            podSubnet: 10.244.0.0/16
+          controllerManager:
+            extraArgs:
+              flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+          ---
+          apiVersion: kubeproxy.config.k8s.io/v1alpha1
+          kind: KubeProxyConfiguration
+          mode: "ipvs"
+          ipvs:
+            strictARP: true
+          ---
+          kind: KubeletConfiguration
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          cgroupDriver: systemd
--- a/butane/k8s-worker.yaml
+++ b/butane/k8s-worker.yaml
@ -66,14 +66,13 @@ systemd:
        TimeoutStartSec=180
        RemainAfterExit=yes
        KillMode=process
-        Environment="RELEASE_VERSION=0.6.0"
-        Environment="RELEASE=v1.25.4"
+        Environment="KUBE_RELEASE=v1.25.4"
        Environment="ARCH=amd64"
        Environment="DOWNLOAD_DIR=/opt/bin"
-        Environment="URL=https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/${ARCH}"
+        Environment="URL=https://storage.googleapis.com/kubernetes-release/release/${KUBE_RELEASE}/bin/linux/${ARCH}"
        ExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}
        ExecStart=/bin/sh -c "/usr/bin/curl -L --remote-name-all ${URL}/{kubeadm,kubelet,kubectl}"
-        ExecStartPost=/bin/chmod +x ${DOWNLOAD_DIR}/{kubeadm,kubelet,kubectl}
+        ExecStartPost=/bin/sh -c "/bin/chmod +x ${DOWNLOAD_DIR}/{kubeadm,kubelet,kubectl}"
        [Install]
        WantedBy=multi-user.target
    - name: kubelet.service
@ -100,13 +99,13 @@ systemd:
            EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
            EnvironmentFile=-/etc/default/kubelet
            ExecStart=
-            ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+            ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
 storage:
  files:
    - path: /etc/hostname
-      mode: 0644
+      mode: 0640
      contents:
-        inline: node1
+        inline: worker.ks.cadoles.com
    - path: /etc/sysctl.d/k8s.conf
      mode: 0600
      contents:
--- a/butane/k8s.hcl
+++ b/butane/k8s.hcl
@ -0,0 +1,40 @@
+KubeCluster = {
+    Domain = "ks.cadoles.com"
+    PodCIDR = "10.244.0.0/16"
+    Users = [
+        {
+            Name = "cadoles"
+            PassHash = "$y$j9T$7dNoyJBSiQ5dJ6.kLs17I0$G9VVQkBUddW2E6TVQchecSE94jNxLz4F3wguz5QkvD8"
+            Groups = [
+                "wheel"
+                "sudo"
+            ]
+        }
+    ]
+    SSHAuthorizedKeys = [
+        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDDph3zh6ojSvH44k13z9B6xj+Hargo3uzkxnYv5s5NI4yagNuBXEc3aS++KdocND+FtVfLK+iVE8qHo2bvmpMmVkqU6WU2apN7DfIP0QGLlSGeo+UOZ9hGeEDlgVO4AOnZKZ5kPGBEPZ84JXuE9CmhKfwEVCK8w3B8XQttA8alFl4A4/4F14x2w4njsSLY1H3b0qah7hgYKU5zHIGLg8Lxx+1BxGIF0l5n5m5rqAskRNaF+aYbs0CcWHv49bPK0sJJ0qPV2r2sq8BlzuZFHExnZRIxpsIXdce4Bm4rdlGi7tBmmurLk4OOtDkwvhD0LMaNJf10k6QLSmRUTVzgsYz/dmGxopbMtwwIXkwi014uSZgi8wAuznXx5I4j2TUGPZHOVf+1iw/yaxWlgTVOSoX7ZxyhDgW5cCgZZGNzU5UWe0vUuVTB+hfSMj50/Q6+Vi92/mDMbPhm4nBoVzD5DT15mB+yGyN45Ej61m0JzVUyZexfvVaffEug1/u5dnwilP0WGKr4i2OXxOXtvSdAs5rlZjvppZk6IxRCwXIcPwEFL97ZrQZAxlVS5Nh+ZnlSwTe3zfQhzHj1ao0AdCAHFPUEdoUPJhSb0OjyCvZ9XZ1KCkXhuhuN/3IUhuoWl4soNCeC3KmU/USx1wda438Exj0hM1mTyBZScDPGyD9nw78DGw== Philippe Caseiro"
+    ]
+    Nodes = [
+        { 
+            Name = "master0"
+            Role = "leader"
+            IP = "192.168.10.240/24"
+            Gateway = "192.168.10.254"
+            DNS = "192.168.5.53"
+        }
+        {
+            Name = "worker1"
+            Role = "worker"
+            IP = "192.168.10.241/24"
+            Gateway = "192.168.10.254"
+            DNS = "192.168.5.53"
+        }
+        {
+            Name = "worker2"
+            Role = "worker"
+            IP = "192.168.10.242/24"
+            Gateway = "192.168.10.254"
+            DNS = "192.168.5.53"
+        }
+    ]
+}
--- a/butane/k8s.json
+++ b/butane/k8s.json
--- a/butane/k8s.yaml.hcl
+++ b/butane/k8s.yaml.hcl
@ -0,0 +1,203 @@
+variant: flatcar
+version: 1.0.0
+passwd:
+  users:
+    - name: cadoles
+      password_hash: "$y$j9T$7dNoyJBSiQ5dJ6.kLs17I0$G9VVQkBUddW2E6TVQchecSE94jNxLz4F3wguz5QkvD8"
+      groups:
+        - wheel
+        - sudo
+      ssh_authorized_keys: 
+        - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDDph3zh6ojSvH44k13z9B6xj+Hargo3uzkxnYv5s5NI4yagNuBXEc3aS++KdocND+FtVfLK+iVE8qHo2bvmpMmVkqU6WU2apN7DfIP0QGLlSGeo+UOZ9hGeEDlgVO4AOnZKZ5kPGBEPZ84JXuE9CmhKfwEVCK8w3B8XQttA8alFl4A4/4F14x2w4njsSLY1H3b0qah7hgYKU5zHIGLg8Lxx+1BxGIF0l5n5m5rqAskRNaF+aYbs0CcWHv49bPK0sJJ0qPV2r2sq8BlzuZFHExnZRIxpsIXdce4Bm4rdlGi7tBmmurLk4OOtDkwvhD0LMaNJf10k6QLSmRUTVzgsYz/dmGxopbMtwwIXkwi014uSZgi8wAuznXx5I4j2TUGPZHOVf+1iw/yaxWlgTVOSoX7ZxyhDgW5cCgZZGNzU5UWe0vUuVTB+hfSMj50/Q6+Vi92/mDMbPhm4nBoVzD5DT15mB+yGyN45Ej61m0JzVUyZexfvVaffEug1/u5dnwilP0WGKr4i2OXxOXtvSdAs5rlZjvppZk6IxRCwXIcPwEFL97ZrQZAxlVS5Nh+ZnlSwTe3zfQhzHj1ao0AdCAHFPUEdoUPJhSb0OjyCvZ9XZ1KCkXhuhuN/3IUhuoWl4soNCeC3KmU/USx1wda438Exj0hM1mTyBZScDPGyD9nw78DGw== Philippe Caseiro"
+systemd:
+  units:
+    - name: install-cni.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install CNI plugins K8S
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/cni/bin/firewall
+        [Service]
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="CNI_VERSION=v0.8.2"
+        Environment="ARCH=amd64"
+        Environment="URL=https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz"
+        ExecStartPre=/bin/mkdir -p /opt/cni/bin
+        ExecStart=/bin/sh -c "/usr/bin/curl -L ${URL} | /usr/bin/tar -C /opt/cni/bin -xz"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-crictl.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install crictl
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/bin/crictl
+        [Service]
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="DOWNLOAD_DIR=/opt/bin"
+        Environment="CRICTL_VERSION=v1.22.0"
+        Environment="ARCH=amd64"
+        Environment="URL=https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz"
+        ExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}
+        ExecStart=/bin/sh -c "/usr/bin/curl -L ${URL} | /usr/bin/tar -C $DOWNLOAD_DIR -xz"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-kubernetes.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install K8S
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/bin/kubelet
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="KUBE_RELEASE=v1.25.4"
+        Environment="ARCH=amd64"
+        Environment="DOWNLOAD_DIR=/opt/bin"
+        Environment="URL=https://storage.googleapis.com/kubernetes-release/release/${KUBE_RELEASE}/bin/linux/${ARCH}"
+        ExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}
+        ExecStart=/bin/sh -c "/usr/bin/curl -L --remote-name-all ${URL}/{kubeadm,kubelet,kubectl}"
+        ExecStartPost=/bin/sh -c "/bin/chmod +x ${DOWNLOAD_DIR}/{kubeadm,kubelet,kubectl}"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-cilium.service
+      enabled: false
+      contents: |
+        [Unit]
+        Description=Install Cilium client
+        Wants = network-online.target
+        After = network.target network-online.target
+        ConditionPathExists=!/opt/bin/cilium
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=180
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="CILIUM_RELEASE=v0.12.8"
+        Environment="ARCH=amd64"
+        Environment="DOWNLOAD_DIR=/opt/bin"
+        Environment="URL=https://github.com/cilium/cilium-cli/releases/download/${CILIUM_RELEASE}/cilium-linux-${ARCH}.tar.gz
+        ExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}
+        ExecStart=/bin/sh -c "/usr/bin/curl -L ${URL} | /usr/bin/tar -C ${DOWNLOAD_DIR} -xz"
+        [Install]
+        WantedBy=multi-user.target
+    - name: kubelet.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=The Kubernetes Node Agent
+        Documentation=https://kubernetes.io/docs/home/
+        Wants=network-online.target
+        After=network-online.target
+        [Service]
+        ExecStart=/opt/bin/kubelet
+        Restart=always
+        StartLimitInterval=0
+        RestartSec=10
+        [Install]
+        WantedBy=multi-user.target
+      dropins:
+        - name: 10-kubeadm.conf
+          contents: |          
+            [Service]
+            Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+            Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
+            EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
+            EnvironmentFile=-/etc/default/kubelet
+            ExecStart=
+            ExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+    - name: init-cluster.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Kubernetes cluster init
+        Documentation=https://kubernetes.io/docs/home/
+        Wants=network-online.target
+        After=network-online.target install-kubernetes.service
+        ConditionPathExists=!/opt/libexec/kubernetes/kubelet-plugins/volume/exec/
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=360
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="caSecretKey=$(date | sha256sum | awk \'{print $1}`')"
+        Environment="SET_HOSTNAME=leader.ks.cadoles.com"
+        Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/bin"
+        Environment="KUBECONFIG=/etc/kubernetes/admin.conf"
+        ExecStartPre=/bin/sh -c "/usr/bin/hostnamectl hostname ${SET_HOSTNAME}"
+        ExecStart=/opt/bin/kubeadm init --config /root/kubeadm-config.yaml --upload-certs
+        ExecStartPost=/bin/sh -c "/bin/mkdir -p /home/cadoles/.kube"
+        ExecStartPost=/bin/sh -c "/bin/cp -i /etc/kubernetes/admin.conf /home/cadoles/.kube/config"
+        ExecStartPost=/bin/sh -c "/bin/chown cadoles:cadoles /home/cadoles/.kube/config"
+        [Install]
+        WantedBy=multi-user.target
+    - name: install-network-cni.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=Install network CNI (calico)
+        Documentation=https://kubernetes.io/docs/home/
+        Wants=network-online.target
+        After=network-online.target init-cluster.service
+        ConditionPathExists=!/opt/cni/bin/calico
+        [Service]
+        WorkingDirectory=/opt/bin
+        Type=forking
+        TimeoutStartSec=360
+        RemainAfterExit=yes
+        KillMode=process
+        Environment="KUBECONFIG=/etc/kubernetes/admin.conf"
+        Environment="CIDR=10.244.0.0/16"
+        Environment="URL=https://raw.githubusercontent.com/projectcalico/calico/v3.24.5/manifests/calico.yaml"
+        ExecStart=/opt/bin/kubectl apply -f ${URL}
+        [Install]
+        WantedBy=multi-user.target
+
+storage:
+  files:
+    - path: /etc/sysctl.d/k8s.conf
+      mode: 0600
+      contents:
+        inline: |
+          net.bridge.bridge-nf-call-ip6tables = 1
+          net.bridge.bridge-nf-call-iptables = 1
+    - path: /root/kubeadm-config.yaml
+      mode: 0644
+      contents:
+        inline: |
+          apiVersion: kubeadm.k8s.io/v1beta3
+          kind: InitConfiguration
+          certificateKey: b4edd5fb7aec1b90e06c68e090213621f621e31353a165dabeed0f8617c341b6
+          nodeRegistration:
+            name: leader.ks.cadoles.com
+            kubeletExtraArgs:
+              volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+          ---
+          apiVersion: kubeadm.k8s.io/v1beta3
+          kind: ClusterConfiguration
+          networking:
+            podSubnet: 10.244.0.0/16
+          controllerManager:
+            extraArgs:
+              flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
+          ---
+          kind: KubeletConfiguration
+          apiVersion: kubelet.config.k8s.io/v1beta1
+          cgroupDriver: systemd
--- a/butane/leader.json
+++ b/butane/leader.json
--- a/butane/worker.json
+++ b/butane/worker.json
@ -0,0 +1 @@
+{"ignition":{"version":"3.3.0"},"passwd":{"users":[{"groups":["wheel","sudo"],"name":"cadoles","passwordHash":"$y$j9T$7dNoyJBSiQ5dJ6.kLs17I0$G9VVQkBUddW2E6TVQchecSE94jNxLz4F3wguz5QkvD8","sshAuthorizedKeys":["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDDph3zh6ojSvH44k13z9B6xj+Hargo3uzkxnYv5s5NI4yagNuBXEc3aS++KdocND+FtVfLK+iVE8qHo2bvmpMmVkqU6WU2apN7DfIP0QGLlSGeo+UOZ9hGeEDlgVO4AOnZKZ5kPGBEPZ84JXuE9CmhKfwEVCK8w3B8XQttA8alFl4A4/4F14x2w4njsSLY1H3b0qah7hgYKU5zHIGLg8Lxx+1BxGIF0l5n5m5rqAskRNaF+aYbs0CcWHv49bPK0sJJ0qPV2r2sq8BlzuZFHExnZRIxpsIXdce4Bm4rdlGi7tBmmurLk4OOtDkwvhD0LMaNJf10k6QLSmRUTVzgsYz/dmGxopbMtwwIXkwi014uSZgi8wAuznXx5I4j2TUGPZHOVf+1iw/yaxWlgTVOSoX7ZxyhDgW5cCgZZGNzU5UWe0vUuVTB+hfSMj50/Q6+Vi92/mDMbPhm4nBoVzD5DT15mB+yGyN45Ej61m0JzVUyZexfvVaffEug1/u5dnwilP0WGKr4i2OXxOXtvSdAs5rlZjvppZk6IxRCwXIcPwEFL97ZrQZAxlVS5Nh+ZnlSwTe3zfQhzHj1ao0AdCAHFPUEdoUPJhSb0OjyCvZ9XZ1KCkXhuhuN/3IUhuoWl4soNCeC3KmU/USx1wda438Exj0hM1mTyBZScDPGyD9nw78DGw== Philippe Caseiro"]}]},"storage":{"files":[{"path":"/etc/hostname","contents":{"compression":"","source":"data:,worker.ks.cadoles.com"},"mode":416},{"path":"/etc/sysctl.d/k8s.conf","contents":{"compression":"","source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201"},"mode":384}]},"systemd":{"units":[{"contents":"[Unit]\nDescription=Install CNI plugins K8S\nWants = network-online.target\nAfter = network.target network-online.target\nConditionPathExists=!/opt/cni/bin/firewall\n[Service]\nType=forking\nTimeoutStartSec=180\nRemainAfterExit=yes\nKillMode=process\nEnvironment=\"CNI_VERSION=v0.8.2\"\nEnvironment=\"ARCH=amd64\"\nEnvironment=\"URL=https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz\"\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStart=/bin/sh -c \"/usr/bin/curl -L ${URL} | /usr/bin/tar -C /opt/cni/bin -xz\"\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"install-cni.service"},{"contents":"[Unit]\nDescription=Install crictl\nWants = network-online.target\nAfter = network.target network-online.target\nConditionPathExists=!/opt/bin/crictl\n[Service]\nType=forking\nTimeoutStartSec=180\nRemainAfterExit=yes\nKillMode=process\nEnvironment=\"DOWNLOAD_DIR=/opt/bin\"\nEnvironment=\"CRICTL_VERSION=v1.22.0\"\nEnvironment=\"ARCH=amd64\"\nEnvironment=\"URL=https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz\"\nExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}\nExecStart=/bin/sh -c \"/usr/bin/curl -L ${URL} | /usr/bin/tar -C $DOWNLOAD_DIR -xz\"\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"install-crictl.service"},{"contents":"[Unit]\nDescription=Install K8S\nWants = network-online.target\nAfter = network.target network-online.target\nConditionPathExists=!/opt/bin/kubelet\n[Service]\nWorkingDirectory=/opt/bin\nType=forking\nTimeoutStartSec=180\nRemainAfterExit=yes\nKillMode=process\nEnvironment=\"KUBE_RELEASE=v1.25.4\"\nEnvironment=\"ARCH=amd64\"\nEnvironment=\"DOWNLOAD_DIR=/opt/bin\"\nEnvironment=\"URL=https://storage.googleapis.com/kubernetes-release/release/${KUBE_RELEASE}/bin/linux/${ARCH}\"\nExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}\nExecStart=/bin/sh -c \"/usr/bin/curl -L --remote-name-all ${URL}/{kubeadm,kubelet,kubectl}\"\nExecStartPost=/bin/sh -c \"/bin/chmod +x ${DOWNLOAD_DIR}/{kubeadm,kubelet,kubectl}\"\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"install-kubernetes.service"},{"contents":"[Unit]\nDescription=The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\nWants=network-online.target\nAfter=network-online.target\n[Service]\nExecStart=/opt/bin/kubelet\nRestart=always\nStartLimitInterval=0\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\nEnvironment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\nEnvironmentFile=-/etc/default/kubelet\nExecStart=\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n","name":"10-kubeadm.conf"}],"enabled":true,"name":"kubelet.service"}]}}
--- a/ipxe/etc/dnsmasq.d/pxe.conf
+++ b/ipxe/etc/dnsmasq.d/pxe.conf
@ -0,0 +1,21 @@
+log-queries
+log-dhcp
+
+port=0
+listen-address=0.0.0.0
+interface=eth0
+
+enable-tftp
+tftp-root=/srv/tftpboot
+
+dhcp-no-override
+
+pxe-prompt="Cadoles PXE Boot in",5
+pxe-service=X86PC,"Booting from network the Cadoles way",menu/boot.ipxe
+
+dhcp-range=192.168.40.20,proxy
+dhcp-match=set:ipxe,175
+dhcp-vendorclass=BIOS,PXEClient:Arch:00000
+dhcp-boot=tag:!ipxe,tag:BIOS,undionly.kpxe,192.168.40.20
+dhcp-boot=tag:!ipxe,tag:!BIOS,ipxe.efi,192.168.40.20
+dhcp-boot=tag:ipxe,menu/boot.ipxe
--- a/ipxe/tftpboot/menu.ipxe
+++ b/ipxe/tftpboot/menu.ipxe
@ -0,0 +1,38 @@
+#!ipxe
+
+:start
+menu Cadoles PXE Boot Options
+item shell iPXE shell
+item klead CadolesKube Leader
+item kmaster CadolesKube Master
+item knode CadolesKube Node
+item exit  Exit to BIOS
+
+choose --default exit --timeout 10000 option && goto ${option}
+
+:shell
+shell
+
+:klead
+set base-url http://stable.release.flatcar-linux.net/amd64-usr/current
+set ignition-url http://192.168.10.177:8001/leader.json
+
+kernel ${base-url}/flatcar_production_pxe.vmlinuz initrd=flatcar_production_pxe_image.cpio.gz flatcar.first_boot=1 ignition.config.url=${ignition-url}
+initrd ${base-url}/flatcar_production_pxe_image.cpio.gz
+boot
+
+:kmaster
+set base-url http://stable.release.flatcar-linux.net/amd64-usr/current
+set ignition-url http://192.168.10.177:8001/master.json
+
+kernel ${base-url}/flatcar_production_pxe.vmlinuz initrd=flatcar_production_pxe_image.cpio.gz flatcar.first_boot=1 ignition.config.url=${ignition-url}
+initrd ${base-url}/flatcar_production_pxe_image.cpio.gz
+
+:knode
+set base-url http://stable.release.flatcar-linux.net/amd64-usr/current
+set ignition-url http://192.168.10.177:8001/worker.json
+
+kernel ${base-url}/flatcar_production_pxe.vmlinuz initrd=flatcar_production_pxe_image.cpio.gz flatcar.first_boot=1 ignition.config.url=${ignition-url}
+initrd ${base-url}/flatcar_production_pxe_image.cpio.gz
+:exit
+exit
				`@ -0,0 +1 @@`
				{"ignition":{"version":"3.3.0"},"passwd":{"users":[{"groups":["wheel","sudo"],"name":"cadoles","passwordHash":"$y$j9T$7dNoyJBSiQ5dJ6.kLs17I0$G9VVQkBUddW2E6TVQchecSE94jNxLz4F3wguz5QkvD8","sshAuthorizedKeys":["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDDph3zh6ojSvH44k13z9B6xj+Hargo3uzkxnYv5s5NI4yagNuBXEc3aS++KdocND+FtVfLK+iVE8qHo2bvmpMmVkqU6WU2apN7DfIP0QGLlSGeo+UOZ9hGeEDlgVO4AOnZKZ5kPGBEPZ84JXuE9CmhKfwEVCK8w3B8XQttA8alFl4A4/4F14x2w4njsSLY1H3b0qah7hgYKU5zHIGLg8Lxx+1BxGIF0l5n5m5rqAskRNaF+aYbs0CcWHv49bPK0sJJ0qPV2r2sq8BlzuZFHExnZRIxpsIXdce4Bm4rdlGi7tBmmurLk4OOtDkwvhD0LMaNJf10k6QLSmRUTVzgsYz/dmGxopbMtwwIXkwi014uSZgi8wAuznXx5I4j2TUGPZHOVf+1iw/yaxWlgTVOSoX7ZxyhDgW5cCgZZGNzU5UWe0vUuVTB+hfSMj50/Q6+Vi92/mDMbPhm4nBoVzD5DT15mB+yGyN45Ej61m0JzVUyZexfvVaffEug1/u5dnwilP0WGKr4i2OXxOXtvSdAs5rlZjvppZk6IxRCwXIcPwEFL97ZrQZAxlVS5Nh+ZnlSwTe3zfQhzHj1ao0AdCAHFPUEdoUPJhSb0OjyCvZ9XZ1KCkXhuhuN/3IUhuoWl4soNCeC3KmU/USx1wda438Exj0hM1mTyBZScDPGyD9nw78DGw== Philippe Caseiro"]}]},"storage":{"files":[{"path":"/etc/hostname","contents":{"compression":"","source":"data:,worker.ks.cadoles.com"},"mode":416},{"path":"/etc/sysctl.d/k8s.conf","contents":{"compression":"","source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201"},"mode":384}]},"systemd":{"units":[{"contents":"[Unit]\nDescription=Install CNI plugins K8S\nWants = network-online.target\nAfter = network.target network-online.target\nConditionPathExists=!/opt/cni/bin/firewall\n[Service]\nType=forking\nTimeoutStartSec=180\nRemainAfterExit=yes\nKillMode=process\nEnvironment=\"CNI_VERSION=v0.8.2\"\nEnvironment=\"ARCH=amd64\"\nEnvironment=\"URL=https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz\"\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStart=/bin/sh -c \"/usr/bin/curl -L ${URL} \| /usr/bin/tar -C /opt/cni/bin -xz\"\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"install-cni.service"},{"contents":"[Unit]\nDescription=Install crictl\nWants = network-online.target\nAfter = network.target network-online.target\nConditionPathExists=!/opt/bin/crictl\n[Service]\nType=forking\nTimeoutStartSec=180\nRemainAfterExit=yes\nKillMode=process\nEnvironment=\"DOWNLOAD_DIR=/opt/bin\"\nEnvironment=\"CRICTL_VERSION=v1.22.0\"\nEnvironment=\"ARCH=amd64\"\nEnvironment=\"URL=https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz\"\nExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}\nExecStart=/bin/sh -c \"/usr/bin/curl -L ${URL} \| /usr/bin/tar -C $DOWNLOAD_DIR -xz\"\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"install-crictl.service"},{"contents":"[Unit]\nDescription=Install K8S\nWants = network-online.target\nAfter = network.target network-online.target\nConditionPathExists=!/opt/bin/kubelet\n[Service]\nWorkingDirectory=/opt/bin\nType=forking\nTimeoutStartSec=180\nRemainAfterExit=yes\nKillMode=process\nEnvironment=\"KUBE_RELEASE=v1.25.4\"\nEnvironment=\"ARCH=amd64\"\nEnvironment=\"DOWNLOAD_DIR=/opt/bin\"\nEnvironment=\"URL=https://storage.googleapis.com/kubernetes-release/release/${KUBE_RELEASE}/bin/linux/${ARCH}\"\nExecStartPre=/bin/mkdir -p ${DOWNLOAD_DIR}\nExecStart=/bin/sh -c \"/usr/bin/curl -L --remote-name-all ${URL}/{kubeadm,kubelet,kubectl}\"\nExecStartPost=/bin/sh -c \"/bin/chmod +x ${DOWNLOAD_DIR}/{kubeadm,kubelet,kubectl}\"\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"install-kubernetes.service"},{"contents":"[Unit]\nDescription=The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\nWants=network-online.target\nAfter=network-online.target\n[Service]\nExecStart=/opt/bin/kubelet\nRestart=always\nStartLimitInterval=0\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironment=\"KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\"\nEnvironment=\"KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml\"\nEnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env\nEnvironmentFile=-/etc/default/kubelet\nExecStart=\nExecStart=/opt/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS\n","name":"10-kubeadm.conf"}],"enabled":true,"name":"kubelet.service"}]}}