diff --git a/aws/container-linux/kubernetes/bootstrap.tf b/aws/container-linux/kubernetes/bootstrap.tf index 0d7266e7..1db669b9 100644 --- a/aws/container-linux/kubernetes/bootstrap.tf +++ b/aws/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl index c4cb7b2f..246c617e 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/aws/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -108,6 +108,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -135,13 +137,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/aws/container-linux/kubernetes/ssh.tf b/aws/container-linux/kubernetes/ssh.tf index 1d13aa55..f180f312 100644 --- a/aws/container-linux/kubernetes/ssh.tf +++ b/aws/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -14,63 +23,13 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index f8651eae..c2716fb8 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/aws/fedora-coreos/kubernetes/fcc/controller.yaml b/aws/fedora-coreos/kubernetes/fcc/controller.yaml index dc2edb21..8db33872 100644 --- a/aws/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/aws/fedora-coreos/kubernetes/fcc/controller.yaml @@ -119,6 +119,7 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ k8s.gcr.io/hyperkube:v1.16.3 \ @@ -135,12 +136,33 @@ storage: contents: inline: | ${kubeconfig} + - path: /opt/bootstrap/layout + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/aws/fedora-coreos/kubernetes/ssh.tf b/aws/fedora-coreos/kubernetes/ssh.tf index d4a61eb5..f11ae0b2 100644 --- a/aws/fedora-coreos/kubernetes/ssh.tf +++ b/aws/fedora-coreos/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -12,65 +21,15 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - + provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/" + "sudo /opt/bootstrap/layout", ] } } diff --git a/azure/container-linux/kubernetes/bootstrap.tf b/azure/container-linux/kubernetes/bootstrap.tf index b82a5dd0..e26c8cf2 100644 --- a/azure/container-linux/kubernetes/bootstrap.tf +++ b/azure/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl index 49135068..5e83e996 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/azure/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -106,6 +106,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -133,13 +135,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/azure/container-linux/kubernetes/ssh.tf b/azure/container-linux/kubernetes/ssh.tf index 2f0507f6..f93e096d 100644 --- a/azure/container-linux/kubernetes/ssh.tf +++ b/azure/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -13,65 +22,15 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - + provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/bare-metal/container-linux/kubernetes/bootstrap.tf b/bare-metal/container-linux/kubernetes/bootstrap.tf index f04bc00e..cb751a16 100644 --- a/bare-metal/container-linux/kubernetes/bootstrap.tf +++ b/bare-metal/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl index a6c1bece..2a6b32e4 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -121,6 +121,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -148,13 +150,35 @@ storage: contents: inline: ${domain_name} + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/bare-metal/container-linux/kubernetes/ssh.tf b/bare-metal/container-linux/kubernetes/ssh.tf index 5edea88e..fcab4735 100644 --- a/bare-metal/container-linux/kubernetes/ssh.tf +++ b/bare-metal/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = length(var.controllers) @@ -24,64 +33,14 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index eeb55a8b..d0484943 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index a88013b8..2132bb24 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -130,6 +130,7 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/podman run --name bootstrap \ --network host \ + --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \ --volume /opt/bootstrap/assets:/assets:ro,Z \ --volume /opt/bootstrap/apply:/apply:ro,Z \ k8s.gcr.io/hyperkube:v1.16.3 \ @@ -146,12 +147,33 @@ storage: contents: inline: ${domain_name} + - path: /opt/bootstrap/layout + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf index 8ad6d0d1..560d96fe 100644 --- a/bare-metal/fedora-coreos/kubernetes/ssh.tf +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = length(var.controllers) @@ -23,62 +32,14 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/" + "sudo /opt/bootstrap/layout", ] } } diff --git a/digital-ocean/container-linux/kubernetes/bootstrap.tf b/digital-ocean/container-linux/kubernetes/bootstrap.tf index 9a762a5c..196ae533 100644 --- a/digital-ocean/container-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl index ffc8cf32..8c94dd65 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -118,6 +118,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -139,13 +141,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/digital-ocean/container-linux/kubernetes/ssh.tf b/digital-ocean/container-linux/kubernetes/ssh.tf index 12f5bcb9..469fc947 100644 --- a/digital-ocean/container-linux/kubernetes/ssh.tf +++ b/digital-ocean/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. Activates kubelet.service resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -20,64 +29,14 @@ resource "null_resource" "copy-controller-secrets" { } provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } } diff --git a/google-cloud/container-linux/kubernetes/bootstrap.tf b/google-cloud/container-linux/kubernetes/bootstrap.tf index 157944f5..36ae40e3 100644 --- a/google-cloud/container-linux/kubernetes/bootstrap.tf +++ b/google-cloud/container-linux/kubernetes/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl index 49ed0fc2..caf120ee 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml.tmpl @@ -107,6 +107,8 @@ systemd: ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*' ExecStart=/usr/bin/rkt run \ --trust-keys-from-https \ + --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \ + --mount volume=config,target=/etc/kubernetes/secrets \ --volume assets,kind=host,source=/opt/bootstrap/assets \ --mount volume=assets,target=/assets \ --volume script,kind=host,source=/opt/bootstrap/apply \ @@ -134,13 +136,35 @@ storage: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.16.3 + - path: /opt/bootstrap/layout + filesystem: root + mode: 0544 + contents: + inline: | + #!/bin/bash -e + mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking + awk '/#####/ {filename=$2; next} {print > filename}' assets + mkdir -p /etc/ssl/etcd/etcd + mkdir -p /etc/kubernetes/bootstrap-secrets + mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/ + mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/ + chown -R etcd:etcd /etc/ssl/etcd + chmod -R 500 /etc/ssl/etcd + mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/ + mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/ + sudo mkdir -p /etc/kubernetes/manifests + sudo mv static-manifests/* /etc/kubernetes/manifests/ + sudo mkdir -p /opt/bootstrap/assets + sudo mv manifests /opt/bootstrap/assets/manifests + sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking + rm -rf assets auth static-manifests tls - path: /opt/bootstrap/apply filesystem: root mode: 0544 contents: inline: | #!/bin/bash -e - export KUBECONFIG=/assets/auth/kubeconfig + export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig until kubectl version; do echo "Waiting for static pod control plane" sleep 5 diff --git a/google-cloud/container-linux/kubernetes/ssh.tf b/google-cloud/container-linux/kubernetes/ssh.tf index a85e23d1..f6983447 100644 --- a/google-cloud/container-linux/kubernetes/ssh.tf +++ b/google-cloud/container-linux/kubernetes/ssh.tf @@ -1,3 +1,12 @@ +locals { + # format assets for distribution + assets_bundle = [ + # header with the unpack location + for key, value in module.bootstrap.assets_dist: + format("##### %s\n%s", key, value) + ] +} + # Secure copy assets to controllers. resource "null_resource" "copy-controller-secrets" { count = var.controller_count @@ -12,65 +21,15 @@ resource "null_resource" "copy-controller-secrets" { user = "core" timeout = "15m" } - + provisioner "file" { - content = module.bootstrap.etcd_ca_cert - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_cert - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_client_key - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_cert - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_server_key - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_cert - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = module.bootstrap.etcd_peer_key - destination = "$HOME/etcd-peer.key" - } - - provisioner "file" { - source = var.asset_dir + content = join("\n", local.assets_bundle) destination = "$HOME/assets" } provisioner "remote-exec" { inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv $HOME/assets /opt/bootstrap/assets", - "sudo mkdir -p /etc/kubernetes/manifests", - "sudo mkdir -p /etc/kubernetes/bootstrap-secrets", - "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/", - "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/", - "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/", + "sudo /opt/bootstrap/layout", ] } }