From aab14c55738bc0d4c7129c45e8b76a3157d3571c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 13 Jul 2019 23:24:56 -0700 Subject: [PATCH] Run etcd-member.service across controllers * Running the etcd container with NOTIFY_SOCKET mounted (to use systemd Type=notify) causes podman to hang so for now just use exec * https://github.com/opencontainers/runc/pull/1807 --- .../kubernetes/fcc/controller.yaml | 65 ++++++++++++++++- bare-metal/fedora-coreos/kubernetes/ssh.tf | 73 +++++++++++++++++++ 2 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 bare-metal/fedora-coreos/kubernetes/ssh.tf diff --git a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml index e4d8934c..b1d8bdbc 100644 --- a/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml @@ -1,23 +1,82 @@ --- variant: fcos version: 1.0.0 +systemd: + units: + - name: etcd-member.service + enabled: true + contents: | + [Unit] + Description=etcd (System Container) + Documentation=https://github.com/coreos/etcd + Wants=network-online.target network.target + After=network-online.target + [Service] + # https://github.com/opencontainers/runc/pull/1807 + # Type=notify + # NotifyAccess=exec + Type=exec + Restart=on-failure + RestartSec=10s + TimeoutStartSec=0 + LimitNOFILE=40000 + + ExecStartPre=/bin/chcon -t bin_t /opt/bin/etcd-wrapper + ExecStartPre=-/usr/bin/podman rm etcd + ExecStart=/opt/bin/etcd-wrapper + ExecStop=/usr/bin/podman stop etcd + [Install] + WantedBy=multi-user.target storage: files: + - path: /etc/etcd/etcd.env + mode: 0644 + contents: + inline: | + NOTIFY_SOCKET=/run/systemd/notify + ETCD_NAME=${etcd_name} + ETCD_DATA_DIR=/var/lib/etcd + ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379 + ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380 + ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379 + ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380 + ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381 + ETCD_INITIAL_CLUSTER=${etcd_initial_cluster} + ETCD_STRICT_RECONFIG_CHECK=true + ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt + ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt + ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key + ETCD_CLIENT_CERT_AUTH=true + ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt + ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt + ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key + ETCD_PEER_CLIENT_CERT_AUTH=true + - path: /opt/bin/etcd-wrapper + mode: 0544 + contents: + inline: | + #!/usr/bin/bash -e + #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \ + set -x + mkdir -p /var/lib/etcd + exec podman run --name etcd \ + --env-file /etc/etcd/etcd.env \ + --network host \ + --volume /var/lib/etcd:/var/lib/etcd:rw,Z \ + --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ + quay.io/coreos/etcd:v3.3.13 - path: /etc/kubernetes/kubelet.env - filesystem: root mode: 0644 contents: inline: | KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_TAG=v1.15.0 - path: /etc/hostname - filesystem: root mode: 0644 contents: inline: ${domain_name} - path: /etc/sysctl.d/max-user-watches.conf - filesystem: root contents: inline: | fs.inotify.max_user_watches=16184 diff --git a/bare-metal/fedora-coreos/kubernetes/ssh.tf b/bare-metal/fedora-coreos/kubernetes/ssh.tf new file mode 100644 index 00000000..f11ad79f --- /dev/null +++ b/bare-metal/fedora-coreos/kubernetes/ssh.tf @@ -0,0 +1,73 @@ +# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service +resource "null_resource" "copy-controller-secrets" { + count = length(var.controller_names) + + # Without depends_on, remote-exec could start and wait for machines before + # matchbox groups are written, causing a deadlock. + depends_on = [ + matchbox_group.controller, + matchbox_group.worker, + ] + + connection { + type = "ssh" + host = var.controller_domains[count.index] + user = "core" + timeout = "60m" + } + + provisioner "file" { + content = module.bootkube.kubeconfig-kubelet + destination = "$HOME/kubeconfig" + } + + provisioner "file" { + content = module.bootkube.etcd_ca_cert + destination = "$HOME/etcd-client-ca.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_cert + destination = "$HOME/etcd-client.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_client_key + destination = "$HOME/etcd-client.key" + } + + provisioner "file" { + content = module.bootkube.etcd_server_cert + destination = "$HOME/etcd-server.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_server_key + destination = "$HOME/etcd-server.key" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_cert + destination = "$HOME/etcd-peer.crt" + } + + provisioner "file" { + content = module.bootkube.etcd_peer_key + destination = "$HOME/etcd-peer.key" + } + + provisioner "remote-exec" { + inline = [ + "sudo mkdir -p /etc/ssl/etcd/etcd", + "sudo mv etcd-client* /etc/ssl/etcd/", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", + "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", + "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", + "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", + "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", + "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", + "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig", + ] + } +} +