Run etcd-member.service across controllers

* Running the etcd container with NOTIFY_SOCKET mounted
(to use systemd Type=notify) causes podman to hang so
for now just use exec
* https://github.com/opencontainers/runc/pull/1807

bare-metal/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -1,23 +1,82 @@
 ---
 variant: fcos
 version: 1.0.0
+systemd:
+  units:
+    - name: etcd-member.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description=etcd (System Container)
+        Documentation=https://github.com/coreos/etcd
+        Wants=network-online.target network.target
+        After=network-online.target
+        [Service]
+        # https://github.com/opencontainers/runc/pull/1807
+        # Type=notify
+        # NotifyAccess=exec
+        Type=exec
+        Restart=on-failure
+        RestartSec=10s
+        TimeoutStartSec=0
+        LimitNOFILE=40000
+
+        ExecStartPre=/bin/chcon -t bin_t /opt/bin/etcd-wrapper
+        ExecStartPre=-/usr/bin/podman rm etcd
+        ExecStart=/opt/bin/etcd-wrapper
+        ExecStop=/usr/bin/podman stop etcd
+        [Install]
+        WantedBy=multi-user.target
 storage:
   files:
+    - path: /etc/etcd/etcd.env
+      mode: 0644
+      contents:
+        inline: |
+          NOTIFY_SOCKET=/run/systemd/notify
+          ETCD_NAME=${etcd_name}
+          ETCD_DATA_DIR=/var/lib/etcd
+          ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379
+          ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380
+          ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
+          ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
+          ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
+          ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
+          ETCD_STRICT_RECONFIG_CHECK=true
+          ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt
+          ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt
+          ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key
+          ETCD_CLIENT_CERT_AUTH=true
+          ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt
+          ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
+          ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
+          ETCD_PEER_CLIENT_CERT_AUTH=true
+    - path: /opt/bin/etcd-wrapper
+      mode: 0544
+      contents:
+        inline: |
+          #!/usr/bin/bash -e
+          #--volume $${NOTIFY_SOCKET}:/run/systemd/notify \
+          set -x
+          mkdir -p /var/lib/etcd
+          exec podman run --name etcd \
+            --env-file /etc/etcd/etcd.env \
+            --network host \
+            --volume /var/lib/etcd:/var/lib/etcd:rw,Z \
+            --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
+            quay.io/coreos/etcd:v3.3.13
     - path: /etc/kubernetes/kubelet.env
-      filesystem: root
       mode: 0644
       contents:
         inline: |
           KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
           KUBELET_IMAGE_TAG=v1.15.0
     - path: /etc/hostname
-      filesystem: root
       mode: 0644
       contents:
         inline:
           ${domain_name}
     - path: /etc/sysctl.d/max-user-watches.conf
-      filesystem: root
       contents:
         inline: |
           fs.inotify.max_user_watches=16184

bare-metal/fedora-coreos/kubernetes/ssh.tf

@@ -0,0 +1,73 @@
+# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service
+resource "null_resource" "copy-controller-secrets" {
+  count = length(var.controller_names)
+
+  # Without depends_on, remote-exec could start and wait for machines before
+  # matchbox groups are written, causing a deadlock.
+  depends_on = [
+    matchbox_group.controller,
+    matchbox_group.worker,
+  ]
+
+  connection {
+    type    = "ssh"
+    host    = var.controller_domains[count.index]
+    user    = "core"
+    timeout = "60m"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.kubeconfig-kubelet
+    destination = "$HOME/kubeconfig"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.etcd_ca_cert
+    destination = "$HOME/etcd-client-ca.crt"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.etcd_client_cert
+    destination = "$HOME/etcd-client.crt"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.etcd_client_key
+    destination = "$HOME/etcd-client.key"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.etcd_server_cert
+    destination = "$HOME/etcd-server.crt"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.etcd_server_key
+    destination = "$HOME/etcd-server.key"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.etcd_peer_cert
+    destination = "$HOME/etcd-peer.crt"
+  }
+
+  provisioner "file" {
+    content     = module.bootkube.etcd_peer_key
+    destination = "$HOME/etcd-peer.key"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo mkdir -p /etc/ssl/etcd/etcd",
+      "sudo mv etcd-client* /etc/ssl/etcd/",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
+      "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
+      "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
+      "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
+      "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
+      "sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
+    ]
+  }
+}
+