Run etcd-member.service across controllers

* Running the etcd container with NOTIFY_SOCKET mounted
(to use systemd Type=notify) causes podman to hang so
for now just use exec
* https://github.com/opencontainers/runc/pull/1807
This commit is contained in:
Dalton Hubble 2019-07-13 23:24:56 -07:00
parent eb92f67125
commit aab14c5573
2 changed files with 135 additions and 3 deletions

View File

@ -1,23 +1,82 @@
--- ---
variant: fcos variant: fcos
version: 1.0.0 version: 1.0.0
systemd:
units:
- name: etcd-member.service
enabled: true
contents: |
[Unit]
Description=etcd (System Container)
Documentation=https://github.com/coreos/etcd
Wants=network-online.target network.target
After=network-online.target
[Service]
# https://github.com/opencontainers/runc/pull/1807
# Type=notify
# NotifyAccess=exec
Type=exec
Restart=on-failure
RestartSec=10s
TimeoutStartSec=0
LimitNOFILE=40000
ExecStartPre=/bin/chcon -t bin_t /opt/bin/etcd-wrapper
ExecStartPre=-/usr/bin/podman rm etcd
ExecStart=/opt/bin/etcd-wrapper
ExecStop=/usr/bin/podman stop etcd
[Install]
WantedBy=multi-user.target
storage: storage:
files: files:
- path: /etc/etcd/etcd.env
mode: 0644
contents:
inline: |
NOTIFY_SOCKET=/run/systemd/notify
ETCD_NAME=${etcd_name}
ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=https://${domain_name}:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${domain_name}:2380
ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
ETCD_STRICT_RECONFIG_CHECK=true
ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt
ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt
ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key
ETCD_CLIENT_CERT_AUTH=true
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt
ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
ETCD_PEER_CLIENT_CERT_AUTH=true
- path: /opt/bin/etcd-wrapper
mode: 0544
contents:
inline: |
#!/usr/bin/bash -e
#--volume $${NOTIFY_SOCKET}:/run/systemd/notify \
set -x
mkdir -p /var/lib/etcd
exec podman run --name etcd \
--env-file /etc/etcd/etcd.env \
--network host \
--volume /var/lib/etcd:/var/lib/etcd:rw,Z \
--volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
quay.io/coreos/etcd:v3.3.13
- path: /etc/kubernetes/kubelet.env - path: /etc/kubernetes/kubelet.env
filesystem: root
mode: 0644 mode: 0644
contents: contents:
inline: | inline: |
KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
KUBELET_IMAGE_TAG=v1.15.0 KUBELET_IMAGE_TAG=v1.15.0
- path: /etc/hostname - path: /etc/hostname
filesystem: root
mode: 0644 mode: 0644
contents: contents:
inline: inline:
${domain_name} ${domain_name}
- path: /etc/sysctl.d/max-user-watches.conf - path: /etc/sysctl.d/max-user-watches.conf
filesystem: root
contents: contents:
inline: | inline: |
fs.inotify.max_user_watches=16184 fs.inotify.max_user_watches=16184

View File

@ -0,0 +1,73 @@
# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service
resource "null_resource" "copy-controller-secrets" {
count = length(var.controller_names)
# Without depends_on, remote-exec could start and wait for machines before
# matchbox groups are written, causing a deadlock.
depends_on = [
matchbox_group.controller,
matchbox_group.worker,
]
connection {
type = "ssh"
host = var.controller_domains[count.index]
user = "core"
timeout = "60m"
}
provisioner "file" {
content = module.bootkube.kubeconfig-kubelet
destination = "$HOME/kubeconfig"
}
provisioner "file" {
content = module.bootkube.etcd_ca_cert
destination = "$HOME/etcd-client-ca.crt"
}
provisioner "file" {
content = module.bootkube.etcd_client_cert
destination = "$HOME/etcd-client.crt"
}
provisioner "file" {
content = module.bootkube.etcd_client_key
destination = "$HOME/etcd-client.key"
}
provisioner "file" {
content = module.bootkube.etcd_server_cert
destination = "$HOME/etcd-server.crt"
}
provisioner "file" {
content = module.bootkube.etcd_server_key
destination = "$HOME/etcd-server.key"
}
provisioner "file" {
content = module.bootkube.etcd_peer_cert
destination = "$HOME/etcd-peer.crt"
}
provisioner "file" {
content = module.bootkube.etcd_peer_key
destination = "$HOME/etcd-peer.key"
}
provisioner "remote-exec" {
inline = [
"sudo mkdir -p /etc/ssl/etcd/etcd",
"sudo mv etcd-client* /etc/ssl/etcd/",
"sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
"sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
"sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
"sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
"sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
"sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
"sudo mv $HOME/kubeconfig /etc/kubernetes/kubeconfig",
]
}
}