diff --git a/digital-ocean/fedora-cloud/kubernetes/LICENSE b/digital-ocean/fedora-atomic/kubernetes/LICENSE similarity index 100% rename from digital-ocean/fedora-cloud/kubernetes/LICENSE rename to digital-ocean/fedora-atomic/kubernetes/LICENSE diff --git a/digital-ocean/fedora-cloud/kubernetes/README.md b/digital-ocean/fedora-atomic/kubernetes/README.md similarity index 91% rename from digital-ocean/fedora-cloud/kubernetes/README.md rename to digital-ocean/fedora-atomic/kubernetes/README.md index 9e83a163..43142532 100644 --- a/digital-ocean/fedora-cloud/kubernetes/README.md +++ b/digital-ocean/fedora-atomic/kubernetes/README.md @@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features -* Kubernetes v1.9.6 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) +* Kubernetes v1.10.0 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube)) * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-cloud/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf similarity index 84% rename from digital-ocean/fedora-cloud/kubernetes/bootkube.tf rename to digital-ocean/fedora-atomic/kubernetes/bootkube.tf index b0488323..efeefce7 100644 --- a/digital-ocean/fedora-cloud/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=5f3546b66ffb9946b36e612537bb6a1830ae7746" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=61fb176647e15d4d0e72fdccb34d27e47430407c" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -11,4 +11,7 @@ module "bootkube" { pod_cidr = "${var.pod_cidr}" service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" + + # Fedora + trusted_certs_dir = "/etc/pki/tls/certs" } diff --git a/digital-ocean/fedora-cloud/kubernetes/cloudinit/controller.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl similarity index 80% rename from digital-ocean/fedora-cloud/kubernetes/cloudinit/controller.yaml.tmpl rename to digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl index bd1fd307..ad137469 100644 --- a/digital-ocean/fedora-cloud/kubernetes/cloudinit/controller.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/controller.yaml.tmpl @@ -1,29 +1,5 @@ #cloud-config -yum_repos: - kubernetes: - name: kubernetes - baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 - enabled: true - gpgcheck: true - gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -packages: - - [docker, 1.13.1] - - [kubelet, 1.10.0] - - nfs-utils write_files: - - path: /etc/systemd/system/cloud-metadata.service - content: | - [Unit] - Description=Digital Ocean metadata agent - [Service] - Type=oneshot - Environment=OUTPUT=/run/metadata/digitalocean - ExecStart=/usr/bin/mkdir -p /run/metadata - ExecStart=/usr/bin/bash -c 'echo "DIGITALOCEAN_IPV4_PUBLIC_0=$(curl\ - --url http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address\ - --retry 10)\nDIGITALOCEAN_IPV4_PRIVATE_0=$(curl\ - --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\ - --retry 10)" > $${OUTPUT}' - path: /etc/systemd/system/etcd-member.service content: | [Unit] @@ -38,7 +14,7 @@ write_files: ExecStartPre=/bin/mkdir -p /var/lib/etcd ExecStart=/usr/bin/docker run --rm --name etcd-member \ --net=host \ - -v /usr/share/ca-certificates:/usr/share/ca-certificates:ro,z \ + -v /etc/pki/tls/certs:/usr/share/ca-certificates:ro,z \ -v /etc/ssl/etcd:/etc/ssl/certs:ro,Z \ -v /var/lib/etcd:/var/lib/etcd:Z \ --env-file=/etc/etcd/etcd.conf \ @@ -68,7 +44,20 @@ write_files: ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key ETCD_PEER_CLIENT_CERT_AUTH=true - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf + - path: /etc/systemd/system/cloud-metadata.service + content: | + [Unit] + Description=Digital Ocean metadata agent + [Service] + Type=oneshot + Environment=OUTPUT=/run/metadata/digitalocean + ExecStart=/usr/bin/mkdir -p /run/metadata + ExecStart=/usr/bin/bash -c 'echo "DIGITALOCEAN_IPV4_PUBLIC_0=$(curl\ + --url http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address\ + --retry 10)\nDIGITALOCEAN_IPV4_PRIVATE_0=$(curl\ + --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\ + --retry 10)" > $${OUTPUT}' + - path: /etc/systemd/system/kubelet.service content: | [Unit] Description=Kubelet @@ -76,6 +65,7 @@ write_files: After=cloud-metadata.service Wants=rpc-statd.service [Service] + WorkingDirectory=/etc/kubernetes EnvironmentFile=/run/metadata/digitalocean ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -85,8 +75,11 @@ write_files: ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + # Atomic's system containers and RPMs are old and unfriendly. Use this. + ExecStartPre=/usr/bin/curl -L https://dl.k8s.io/v1.10.0/kubernetes-node-linux-amd64.tar.gz -o kubernetes-node-linux-amd64.tar.gz + ExecStartPre=/usr/bin/tar xzf kubernetes-node-linux-amd64.tar.gz -C /usr/local/bin --strip-components=3 kubernetes/node/bin/kubelet ExecStart= - ExecStart=/usr/bin/kubelet \ + ExecStart=/usr/local/bin/kubelet \ --allow-privileged \ --anonymous-auth=false \ --cgroup-driver=systemd \ @@ -116,43 +109,50 @@ write_files: PathExists=/etc/kubernetes/kubeconfig [Install] WantedBy=multi-user.target - - path: /etc/kubernetes/.keep - - path: /etc/selinux/config - content: | - SELINUX=permissive - path: /etc/systemd/system/bootkube.service content: | [Unit] Description=Bootstrap a Kubernetes cluster - ConditionPathExists=!/opt/bootkube/init_bootkube.done + ConditionPathExists=!/var/bootkube/init_bootkube.done [Service] Type=oneshot RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/opt/bootkube/bootkube-start - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done + WorkingDirectory=/var/bootkube + ExecStartPre=/bin/mkdir -p /var/bootkube + ExecStart=/usr/local/bin/bootkube-start + ExecStartPost=/bin/touch /var/bootkube/init_bootkube.done [Install] WantedBy=multi-user.target - - path: /opt/bootkube/bootkube-start - permissions: '0544' + - path: /etc/kubernetes/.keep + - path: /var/bootkube/.keep + - path: /etc/selinux/config + owner: root:root + permissions: '0644' + content: | + SELINUX=permissive + SELINUXTYPE=targeted + - path: /usr/local/bin/bootkube-start + permissions: '0755' content: | #!/bin/bash -e # Wrapper for bootkube start - [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-* + [ -n "$(ls /var/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /var/bootkube/assets/manifests-*/* /var/bootkube/assets/manifests && rm -rf /var/bootkube/assets/manifests-* /usr/bin/docker run --rm --name bootkube \ + --net=host \ --volume /etc/kubernetes:/etc/kubernetes:Z \ - --volume /opt/bootkube/assets:/assets:Z \ + --volume /var/bootkube/assets:/assets:Z \ --entrypoint=/bootkube \ quay.io/coreos/bootkube:v0.11.0 start --asset-dir=/assets +bootcmd: + - [setenforce, Permissive] runcmd: - [systemctl, daemon-reload] - - [systemctl, enable, docker.service] - - [systemctl, start, --no-block, docker.service] - [systemctl, enable, etcd-member.service] - [systemctl, start, --no-block, etcd-member.service] - [systemctl, enable, cloud-metadata.service] - [systemctl, enable, kubelet.path] - [systemctl, start, --no-block, kubelet.path] + - [systemctl, disable, firewalld, --now] users: - default - name: fedora diff --git a/digital-ocean/fedora-cloud/kubernetes/cloudinit/worker.yaml.tmpl b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl similarity index 82% rename from digital-ocean/fedora-cloud/kubernetes/cloudinit/worker.yaml.tmpl rename to digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl index 8b209a7c..0adee7e0 100644 --- a/digital-ocean/fedora-cloud/kubernetes/cloudinit/worker.yaml.tmpl +++ b/digital-ocean/fedora-atomic/kubernetes/cloudinit/worker.yaml.tmpl @@ -1,15 +1,4 @@ #cloud-config -yum_repos: - kubernetes: - name: kubernetes - baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 - enabled: true - gpgcheck: true - gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -packages: - - [docker, 1.13.1] - - [kubelet, 1.10.0] - - nfs-utils write_files: - path: /etc/systemd/system/cloud-metadata.service content: | @@ -24,7 +13,7 @@ write_files: --retry 10)\nDIGITALOCEAN_IPV4_PRIVATE_0=$(curl\ --url http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address\ --retry 10)" > $${OUTPUT}' - - path: /etc/systemd/system/kubelet.service.d/10-typhoon.conf + - path: /etc/systemd/system/kubelet.service content: | [Unit] Description=Kubelet @@ -32,6 +21,7 @@ write_files: After=cloud-metadata.service Wants=rpc-statd.service [Service] + WorkingDirectory=/etc/kubernetes EnvironmentFile=/run/metadata/digitalocean ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -41,8 +31,11 @@ write_files: ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" + # Atomic's system containers and RPMs are old and unfriendly. Use this. + ExecStartPre=/usr/bin/curl -L https://dl.k8s.io/v1.10.0/kubernetes-node-linux-amd64.tar.gz -o kubernetes-node-linux-amd64.tar.gz + ExecStartPre=/usr/bin/tar xzf kubernetes-node-linux-amd64.tar.gz -C /usr/local/bin --strip-components=3 kubernetes/node/bin/kubelet ExecStart= - ExecStart=/usr/bin/kubelet \ + ExecStart=/usr/local/bin/kubelet \ --allow-privileged \ --anonymous-auth=false \ --cgroup-driver=systemd \ @@ -72,15 +65,19 @@ write_files: WantedBy=multi-user.target - path: /etc/kubernetes/.keep - path: /etc/selinux/config + owner: root:root + permissions: '0644' content: | SELINUX=permissive + SELINUXTYPE=targeted +bootcmd: + - [setenforce, Permissive] runcmd: - [systemctl, daemon-reload] - - [systemctl, enable, docker.service] - - [systemctl, start, --no-block, docker.service] - - [systemctl, enable, cloud-metadata.service] - [systemctl, enable, kubelet.path] + - [systemctl, enable, cloud-metadata.service] - [systemctl, start, --no-block, kubelet.path] + - [systemctl, disable, firewalld, --now] users: - default - name: fedora diff --git a/digital-ocean/fedora-cloud/kubernetes/controllers.tf b/digital-ocean/fedora-atomic/kubernetes/controllers.tf similarity index 100% rename from digital-ocean/fedora-cloud/kubernetes/controllers.tf rename to digital-ocean/fedora-atomic/kubernetes/controllers.tf diff --git a/digital-ocean/fedora-cloud/kubernetes/network.tf b/digital-ocean/fedora-atomic/kubernetes/network.tf similarity index 100% rename from digital-ocean/fedora-cloud/kubernetes/network.tf rename to digital-ocean/fedora-atomic/kubernetes/network.tf diff --git a/digital-ocean/fedora-cloud/kubernetes/outputs.tf b/digital-ocean/fedora-atomic/kubernetes/outputs.tf similarity index 100% rename from digital-ocean/fedora-cloud/kubernetes/outputs.tf rename to digital-ocean/fedora-atomic/kubernetes/outputs.tf diff --git a/digital-ocean/fedora-cloud/kubernetes/require.tf b/digital-ocean/fedora-atomic/kubernetes/require.tf similarity index 100% rename from digital-ocean/fedora-cloud/kubernetes/require.tf rename to digital-ocean/fedora-atomic/kubernetes/require.tf diff --git a/digital-ocean/fedora-cloud/kubernetes/ssh.tf b/digital-ocean/fedora-atomic/kubernetes/ssh.tf similarity index 98% rename from digital-ocean/fedora-cloud/kubernetes/ssh.tf rename to digital-ocean/fedora-atomic/kubernetes/ssh.tf index 868195c7..582944dd 100644 --- a/digital-ocean/fedora-cloud/kubernetes/ssh.tf +++ b/digital-ocean/fedora-atomic/kubernetes/ssh.tf @@ -110,7 +110,7 @@ resource "null_resource" "bootkube-start" { provisioner "remote-exec" { inline = [ "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do sleep 4; done", - "sudo mv $HOME/assets /opt/bootkube", + "sudo mv $HOME/assets /var/bootkube", "sudo systemctl start bootkube", ] } diff --git a/digital-ocean/fedora-cloud/kubernetes/variables.tf b/digital-ocean/fedora-atomic/kubernetes/variables.tf similarity index 97% rename from digital-ocean/fedora-cloud/kubernetes/variables.tf rename to digital-ocean/fedora-atomic/kubernetes/variables.tf index 575cf519..a3cbcab4 100644 --- a/digital-ocean/fedora-cloud/kubernetes/variables.tf +++ b/digital-ocean/fedora-atomic/kubernetes/variables.tf @@ -43,8 +43,8 @@ variable "worker_type" { variable "image" { type = "string" - default = "fedora-27-x64" - description = "OS image from which to initialize the disk (e.g. fedora-27-x64)" + default = "fedora-27-x64-atomic" + description = "OS image from which to initialize the disk (e.g. fedora-27-x64-atomic)" } # configuration diff --git a/digital-ocean/fedora-cloud/kubernetes/workers.tf b/digital-ocean/fedora-atomic/kubernetes/workers.tf similarity index 100% rename from digital-ocean/fedora-cloud/kubernetes/workers.tf rename to digital-ocean/fedora-atomic/kubernetes/workers.tf