Bump bootkube and terraform-render-bootkube to v0.8.1

* Use the v0.8.1 tagged terraform-render-bootkube module
* Use the v0.8.1 quay.io/coreos/bootkube image to bootstrap

CHANGES.md

@@ -2,6 +2,29 @@
 
 Notable changes between versions.
 
+## Latest
+
+## v1.8.2
+
+* Kubernetes v1.8.2
+  * Fixes a memory leak in the v1.8.1 apiserver ([kubernetes#53485](https://github.com/kubernetes/kubernetes/issues/53485))
+* Switch to using the `gcr.io/google_containers/hyperkube`
+* Update flannel from v0.8.0 to v0.9.0
+* Add `hairpinMode` to flannel CNI config
+* Add `--no-negcache` to kube-dns dnsmasq
+* Use kubernetes-incubator/bootkube v0.8.1
+
+## v1.8.1
+
+* Kubernetes v1.8.1
+* Use kubernetes-incubator/bootkube v0.8.0
+
+#### Digital Ocean
+
+* Run etcd cluster across controller nodes (etcd-member.service)
+* Remove support for self-hosted etcd
+* Reduce time to bootstrap a cluster
+
 ## v1.7.7
 
 * Kubernetes v1.7.7

README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features
 
-* Kubernetes v1.7.7 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.8.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Ready for Ingress, Dashboards, Metrics, and other optional [addons](https://typhoon.psdn.io/addons/overview/)
@@ -23,9 +23,9 @@ Typhoon provides a Terraform Module for each supported operating system and plat
 | Platform      | Operating System | Terraform Module | Status |
 |---------------|------------------|------------------|--------|
 | AWS           | Container Linux  | [aws/container-linux/kubernetes](aws/container-linux/kubernetes) | alpha |
-| Bare-Metal    | Container Linux  | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | production |
+| Bare-Metal    | Container Linux  | [bare-metal/container-linux/kubernetes](bare-metal/container-linux/kubernetes) | stable |
 | Digital Ocean | Container Linux  | [digital-ocean/container-linux/kubernetes](digital-ocean/container-linux/kubernetes) | beta |
-| Google Cloud  | Container Linux  | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | production |
+| Google Cloud  | Container Linux  | [google-cloud/container-linux/kubernetes](google-cloud/container-linux/kubernetes) | beta |
 
 ## Usage
 
@@ -78,9 +78,9 @@ In 5-10 minutes (varies by platform), the cluster will be ready. This Google Clo
 $ KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig
 $ kubectl get nodes
 NAME                                          STATUS   AGE    VERSION
-yavin-controller-1682.c.example-com.internal  Ready    6m     v1.7.7+coreos.0
-yavin-worker-jrbf.c.example-com.internal      Ready    5m     v1.7.7+coreos.0
-yavin-worker-mzdm.c.example-com.internal      Ready    5m     v1.7.7+coreos.0
+yavin-controller-1682.c.example-com.internal  Ready    6m     v1.8.2
+yavin-worker-jrbf.c.example-com.internal      Ready    5m     v1.8.2
+yavin-worker-mzdm.c.example-com.internal      Ready    5m     v1.8.2
 ```
 
 List the pods.

addons/cluo/cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: reboot-coordinator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: reboot-coordinator
+subjects:
+  - kind: ServiceAccount
+    namespace: reboot-coordinator
+    name: default

addons/cluo/cluster-role.yaml

@@ -0,0 +1,45 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: reboot-coordinator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+      - get
+      - update
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - delete
+  - apiGroups:
+      - "extensions"
+    resources:
+      - daemonsets
+    verbs:
+      - get

addons/cluo/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: reboot-coordinator

addons/cluo/update-agent.yaml

@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
   name: container-linux-update-agent
-  namespace: kube-system
+  namespace: reboot-coordinator
 spec:
   updateStrategy:
     type: RollingUpdate
@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: update-agent
-        image: quay.io/coreos/container-linux-update-operator:v0.3.1
+        image: quay.io/coreos/container-linux-update-operator:v0.4.1
         command:
         - "/bin/update-agent"
         volumeMounts:

addons/cluo/update-operator.yaml

@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: container-linux-update-operator
-  namespace: kube-system
+  namespace: reboot-coordinator
 spec:
   replicas: 1
   template:
@@ -12,12 +12,15 @@ spec:
     spec:
       containers:
       - name: update-operator
-        image: quay.io/coreos/container-linux-update-operator:v0.3.1
+        image: quay.io/coreos/container-linux-update-operator:v0.4.1
         command:
         - "/bin/update-operator"
-        - "--analytics=false"
         env:
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule

addons/heapster/deployment.yaml

@@ -3,27 +3,23 @@ kind: Deployment
 metadata:
   name: heapster
   namespace: kube-system
-  labels:
-    k8s-app: heapster
-    kubernetes.io/cluster-service: "true"
-    version: v1.4.0
 spec:
   replicas: 1
   selector:
     matchLabels:
-      k8s-app: heapster
-      version: v1.4.0
+      name: heapster
+      phase: prod
   template:
     metadata:
       labels:
-        k8s-app: heapster
-        version: v1.4.0
+        name: heapster
+        phase: prod
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
         - name: heapster
-          image: gcr.io/google_containers/heapster-amd64:v1.4.0
+          image: gcr.io/google_containers/heapster-amd64:v1.4.3
           command:
             - /heapster
             - --source=kubernetes.summary_api:''
@@ -42,7 +38,7 @@ spec:
             - --extra-cpu=0.5m
             - --memory=140Mi
             - --extra-memory=4Mi
-            - --deployment=heapster-v1.4.0
+            - --deployment=heapster
             - --container=heapster
             - --poll-period=300000
           env:

addons/heapster/service.yaml

@@ -3,13 +3,10 @@ kind: Service
 metadata: 
   name: heapster
   namespace: kube-system
-  labels: 
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "Heapster"
 spec: 
   type: ClusterIP
   selector:
-    k8s-app: heapster
+    name: heapster
   ports: 
     - port: 80
       targetPort: 8082

addons/nginx-ingress/aws/default-backend/deployment.yaml

@@ -16,7 +16,7 @@ spec:
           # Any image is permissable as long as:
           # 1. It serves a 404 page at /
           # 2. It serves 200 on a /healthz endpoint
-          image: gcr.io/google_containers/defaultbackend:1.0
+          image: gcr.io/google_containers/defaultbackend:1.4
           ports:
             - containerPort: 8080
           resources:

addons/nginx-ingress/aws/deployment.yaml

@@ -16,9 +16,10 @@ spec:
     spec:
       nodeSelector:
         node-role.kubernetes.io/node: ""
+      hostNetwork: true
       containers:
         - name: nginx-ingress-controller
-          image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11
+          image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.15
           args:
             - /nginx-ingress-controller
             - --default-backend-service=$(POD_NAMESPACE)/default-backend
@@ -43,19 +44,24 @@ spec:
             - name: health
               containerPort: 10254
               hostPort: 10254
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: 10254
-              scheme: HTTP
           livenessProbe:
-            initialDelaySeconds: 11
-            timeoutSeconds: 1
+            failureThreshold: 3
             httpGet:
               path: /healthz
               port: 10254
               scheme: HTTP
-      hostNetwork: true
-      dnsPolicy: ClusterFirst
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
       restartPolicy: Always
       terminationGracePeriodSeconds: 60

addons/nginx-ingress/digital-ocean/daemonset.yaml

@@ -16,9 +16,10 @@ spec:
     spec:
       nodeSelector:
         node-role.kubernetes.io/node: ""
+      hostNetwork: true
       containers:
         - name: nginx-ingress-controller
-          image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11
+          image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.15
           args:
             - /nginx-ingress-controller
             - --default-backend-service=$(POD_NAMESPACE)/default-backend
@@ -43,19 +44,24 @@ spec:
             - name: health
               containerPort: 10254
               hostPort: 10254
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: 10254
-              scheme: HTTP
           livenessProbe:
-            initialDelaySeconds: 10
-            timeoutSeconds: 1
+            failureThreshold: 3
             httpGet:
               path: /healthz
               port: 10254
               scheme: HTTP
-      hostNetwork: true
-      dnsPolicy: ClusterFirst
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
       restartPolicy: Always
       terminationGracePeriodSeconds: 60

addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml

@@ -16,7 +16,7 @@ spec:
           # Any image is permissable as long as:
           # 1. It serves a 404 page at /
           # 2. It serves 200 on a /healthz endpoint
-          image: gcr.io/google_containers/defaultbackend:1.0
+          image: gcr.io/google_containers/defaultbackend:1.4
           ports:
             - containerPort: 8080
           resources:

addons/nginx-ingress/google-cloud/default-backend/deployment.yaml

@@ -16,7 +16,7 @@ spec:
           # Any image is permissable as long as:
           # 1. It serves a 404 page at /
           # 2. It serves 200 on a /healthz endpoint
-          image: gcr.io/google_containers/defaultbackend:1.0
+          image: gcr.io/google_containers/defaultbackend:1.4
           ports:
             - containerPort: 8080
           resources:

addons/nginx-ingress/google-cloud/deployment.yaml

@@ -16,9 +16,10 @@ spec:
     spec:
       nodeSelector:
         node-role.kubernetes.io/node: ""
+      hostNetwork: true
       containers:
         - name: nginx-ingress-controller
-          image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11
+          image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.15
           args:
             - /nginx-ingress-controller
             - --default-backend-service=$(POD_NAMESPACE)/default-backend
@@ -43,19 +44,24 @@ spec:
             - name: health
               containerPort: 10254
               hostPort: 10254
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: 10254
-              scheme: HTTP
           livenessProbe:
-            initialDelaySeconds: 10
-            timeoutSeconds: 1
+            failureThreshold: 3
             httpGet:
               path: /healthz
               port: 10254
               scheme: HTTP
-      hostNetwork: true
-      dnsPolicy: ClusterFirst
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
       restartPolicy: Always
       terminationGracePeriodSeconds: 60

addons/prometheus/config.yaml

@@ -0,0 +1,216 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  namespace: monitoring
+data:
+  prometheus.yaml: |-
+    # Global config
+    global:
+      scrape_interval: 15s
+    # Scrape configs for running Prometheus on a Kubernetes cluster.
+    # This uses separate scrape configs for cluster components (i.e. API server, node)
+    # and services to allow each to use different authentication configs.
+    #
+    # Kubernetes labels will be added as Prometheus labels on metrics via the
+    # `labelmap` relabeling action.
+    scrape_configs:
+
+    # Scrape config for API servers.
+    #
+    # Kubernetes exposes API servers as endpoints to the default/kubernetes
+    # service so this uses `endpoints` role and uses relabelling to only keep
+    # the endpoints associated with the default/kubernetes service using the
+    # default named port `https`. This works for single API server deployments as
+    # well as HA API server deployments.
+    - job_name: 'kubernetes-apiservers'
+      kubernetes_sd_configs:
+      - role: endpoints
+      
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        # Using endpoints to discover kube-apiserver targets finds the pod IP
+        # (host IP since apiserver is uses host network) which is not used in
+        # the server certificate.
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+      # Keep only the default/kubernetes service endpoints for the https port. This
+      # will add targets for each API server which Kubernetes adds an endpoint to
+      # the default/kubernetes service.
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: default;kubernetes;https
+
+    # Scrape config for node (i.e. kubelet) /metrics (e.g. 'kubelet_'). Explore
+    # metrics from a node by scraping kubelet (127.0.0.1:10255/metrics).
+    #
+    # Rather than connecting directly to the node, the scrape is proxied though the
+    # Kubernetes apiserver.  This means it will work if Prometheus is running out of
+    # cluster, or can't connect to nodes for some other reason (e.g. because of
+    # firewalling).
+    - job_name: 'kubernetes-nodes'
+      kubernetes_sd_configs:
+      - role: node
+      
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+      relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics
+
+    # Scrape config for Kubelet cAdvisor. Explore metrics from a node by
+    # scraping kubelet (127.0.0.1:10255/metrics/cadvisor).
+    #
+    # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics
+    # (those whose names begin with 'container_') have been removed from the
+    # Kubelet metrics endpoint.  This job scrapes the cAdvisor endpoint to
+    # retrieve those metrics.
+    #
+    # Rather than connecting directly to the node, the scrape is proxied though the
+    # Kubernetes apiserver.  This means it will work if Prometheus is running out of
+    # cluster, or can't connect to nodes for some other reason (e.g. because of
+    # firewalling).
+    - job_name: 'kubernetes-cadvisor'
+      kubernetes_sd_configs:
+      - role: node
+      
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+      relabel_configs:
+      - action: labelmap
+        regex: __meta_kubernetes_node_label_(.+)
+      - target_label: __address__
+        replacement: kubernetes.default.svc:443
+      - source_labels: [__meta_kubernetes_node_name]
+        regex: (.+)
+        target_label: __metrics_path__
+        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+    
+    # Scrape config for service endpoints.
+    #
+    # The relabeling allows the actual service scrape endpoint to be configured
+    # via the following annotations:
+    #
+    # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
+    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
+    # to set this to `https` & most likely set the `tls_config` of the scrape config.
+    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+    # * `prometheus.io/port`: If the metrics are exposed on a different port to the
+    # service then set this appropriately.
+    - job_name: 'kubernetes-service-endpoints'
+
+      kubernetes_sd_configs:
+      - role: endpoints
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+        action: replace
+        target_label: __scheme__
+        regex: (https?)
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+        action: replace
+        target_label: __address__
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        action: replace
+        target_label: kubernetes_name
+
+    # Example scrape config for probing services via the Blackbox Exporter.
+    #
+    # The relabeling allows the actual service scrape endpoint to be configured
+    # via the following annotations:
+    #
+    # * `prometheus.io/probe`: Only probe services that have a value of `true`
+    - job_name: 'kubernetes-services'
+
+      metrics_path: /probe
+      params:
+        module: [http_2xx]
+
+      kubernetes_sd_configs:
+      - role: service
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
+        action: keep
+        regex: true
+      - source_labels: [__address__]
+        target_label: __param_target
+      - target_label: __address__
+        replacement: blackbox
+      - source_labels: [__param_target]
+        target_label: instance
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        target_label: kubernetes_name
+
+    # Example scrape config for pods
+    #
+    # The relabeling allows the actual pod scrape endpoint to be configured via the
+    # following annotations:
+    #
+    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
+    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
+    # pod's declared ports (default is a port-free target if none are declared).
+    - job_name: 'kubernetes-pods'
+
+      kubernetes_sd_configs:
+      - role: pod
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: kubernetes_pod_name
+
+    # Rule files
+    rule_files:
+      - "/etc/prometheus/rules/*.rules"

addons/prometheus/deployment.yaml

@@ -0,0 +1,45 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: monitoring
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        name: prometheus
+        phase: prod
+    spec:
+      containers:
+      - name: prometheus
+        image: quay.io/prometheus/prometheus:v1.8.0
+        args:
+          - '-config.file=/etc/prometheus/prometheus.yaml'
+          - '-storage.local.retention=12h'
+          - '-storage.local.memory-chunks=500000'
+        ports:
+        - name: web
+          containerPort: 9090
+        volumeMounts:
+        - name: config
+          mountPath: /etc/prometheus
+        - name: rules
+          mountPath: /etc/prometheus/rules
+        - name: data
+          mountPath: /var/lib/prometheus
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: config
+        configMap:
+          name: prometheus-config
+      - name: rules
+        configMap:
+          name: prometheus-rules
+      - name: data
+        emptyDir: {}

addons/prometheus/exporters/kube-state-metrics/cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+- kind: ServiceAccount
+  name: kube-state-metrics
+  namespace: monitoring

addons/prometheus/exporters/kube-state-metrics/cluster-role.yaml

@@ -0,0 +1,31 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - pods
+  - services
+  - resourcequotas
+  - replicationcontrollers
+  - limitranges
+  - persistentvolumeclaims
+  - namespaces
+  verbs: ["list", "watch"]
+- apiGroups: ["extensions"]
+  resources:
+  - daemonsets
+  - deployments
+  - replicasets
+  verbs: ["list", "watch"]
+- apiGroups: ["apps"]
+  resources:
+  - statefulsets
+  verbs: ["list", "watch"]
+- apiGroups: ["batch"]
+  resources:
+  - cronjobs
+  - jobs
+  verbs: ["list", "watch"]

addons/prometheus/exporters/kube-state-metrics/deployment.yaml

@@ -0,0 +1,67 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  replicas: 1
+  strategy:
+    rollingUpdate:
+      maxUnavilable: 1
+  selector:
+    matchLabels:
+      name: kube-state-metrics
+      phase: prod
+  template:
+    metadata:
+      labels:
+        name: kube-state-metrics
+        phase: prod
+    spec:
+      serviceAccountName: kube-state-metrics
+      containers:
+      - name: kube-state-metrics
+        image: quay.io/coreos/kube-state-metrics:v1.1.0
+        ports:
+          - name: metrics
+            containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          requests:
+            memory: 100Mi
+            cpu: 100m
+          limits:
+            memory: 200Mi
+            cpu: 200m
+      - name: addon-resizer
+        image: gcr.io/google_containers/addon-resizer:1.0
+        resources:
+          limits:
+            cpu: 100m
+            memory: 30Mi
+          requests:
+            cpu: 100m
+            memory: 30Mi
+        env:
+          - name: MY_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: MY_POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        command:
+          - /pod_nanny
+          - --container=kube-state-metrics
+          - --cpu=100m
+          - --extra-cpu=1m
+          - --memory=100Mi
+          - --extra-memory=2Mi
+          - --threshold=5
+          - --deployment=kube-state-metrics

addons/prometheus/exporters/kube-state-metrics/resizer-role-binding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kube-state-metrics-resizer
+subjects:
+- kind: ServiceAccount
+  name: kube-state-metrics
+  namespace: monitoring

addons/prometheus/exporters/kube-state-metrics/resizer-role.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kube-state-metrics-resizer
+  namespace: monitoring
+rules:
+- apiGroups: [""]
+  resources:
+  - pods
+  verbs: ["get"]
+- apiGroups: ["extensions"]
+  resources:
+  - deployments
+  resourceNames: ["kube-state-metrics"]
+  verbs: ["get", "update"]

addons/prometheus/exporters/kube-state-metrics/service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring

addons/prometheus/exporters/kube-state-metrics/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+  annotations:
+    prometheus.io/scrape: 'true'
+spec:
+  type: ClusterIP
+  selector:
+    name: kube-state-metrics
+    phase: prod
+  ports:
+    - name: metrics
+      protocol: TCP
+      port: 80
+      targetPort: 8080

addons/prometheus/exporters/node-exporter/daemonset.yaml

@@ -0,0 +1,57 @@
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  name: node-exporter
+  namespace: monitoring
+spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      name: node-exporter
+      phase: prod
+  template:
+    metadata:
+      labels:
+        name: node-exporter
+        phase: prod
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+      - name: node-exporter
+        image: quay.io/prometheus/node-exporter:v0.15.0
+        args:
+          - "--path.procfs=/host/proc"
+          - "--path.sysfs=/host/sys"
+        ports:
+          - name: metrics
+            containerPort: 9100
+            hostPort: 9100
+        resources:
+          requests:
+            memory: 30Mi
+            cpu: 100m
+          limits:
+            memory: 50Mi
+            cpu: 200m
+        volumeMounts:
+          - name: proc
+            mountPath: /host/proc
+            readOnly:  true
+          - name: sys
+            mountPath: /host/sys
+            readOnly: true
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      volumes:
+        - name: proc
+          hostPath:
+            path: /proc
+        - name: sys
+          hostPath:
+            path: /sys

addons/prometheus/exporters/node-exporter/service.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: node-exporter
+  namespace: monitoring
+  annotations:
+    prometheus.io/scrape: 'true'
+spec:
+  type: ClusterIP
+  # service is created to allow prometheus to scape endpoints
+  clusterIP: None
+  selector:
+    name: node-exporter
+    phase: prod
+  ports:
+    - name: metrics
+      protocol: TCP
+      port: 80
+      targetPort: 9100

addons/prometheus/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring

addons/prometheus/rbac/cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: monitoring

addons/prometheus/rbac/cluster-role.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - nodes/proxy
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]

addons/prometheus/rules.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-rules
+  namespace: monitoring
+data:
+  example.rule: |
+    job_service:rpc_durations_seconds_count:avg_rate5m = avg(rate(rpc_durations_seconds_count[5m])) by (job, service)

addons/prometheus/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: monitoring
+spec:
+  type: ClusterIP
+  selector:
+    name: prometheus
+    phase: prod
+  ports:
+    - name: web
+      protocol: TCP
+      port: 80
+      targetPort: 9090

aws/container-linux/kubernetes/README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features
 
-* Kubernetes v1.7.7 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.8.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Ready for Ingress, Dashboards, Metrics, and other optional [addons](https://typhoon.psdn.io/addons/overview/)

aws/container-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.7.0"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.8.1"
 
   cluster_name                  = "${var.cluster_name}"
   api_servers                   = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

aws/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -34,7 +34,8 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -104,8 +105,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -128,7 +129,7 @@ storage:
           [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
           [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
           BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
-          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.7.0}"
+          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.8.1}"
           BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
           exec /usr/bin/rkt run \
             --trust-keys-from-https \

aws/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -34,7 +34,8 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -102,8 +103,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -120,7 +121,8 @@ storage:
             --trust-keys-from-https \
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
-            quay.io/coreos/hyperkube:v1.7.7_coreos.0 \
+            --insecure-options=image \
+            docker://gcr.io/google_containers/hyperkube:v1.8.2 \
             --net=host \
             --dns=host \
             --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

aws/container-linux/kubernetes/outputs.tf

@@ -1,5 +1,4 @@
 output "ingress_dns_name" {
-  value = "${aws_elb.ingress.dns_name}"
+  value       = "${aws_elb.ingress.dns_name}"
   description = "DNS name of the ELB for distributing traffic to Ingress controllers"
 }
-

aws/container-linux/kubernetes/require.tf

@@ -0,0 +1,25 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.10.4"
+}
+
+provider "aws" {
+  version = "~> 1.0"
+}
+
+provider "local" {
+  version = "~> 1.0"
+}
+
+provider "null" {
+  version = "~> 1.0"
+}
+
+provider "template" {
+  version = "~> 1.0"
+}
+
+provider "tls" {
+  version = "~> 1.0"
+}

bare-metal/container-linux/kubernetes/README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features
 
-* Kubernetes v1.7.7 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.8.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Ready for Ingress, Dashboards, Metrics, and other optional [addons](https://typhoon.psdn.io/addons/overview/)

bare-metal/container-linux/kubernetes/bootkube.tf

@@ -1,13 +1,13 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.7.0"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.8.1"
 
-  cluster_name                  = "${var.cluster_name}"
-  api_servers                   = ["${var.k8s_domain_name}"]
-  etcd_servers                  = ["${var.controller_domains}"]
-  asset_dir                     = "${var.asset_dir}"
-  networking                    = "${var.networking}"
-  network_mtu                   = "${var.network_mtu}"
-  pod_cidr                      = "${var.pod_cidr}"
-  service_cidr                  = "${var.service_cidr}"
+  cluster_name = "${var.cluster_name}"
+  api_servers  = ["${var.k8s_domain_name}"]
+  etcd_servers = ["${var.controller_domains}"]
+  asset_dir    = "${var.asset_dir}"
+  networking   = "${var.networking}"
+  network_mtu  = "${var.network_mtu}"
+  pod_cidr     = "${var.pod_cidr}"
+  service_cidr = "${var.service_cidr}"
 }

bare-metal/container-linux/kubernetes/cl/container-linux-install.yaml.tmpl

@@ -32,6 +32,11 @@ storage:
           systemctl reboot
 passwd:
   users:
-    - name: core
+    # Avoid using standard name "core" so terraform apply cannot SSH until post-install.
+    - name: debug
+      create:
+        groups:
+          - sudo
+          - docker
       ssh_authorized_keys:
         - {{.ssh_authorized_key}}

bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -65,7 +65,8 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -107,30 +108,14 @@ systemd:
         ExecStart=/opt/bootkube/bootkube-start
         ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done
 storage:
-  {{ if index . "pxe" }}
-  disks:
-    - device: /dev/sda
-      wipe_table: true
-      partitions:
-        - label: ROOT
-  filesystems:
-    - name: root
-      mount:
-        device: "/dev/sda1"
-        format: "ext4"
-        create:
-          force: true
-          options:
-            - "-LROOT"
-  {{end}}
   files:
     - path: /etc/kubernetes/kubelet.env
       filesystem: root
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/hostname
       filesystem: root
       mode: 0644
@@ -159,7 +144,7 @@ storage:
           [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
           [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
           BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
-          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.7.0}"
+          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.8.1}"
           BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
           exec /usr/bin/rkt run \
             --trust-keys-from-https \
@@ -172,6 +157,8 @@ storage:
             --net=host \
             --dns=host \
             --exec=/bootkube -- start --asset-dir=/assets "$@"
+networkd:
+  ${networkd_content}
 passwd:
   users:
     - name: core

bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -42,7 +42,8 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -73,30 +74,14 @@ systemd:
         WantedBy=multi-user.target
 
 storage:
-  {{ if index . "pxe" }}
-  disks:
-    - device: /dev/sda
-      wipe_table: true
-      partitions:
-        - label: ROOT
-  filesystems:
-    - name: root
-      mount:
-        device: "/dev/sda1"
-        format: "ext4"
-        create:
-          force: true
-          options:
-            - "-LROOT"
-  {{end}}
   files:
     - path: /etc/kubernetes/kubelet.env
       filesystem: root
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/hostname
       filesystem: root
       mode: 0644
@@ -108,6 +93,8 @@ storage:
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
+networkd:
+  ${networkd_content}
 passwd:
   users:
     - name: core

bare-metal/container-linux/kubernetes/profiles.tf

@@ -12,6 +12,7 @@ resource "matchbox_profile" "container-linux-install" {
     "coreos.first_boot=yes",
     "console=tty0",
     "console=ttyS0",
+    "${var.kernel_args}",
   ]
 
   container_linux_config = "${data.template_file.container-linux-install-config.rendered}"
@@ -47,6 +48,7 @@ resource "matchbox_profile" "cached-container-linux-install" {
     "coreos.first_boot=yes",
     "console=tty0",
     "console=ttyS0",
+    "${var.kernel_args}",
   ]
 
   container_linux_config = "${data.template_file.cached-container-linux-install-config.rendered}"
@@ -85,6 +87,9 @@ data "template_file" "controller-configs" {
     etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}"
     k8s_dns_service_ip   = "${module.bootkube.kube_dns_service_ip}"
     ssh_authorized_key   = "${var.ssh_authorized_key}"
+
+    # Terraform evaluates both sides regardless and element cannot be used on 0 length lists
+    networkd_content = "${length(var.controller_networkds) == 0 ? "" : element(concat(var.controller_networkds, list("")), count.index)}"
   }
 }
 
@@ -104,5 +109,8 @@ data "template_file" "worker-configs" {
     domain_name        = "${element(var.worker_domains, count.index)}"
     k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
     ssh_authorized_key = "${var.ssh_authorized_key}"
+
+    # Terraform evaluates both sides regardless and element cannot be used on 0 length lists
+    networkd_content = "${length(var.worker_networkds) == 0 ? "" : element(concat(var.worker_networkds, list("")), count.index)}"
   }
 }

bare-metal/container-linux/kubernetes/require.tf

@@ -0,0 +1,21 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.10.4"
+}
+
+provider "local" {
+  version = "~> 1.0"
+}
+
+provider "null" {
+  version = "~> 1.0"
+}
+
+provider "template" {
+  version = "~> 1.0"
+}
+
+provider "tls" {
+  version = "~> 1.0"
+}

bare-metal/container-linux/kubernetes/ssh.tf

@@ -1,10 +1,10 @@
-# Secure copy etcd TLS assets and kubeconfig to all nodes. Activates kubelet.service
-resource "null_resource" "copy-secrets" {
-  count = "${length(var.controller_names) + length(var.worker_names)}"
+# Secure copy etcd TLS assets and kubeconfig to controllers. Activates kubelet.service
+resource "null_resource" "copy-etcd-secrets" {
+  count = "${length(var.controller_names)}"
 
   connection {
     type    = "ssh"
-    host    = "${element(concat(var.controller_domains, var.worker_domains), count.index)}"
+    host    = "${element(var.controller_domains, count.index)}"
     user    = "core"
     timeout = "60m"
   }
@@ -66,19 +66,42 @@ resource "null_resource" "copy-secrets" {
   }
 }
 
+# Secure copy kubeconfig to all workers. Activates kubelet.service
+resource "null_resource" "copy-kubeconfig" {
+  count = "${length(var.worker_names)}"
+
+  connection {
+    type    = "ssh"
+    host    = "${element(var.worker_domains, count.index)}"
+    user    = "core"
+    timeout = "60m"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.kubeconfig}"
+    destination = "$HOME/kubeconfig"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
+    ]
+  }
+}
+
 # Secure copy bootkube assets to ONE controller and start bootkube to perform
 # one-time self-hosted cluster bootstrapping.
 resource "null_resource" "bootkube-start" {
   # Without depends_on, this remote-exec may start before the kubeconfig copy.
   # Terraform only does one task at a time, so it would try to bootstrap
   # while no Kubelets are running.
-  depends_on = ["null_resource.copy-secrets"]
+  depends_on = ["null_resource.copy-etcd-secrets", "null_resource.copy-kubeconfig"]
 
   connection {
     type    = "ssh"
     host    = "${element(var.controller_domains, 0)}"
     user    = "core"
-    timeout = "60m"
+    timeout = "30m"
   }
 
   provisioner "file" {

bare-metal/container-linux/kubernetes/variables.tf

@@ -109,3 +109,23 @@ variable "container_linux_oem" {
   default     = ""
   description = "Specify an OEM image id to use as base for the installation (e.g. ami, vmware_raw, xen) or leave blank for the default image"
 }
+
+variable "kernel_args" {
+  description = "Additional kernel arguments to provide at PXE boot."
+  type        = "list"
+  default     = []
+}
+
+# unofficial, undocumented, unsupported, temporary
+
+variable "controller_networkds" {
+  type        = "list"
+  description = "Controller Container Linux config networkd section"
+  default     = []
+}
+
+variable "worker_networkds" {
+  type        = "list"
+  description = "Worker Container Linux config networkd section"
+  default     = []
+}

bare-metal/container-linux/pxe-worker/cl/bootkube-worker.yaml.tmpl

@@ -42,7 +42,8 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -95,8 +96,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/hostname
       filesystem: root
       mode: 0644

bare-metal/container-linux/pxe-worker/profiles.tf

@@ -8,12 +8,11 @@ resource "matchbox_profile" "bootkube-worker-pxe" {
   ]
 
   args = [
-    "root=/dev/sda1",
     "coreos.config.url=${var.matchbox_http_endpoint}/ignition?uuid=$${uuid}&mac=$${mac:hexhyp}",
     "coreos.first_boot=yes",
     "console=tty0",
     "console=ttyS0",
-    "kvm-intel.nested=1",
+    "${var.kernel_args}",
   ]
 
   container_linux_config = "${file("${path.module}/cl/bootkube-worker.yaml.tmpl")}"

bare-metal/container-linux/pxe-worker/variables.tf

@@ -53,3 +53,14 @@ variable "kube_dns_service_ip" {
   type        = "string"
   default     = "10.3.0.10"
 }
+
+# optional
+
+variable "kernel_args" {
+  description = "Additional kernel arguments to provide at PXE boot."
+  type        = "list"
+
+  default = [
+    "root=/dev/sda1",
+  ]
+}

digital-ocean/container-linux/kubernetes/README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features
 
-* Kubernetes v1.7.7 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.8.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Ready for Ingress, Dashboards, Metrics, and other optional [addons](https://typhoon.psdn.io/addons/overview/)

digital-ocean/container-linux/kubernetes/bootkube.tf

@@ -1,14 +1,13 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.7.0"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.8.1"
 
-  cluster_name                  = "${var.cluster_name}"
-  api_servers                   = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
-  etcd_servers                  = ["http://127.0.0.1:2379"]
-  asset_dir                     = "${var.asset_dir}"
-  networking                    = "${var.networking}"
-  network_mtu                   = 1440
-  pod_cidr                      = "${var.pod_cidr}"
-  service_cidr                  = "${var.service_cidr}"
-  experimental_self_hosted_etcd = "true"
+  cluster_name = "${var.cluster_name}"
+  api_servers  = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
+  etcd_servers = "${digitalocean_record.etcds.*.fqdn}"
+  asset_dir    = "${var.asset_dir}"
+  networking   = "${var.networking}"
+  network_mtu  = 1440
+  pod_cidr     = "${var.pod_cidr}"
+  service_cidr = "${var.service_cidr}"
 }

digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -1,6 +1,29 @@
 ---
 systemd:
   units:
+    - name: etcd-member.service
+      enable: true
+      dropins:
+        - name: 40-etcd-cluster.conf
+          contents: |
+            [Service]
+            Environment="ETCD_IMAGE_TAG=v3.2.0"
+            Environment="ETCD_NAME=${etcd_name}"
+            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
+            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
+            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
+            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
+            Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
+            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
+            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
+            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
+            Environment="ETCD_CLIENT_CERT_AUTH=true"
+            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
+            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
+            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
+            Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
     - name: docker.service
       enable: true
     - name: locksmithd.service
@@ -45,7 +68,8 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -95,8 +119,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -119,7 +143,7 @@ storage:
           [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
           [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
           BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
-          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.7.0}"
+          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.8.1}"
           BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
           exec /usr/bin/rkt run \
             --trust-keys-from-https \

digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -45,7 +45,8 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -93,8 +94,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -111,7 +112,8 @@ storage:
             --trust-keys-from-https \
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
-            quay.io/coreos/hyperkube:v1.7.7_coreos.0 \
+            --insecure-options=image \
+            docker://gcr.io/google_containers/hyperkube:v1.8.2 \
             --net=host \
             --dns=host \
             --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

digital-ocean/container-linux/kubernetes/controllers.tf

@@ -14,6 +14,21 @@ resource "digitalocean_record" "controllers" {
   value = "${element(digitalocean_droplet.controllers.*.ipv4_address, count.index)}"
 }
 
+resource "digitalocean_record" "etcds" {
+  count = "${var.controller_count}"
+
+  # DNS zone where record should be created
+  domain = "${var.dns_zone}"
+
+  # DNS record (will be prepended to domain)
+  name = "${var.cluster_name}-etcd${count.index}"
+  type = "A"
+  ttl  = 300
+
+  # IPv4 addresses of controllers
+  value = "${element(digitalocean_droplet.controllers.*.ipv4_address_private, count.index)}"
+}
+
 # Controller droplet instances
 resource "digitalocean_droplet" "controllers" {
   count = "${var.controller_count}"
@@ -28,7 +43,7 @@ resource "digitalocean_droplet" "controllers" {
   ipv6               = true
   private_networking = true
 
-  user_data = "${data.ct_config.controller_ign.rendered}"
+  user_data = "${element(data.ct_config.controller_ign.*.rendered, count.index)}"
   ssh_keys  = "${var.ssh_fingerprints}"
 
   tags = [
@@ -43,15 +58,34 @@ resource "digitalocean_tag" "controllers" {
 
 # Controller Container Linux Config
 data "template_file" "controller_config" {
+  count = "${var.controller_count}"
+
   template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
 
   vars = {
-    k8s_dns_service_ip  = "${cidrhost(var.service_cidr, 10)}"
-    k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
+    # Cannot use cyclic dependencies on controllers or their DNS records
+    etcd_name   = "etcd${count.index}"
+    etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
+
+    # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
+    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
+    k8s_dns_service_ip   = "${cidrhost(var.service_cidr, 10)}"
+  }
+}
+
+# Horrible hack to generate a Terraform list of a desired length without dependencies.
+# Ideal ${repeat("etcd", 3) -> ["etcd", "etcd", "etcd"]}
+resource null_resource "repeat" {
+  count = "${var.controller_count}"
+
+  triggers {
+    name   = "etcd${count.index}"
+    domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
   }
 }
 
 data "ct_config" "controller_ign" {
-  content      = "${data.template_file.controller_config.rendered}"
+  count        = "${var.controller_count}"
+  content      = "${element(data.template_file.controller_config.*.rendered, count.index)}"
   pretty_print = false
 }

digital-ocean/container-linux/kubernetes/require.tf

@@ -0,0 +1,25 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.10.4"
+}
+
+provider "digitalocean" {
+  version = "0.1.2"
+}
+
+provider "local" {
+  version = "~> 1.0"
+}
+
+provider "null" {
+  version = "~> 1.0"
+}
+
+provider "template" {
+  version = "~> 1.0"
+}
+
+provider "tls" {
+  version = "~> 1.0"
+}

digital-ocean/container-linux/kubernetes/ssh.tf

@@ -14,8 +14,53 @@ resource "null_resource" "copy-secrets" {
     destination = "$HOME/kubeconfig"
   }
 
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_ca_cert}"
+    destination = "$HOME/etcd-client-ca.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_client_cert}"
+    destination = "$HOME/etcd-client.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_client_key}"
+    destination = "$HOME/etcd-client.key"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_server_cert}"
+    destination = "$HOME/etcd-server.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_server_key}"
+    destination = "$HOME/etcd-server.key"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_peer_cert}"
+    destination = "$HOME/etcd-peer.crt"
+  }
+
+  provisioner "file" {
+    content     = "${module.bootkube.etcd_peer_key}"
+    destination = "$HOME/etcd-peer.key"
+  }
+
   provisioner "remote-exec" {
     inline = [
+      "sudo mkdir -p /etc/ssl/etcd/etcd",
+      "sudo mv etcd-client* /etc/ssl/etcd/",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
+      "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
+      "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
+      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
+      "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
+      "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
+      "sudo chown -R etcd:etcd /etc/ssl/etcd",
+      "sudo chmod -R 500 /etc/ssl/etcd",
       "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
     ]
   }

docs/addons/ingress.md

@@ -9,7 +9,7 @@ On AWS, an elastic load balancer distributes traffic across worker nodes (i.e. a
 Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace.
 
 ```
-kubectl apply -R addons/nginx-ingress/aws
+kubectl apply -R -f addons/nginx-ingress/aws
 ```
 
 For each application, add a DNS CNAME resolving to the ELB's DNS record.
@@ -42,7 +42,7 @@ On Digital Ocean, a DNS A record (e.g. `nemo-workers.example.com`) resolves to e
 Create the Ingress controller daemonset, service, RBAC roles, RBAC bindings, default backend, and namespace.
 
 ```
-kubectl apply -R addons/nginx-ingress/digital-ocean
+kubectl apply -R -f addons/nginx-ingress/digital-ocean
 ```
 
 For each application, add a CNAME record resolving to the worker(s) DNS record. Use the Typhoon module's output `workers_dns` to find the worker DNS value. For example, you might use Terraform to manage a Google Cloud DNS record:
@@ -69,7 +69,7 @@ On Google Cloud, a network load balancer distributes traffic across worker nodes
 Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace.
 
 ```
-kubectl apply -R addons/nginx-ingress/google-cloud
+kubectl apply -R -f addons/nginx-ingress/google-cloud
 ```
 
 For each application, add a DNS record resolving to the network load balancer's IPv4 address.

docs/addons/overview.md

@@ -2,10 +2,10 @@
 
 Every Typhoon cluster is verified to work well with several post-install addons.
 
+* [CLUO](cluo.md) (Container Linux only)
 * Nginx [Ingress Controller](ingress.md)
 * [Heapster](heapster.md)
 * Kubernetes [Dashboard](dashboard.md)
-* [CLUO](cluo.md) (Container Linux only)
-* Prometheus
+* [Prometheus](prometheus.md)
 * Grafana
 

docs/addons/prometheus.md

@@ -0,0 +1,50 @@
+# Prometheus
+
+Prometheus collects metrics (e.g. `node_memory_usage_bytes`) from *targets* by scraping their HTTP metrics endpoints. Targets are organized into *jobs*, defined in the Prometheus config. Targets may expose counter, gauge, histogram, or summary metrics.
+
+Here's a simple config from the Prometheus [tutorial](https://prometheus.io/docs/introduction/getting_started/).
+
+```
+global:
+  scrape_interval: 15s
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:9090']
+```
+
+On Kubernetes clusters, Prometheus is run as a Deployment, configured with a ConfigMap, and accessed via a Service or Ingress.
+
+```
+kubectl apply -f addons/prometheus -R
+```
+
+The ConfigMap configures Prometheus to target apiserver endpoints, node metrics, cAdvisor metrics, and exporters. By default, data is kept in an `emptyDir` so it is persisted until the pod is rescheduled.
+
+### Exporters
+
+Exporters expose metrics for 3rd-party systems that don't natively expose Prometheus metrics.
+
+* [node_exporter](https://github.com/prometheus/node_exporter) - DaemonSet that exposes a machine's hardware and OS metrics
+* [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) - Deployment that exposes Kubernetes object metrics
+* [blackbox_exporter](https://github.com/prometheus/blackbox_exporter) - Scrapes HTTP, HTTPS, DNS, TCP, or ICMP endpoints and exposes availability as metrics
+
+### Queries and Graphs
+
+Prometheus provides a simplistic UI for querying and graphing metrics. Use `kubectl` to authenticate to the apiserver and create a local port-forward to the Prometheus pod.
+
+```
+kubectl get pods -n monitoring
+kubectl port-forward prometheus-POD-ID 9090 -n monitoring
+```
+
+Visit [127.0.0.1:9090](http://127.0.0.1:9090) to query [expressions](http://127.0.0.1:9090/graph), view [targets](http://127.0.0.1:9090/targets), or check [alerts](http://127.0.0.1:9090/alerts).
+
+![Prometheus Graph](/img/prometheus-graph.png)
+<br/>
+![Prometheus Targets](/img/prometheus-targets.png)
+
+### Visualization
+
+Grafana can be used to build dashboards and rich visualizations that use Prometheus as the datasource. Favor Grafana for these use cases and use the Prometheus for debugging or quickly checking available metrics.

docs/aws.md

@@ -1,6 +1,6 @@
 # AWS
 
-In this tutorial, we'll create a Kubernetes v1.7.7 cluster on AWS.
+In this tutorial, we'll create a Kubernetes v1.8.2 cluster on AWS.
 
 We'll declare a Kubernetes cluster in Terraform using the Typhoon Terraform module. On apply, a VPC, gateway, subnets, auto-scaling groups of controllers and workers, network load balancers for controllers and workers, and security groups will be created.
 
@@ -16,7 +16,7 @@ Controllers and workers are provisioned to run a `kubelet`. A one-time [bootkube
 
 * AWS Account and IAM credentials
 * AWS Route53 DNS Zone (registered Domain Name or delegated subdomain)
-* Terraform v0.10.1+ and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally
+* Terraform v0.10.4+ and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally
 
 ## Terraform Setup
 
@@ -24,7 +24,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.10.1 on your sys
 
 ```sh
 $ terraform version
-Terraform v0.10.1
+Terraform v0.10.7
 ```
 
 Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system.
@@ -125,7 +125,7 @@ Get or update Terraform modules.
 $ terraform get            # downloads missing modules
 $ terraform get --update   # updates all modules
 Get: git::https://github.com/poseidon/typhoon (update)
-Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.7.0 (update)
+Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.8.1 (update)
 ```
 
 Plan the resources to be created.
@@ -160,9 +160,9 @@ In 10-20 minutes, the Kubernetes cluster will be ready.
 $ KUBECONFIG=/home/user/.secrets/clusters/tempest/auth/kubeconfig
 $ kubectl get nodes
 NAME             STATUS    AGE       VERSION        
-ip-10-0-12-221   Ready     34m       v1.7.7+coreos.0
-ip-10-0-19-112   Ready     34m       v1.7.7+coreos.0
-ip-10-0-4-22     Ready     34m       v1.7.7+coreos.0
+ip-10-0-12-221   Ready     34m       v1.8.2
+ip-10-0-19-112   Ready     34m       v1.8.2
+ip-10-0-4-22     Ready     34m       v1.8.2
 ```
 
 List the pods.

docs/bare-metal.md

@@ -1,6 +1,6 @@
 # Bare-Metal
 
-In this tutorial, we'll network boot and provison a Kubernetes v1.7.7 cluster on bare-metal.
+In this tutorial, we'll network boot and provison a Kubernetes v1.8.2 cluster on bare-metal.
 
 First, we'll deploy a [Matchbox](https://github.com/coreos/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster in Terraform using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers.
 
@@ -12,7 +12,7 @@ Controllers are provisioned as etcd peers and run `etcd-member` (etcd3) and `kub
 * PXE-enabled [network boot](https://coreos.com/matchbox/docs/latest/network-setup.html) environment
 * Matchbox v0.6+ deployment with API enabled
 * Matchbox credentials `client.crt`, `client.key`, `ca.crt`
-* Terraform v0.9.2+ and [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) installed locally
+* Terraform v0.10.4+ and [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) installed locally
 
 ## Machines
 
@@ -94,7 +94,7 @@ For networks already supporting iPXE clients, you can add a `default.ipxe` confi
 chain http://matchbox.foo:8080/boot.ipxe
 ```
 
-For networks with Ubiquiti Routers, you can [configure the router](TODO) itself to chainload machines to iPXE and Matchbox.
+For networks with Ubiquiti Routers, you can [configure the router](/topics/hardware.md#ubiquiti) itself to chainload machines to iPXE and Matchbox.
 
 For a small lab, you may wish to checkout the [quay.io/coreos/dnsmasq](https://quay.io/repository/coreos/dnsmasq) container image and [copy-paste examples](https://github.com/coreos/matchbox/blob/master/Documentation/network-setup.md#coreosdnsmasq).
 
@@ -113,7 +113,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.9.2+ on your sys
 
 ```sh
 $ terraform version
-Terraform v0.10.1
+Terraform v0.10.7
 ```
 
 Add the [terraform-provider-matchbox](https://github.com/coreos/terraform-provider-matchbox) plugin binary for your system.
@@ -124,19 +124,10 @@ tar xzf terraform-provider-matchbox-v0.2.2-linux-amd64.tar.gz
 sudo mv terraform-provider-matchbox-v0.2.2-linux-amd64/terraform-provider-matchbox /usr/local/bin/
 ```
 
-Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system.
-
-```sh
-wget https://github.com/coreos/terraform-provider-ct/releases/download/v0.2.0/terraform-provider-ct-v0.2.0-linux-amd64.tar.gz
-tar xzf terraform-provider-ct-v0.2.0-linux-amd64.tar.gz
-sudo mv terraform-provider-ct-v0.2.0-linux-amd64/terraform-provider-ct /usr/local/bin/
-```
-
-Add the plugins to your `~/.terraformrc`.
+Add the plugin to your `~/.terraformrc`.
 
 ```
 providers {
-  ct = "/usr/local/bin/terraform-provider-ct"
   matchbox = "/usr/local/bin/terraform-provider-matchbox"
 }
 ```
@@ -171,7 +162,7 @@ module "bare-metal-mercury" {
   # install
   matchbox_http_endpoint  = "http://matchbox.example.com"
   container_linux_channel = "stable"
-  container_linux_version = "1465.6.0"
+  container_linux_version = "1520.6.0"
   ssh_authorized_key      = "ssh-rsa AAAAB3Nz..."
 
   # cluster
@@ -228,7 +219,7 @@ Get or update Terraform modules.
 $ terraform get            # downloads missing modules
 $ terraform get --update   # updates all modules
 Get: git::https://github.com/poseidon/typhoon (update)
-Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.7.0 (update)
+Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.8.1 (update)
 ```
 
 Plan the resources to be created.
@@ -241,20 +232,15 @@ Plan: 55 to add, 0 to change, 0 to destroy.
 Apply the changes. Terraform will generate bootkube assets to `asset_dir` and create Matchbox profiles (e.g. controller, worker) and matching rules via the Matchbox API.
 
 ```sh
-module.bare-metal-mercury.null_resource.copy-secrets.0: Provisioning with 'file'...
-module.bare-metal-mercury.null_resource.copy-secrets.2: Provisioning with 'file'...
-module.bare-metal-mercury.null_resource.copy-secrets.1: Provisioning with 'file'...    
-module.bare-metal-mercury.null_resource.copy-secrets.0: Still creating... (10s elapsed)
-module.bare-metal-mercury.null_resource.copy-secrets.2: Still creating... (10s elapsed)
-module.bare-metal-mercury.null_resource.copy-secrets.1: Still creating... (10s elapsed)
+module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Provisioning with 'file'...
+module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Provisioning with 'file'...
+module.bare-metal-mercury.null_resource.copy-kubeconfig.0: Still creating... (10s elapsed)
+module.bare-metal-mercury.null_resource.copy-etcd-secrets.0: Still creating... (10s elapsed)
 ...
 ```
 
 Apply will then loop until it can successfully copy credentials to each machine and start the one-time Kubernetes bootstrap service. Proceed to the next step while this loops.
 
-!!! note ""
-    You may see `terraform apply` fail to `copy-secrets` if it connects before the disk install has completed. Run terraform apply until it reconciles successfully.
-
 ### Power
 
 Power on each machine with the boot device set to `pxe` for the next boot only.
@@ -304,9 +290,9 @@ bootkube[5]: Tearing down temporary bootstrap control plane...
 $ KUBECONFIG=/home/user/.secrets/clusters/mercury/auth/kubeconfig
 $ kubectl get nodes
 NAME                STATUS    AGE       VERSION
-node1.example.com   Ready     11m       v1.7.7+coreos.0
-node2.example.com   Ready     11m       v1.7.7+coreos.0
-node3.example.com   Ready     11m       v1.7.7+coreos.0
+node1.example.com   Ready     11m       v1.8.2
+node2.example.com   Ready     11m       v1.8.2
+node3.example.com   Ready     11m       v1.8.2
 ```
 
 List the pods.
@@ -369,4 +355,5 @@ Learn about [version pinning](concepts.md#versioning), maintenance, and [addons]
 | network_mtu | CNI interface MTU (calico-only) | 1480 | - | 
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assgin to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
+| kernel_args | Additional kernel args to provide at PXE boot | [] | "kvm-intel.nested=1" |
 

docs/concepts.md

@@ -60,7 +60,7 @@ Modules are updated regularly, set the version to a [release tag](https://github
 
 ```tf
 ...
-source = "git:https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.7.7"
+source = "git:https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=hash"
 ```
 
 Module versioning ensures `terraform get --update` only fetches the desired version, so plan and apply don't change cluster resources, unless the version is altered.
@@ -69,14 +69,15 @@ Module versioning ensures `terraform get --update` only fetches the desired vers
 
 Maintain Terraform configs for "live" infrastructure in a versioned repository. Seek to organize configs to reflect resources that should be managed together in a `terraform apply` invocation.
 
-You may choose to organize resources all together, by team, by project, or some other scheme. Here's an example that manages three clusters together:
+You may choose to organize resources all together, by team, by project, or some other scheme. Here's an example that manages four clusters together:
 
 ```sh
 .git/
 infra/
 └── terraform
     └── clusters
-        ├── bare-metal-tungsten.tf
+        ├── aws-tempest.tf
+        ├── bare-metal-mercury.tf
         ├── google-cloud-yavin.tf
         ├── digital-ocean-nemo.tf
         ├── providers.tf

docs/digital-ocean.md

@@ -1,16 +1,16 @@
 # Digital Ocean
 
-In this tutorial, we'll create a Kubernetes v1.7.7 cluster on Digital Ocean.
+In this tutorial, we'll create a Kubernetes v1.8.2 cluster on Digital Ocean.
 
 We'll declare a Kubernetes cluster in Terraform using the Typhoon Terraform module. On apply, firewall rules, DNS records, tags, and droplets for Kubernetes controllers and workers will be created.
 
-Controllers and workers are provisioned to run a `kubelet`. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules `etcd`, `apiserver`, `scheduler`, `controller-manager`, and `kube-dns` on controllers and runs `kube-proxy` and `flannel` on each node. A generated `kubeconfig` provides `kubectl` access to the cluster.
+Controllers and workers are provisioned to run a `kubelet`. A one-time [bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrap schedules an `apiserver`, `scheduler`, `controller-manager`, and `kube-dns` on controllers and runs `kube-proxy` and `flannel` on each node. A generated `kubeconfig` provides `kubectl` access to the cluster.
 
 ## Requirements
 
 * Digital Ocean Account and Token
 * Digital Ocean Domain (registered Domain Name or delegated subdomain)
-* Terraform v0.10.1+ and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally
+* Terraform v0.10.4+ and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally
 
 ## Terraform Setup
 
@@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.10.1+ on your sy
 
 ```sh
 $ terraform version
-Terraform v0.10.1
+Terraform v0.10.7
 ```
 
 Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system.
@@ -114,7 +114,7 @@ Get or update Terraform modules.
 $ terraform get            # downloads missing modules
 $ terraform get --update   # updates all modules
 Get: git::https://github.com/poseidon/typhoon (update)
-Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.7.0 (update)
+Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.8.1 (update)
 ```
 
 Plan the resources to be created.
@@ -137,7 +137,7 @@ module.digital-ocean-nemo.null_resource.bootkube-start: Creation complete (ID: 7
 Apply complete! Resources: 54 added, 0 changed, 0 destroyed.
 ```
 
-In 5-10 minutes, the Kubernetes cluster will be ready.
+In 3-6 minutes, the Kubernetes cluster will be ready.
 
 ## Verify
 
@@ -147,22 +147,19 @@ In 5-10 minutes, the Kubernetes cluster will be ready.
 $ KUBECONFIG=/home/user/.secrets/clusters/nemo/auth/kubeconfig
 $ kubectl get nodes
 NAME             STATUS    AGE       VERSION
-10.132.110.130   Ready     10m       v1.7.7+coreos.0
-10.132.115.81    Ready     10m       v1.7.7+coreos.0
-10.132.124.107   Ready     10m       v1.7.7+coreos.0
+10.132.110.130   Ready     10m       v1.8.2
+10.132.115.81    Ready     10m       v1.8.2
+10.132.124.107   Ready     10m       v1.8.2
 ```
 
 List the pods.
 
 ```
 NAMESPACE     NAME                                       READY     STATUS    RESTARTS   AGE
-kube-system   etcd-operator-3329263108-sgsbl             1/1       Running   1          11m
 kube-system   kube-apiserver-n10qr                       1/1       Running   0          11m
 kube-system   kube-controller-manager-3271970485-37gtw   1/1       Running   1          11m
 kube-system   kube-controller-manager-3271970485-p52t5   1/1       Running   0          11m
 kube-system   kube-dns-1187388186-ld1j7                  3/3       Running   0          11m
-kube-system   kube-etcd-0000                             1/1       Running   0          9m
-kube-system   kube-etcd-network-checkpointer-n9xsk       1/1       Running   0          11m
 kube-system   kube-flannel-1cq1v                         2/2       Running   0          11m
 kube-system   kube-flannel-hq9t0                         2/2       Running   1          11m
 kube-system   kube-flannel-v0g9w                         2/2       Running   0          11m

docs/faq.md

@@ -8,12 +8,12 @@ Formats rise and evolve. Typhoon may choose to adapt the format over time (with
 
 ## Self-hosted etcd
 
-Typhoon clusters on cloud providers run etcd as "self-hosted" pods, managed by the [etcd-operator](https://github.com/coreos/etcd-operator). By contrast, Typhoon bare-metal runs an etcd peer as a systemd `etcd-member.service` on each controller (i.e. on-host).
+AWS and Google Cloud clusters run etcd as "self-hosted" pods, managed by the [etcd-operator](https://github.com/coreos/etcd-operator). By contrast, Typhoon bare-metal and Digital Ocean run an etcd peer as a systemd `etcd-member.service` on each controller (i.e. on-host).
 
-In practice, self-hosted etcd has proven to be *ok*, but not ideal. Running the apiserver's etcd atop Kubernetes itself is inherently complex, but works suitably in most cases. It can be opaque to debug if complex edge cases with upstream Kubernetes bugs arise.
+In practice, self-hosted etcd has proven to be *ok*, but not ideal. Running the apiserver's etcd atop Kubernetes itself is inherently complex, but works in most cases. It can be opaque to debug if complex edge cases with upstream Kubernetes bugs arise.
 
 !!! note ""
-    Typhoon clusters and their defaults power the maintainers' clusters. The edge cases are sufficiently rare that self-hosted etcd is not a pressing issue, but cloud clusters may switch back to on-host etcd in the future.
+    Over time, we plan to deprecate self-hosted etcd and revert to running etcd on-host.
 
 ## Operating Systems
 

docs/google-cloud.md

@@ -1,6 +1,6 @@
 # Google Cloud
 
-In this tutorial, we'll create a Kubernetes v1.7.7 cluster on Google Compute Engine (not GKE).
+In this tutorial, we'll create a Kubernetes v1.8.2 cluster on Google Compute Engine (not GKE).
 
 We'll declare a Kubernetes cluster in Terraform using the Typhoon Terraform module. On apply, a network, firewall rules, managed instance groups of Kubernetes controllers and workers, network load balancers for controllers and workers, and health checks will be created.
 
@@ -10,7 +10,7 @@ Controllers and workers are provisioned to run a `kubelet`. A one-time [bootkube
 
 * Google Cloud Account and Service Account
 * Google Cloud DNS Zone (registered Domain Name or delegated subdomain)
-* Terraform v0.9.2+ and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally
+* Terraform v0.10.4+ and [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) installed locally
 
 ## Terraform Setup
 
@@ -18,7 +18,7 @@ Install [Terraform](https://www.terraform.io/downloads.html) v0.9.2+ on your sys
 
 ```sh
 $ terraform version
-Terraform v0.10.1
+Terraform v0.10.7
 ```
 
 Add the [terraform-provider-ct](https://github.com/coreos/terraform-provider-ct) plugin binary for your system.
@@ -80,7 +80,7 @@ module "google-cloud-yavin" {
   zone          = "us-central1-c"
   dns_zone      = "example.com"
   dns_zone_name = "example-zone"
-  os_image      = "coreos-stable-1465-6-0-v20170817"
+  os_image      = "coreos-stable-1520-6-0-v20171012"
 
   cluster_name       = "yavin"
   controller_count   = 1
@@ -120,7 +120,7 @@ Get or update Terraform modules.
 $ terraform get            # downloads missing modules
 $ terraform get --update   # updates all modules
 Get: git::https://github.com/poseidon/typhoon (update)
-Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.7.0 (update)
+Get: git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.8.1 (update)
 ```
 
 Plan the resources to be created.
@@ -154,9 +154,9 @@ In 5-10 minutes, the Kubernetes cluster will be ready.
 $ KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig
 $ kubectl get nodes
 NAME                                          STATUS   AGE    VERSION
-yavin-controller-1682.c.example-com.internal  Ready    6m     v1.7.7+coreos.0
-yavin-worker-jrbf.c.example-com.internal      Ready    5m     v1.7.7+coreos.0
-yavin-worker-mzdm.c.example-com.internal      Ready    5m     v1.7.7+coreos.0
+yavin-controller-1682.c.example-com.internal  Ready    6m     v1.8.2
+yavin-worker-jrbf.c.example-com.internal      Ready    5m     v1.8.2
+yavin-worker-mzdm.c.example-com.internal      Ready    5m     v1.8.2
 ```
 
 List the pods.

docs/img/spin.svg

@@ -1 +0,0 @@
-<?xml version="1.0" ?><svg enable-background="new 0 0 100 100" version="1.1" viewBox="0 0 100 100" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="Merged"><g><path d="M91.532,51.789c0.988-1.976,0.187-4.379-1.789-5.367L65.272,34.186c-2.099-2.028-4.599-3.642-7.369-4.712l22.729-7.576    c-0.698-2.096-2.964-3.229-5.059-2.53l-25.924,8.641c-3.048,0.048-5.946,0.719-8.575,1.891L51.789,8.468    c-1.976-0.988-4.379-0.187-5.367,1.789L34.186,34.728c-2.028,2.099-3.642,4.599-4.712,7.369l-7.576-22.729    c-2.096,0.698-3.229,2.964-2.53,5.059l8.641,25.924c0.048,3.048,0.719,5.946,1.891,8.575L8.468,48.211    c-0.988,1.976-0.187,4.379,1.789,5.367l24.471,12.236c2.099,2.028,4.599,3.642,7.369,4.712l-22.729,7.576    c0.698,2.096,2.964,3.229,5.059,2.53l25.924-8.641c3.048-0.048,5.946-0.719,8.575-1.891L48.211,91.532    c1.976,0.988,4.379,0.187,5.367-1.789l12.236-24.471c2.028-2.099,3.642-4.599,4.712-7.369l7.576,22.729    c2.096-0.698,3.229-2.964,2.53-5.06l-8.641-25.924c-0.048-3.048-0.719-5.946-1.891-8.575L91.532,51.789z M50,68    c-9.925,0-18-8.075-18-18s8.075-18,18-18s18,8.075,18,18S59.925,68,50,68z"/><path d="M50,38c-6.617,0-12,5.383-12,12s5.383,12,12,12s12-5.383,12-12S56.617,38,50,38z M50,58c-4.411,0-8-3.589-8-8s3.589-8,8-8    s8,3.589,8,8S54.411,58,50,58z"/></g></g></svg>

docs/index.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features
 
-* Kubernetes v1.7.7 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.8.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Ready for Ingress, Dashboards, Metrics and other optional [addons](addons/overview.md)
@@ -24,9 +24,9 @@ Typhoon provides a Terraform Module for each supported operating system and plat
 | Platform      | Operating System | Terraform Module | Status |
 |---------------|------------------|------------------|--------|
 | AWS           | Container Linux  | [aws/container-linux/kubernetes](aws.md) | alpha |
-| Bare-Metal    | Container Linux  | [bare-metal/container-linux/kubernetes](bare-metal.md) | production |
+| Bare-Metal    | Container Linux  | [bare-metal/container-linux/kubernetes](bare-metal.md) | stable |
 | Digital Ocean | Container Linux  | [digital-ocean/container-linux/kubernetes](digital-ocean.md) | beta |
-| Google Cloud  | Container Linux  | [google-cloud/container-linux/kubernetes](google-cloud.md) | production |
+| Google Cloud  | Container Linux  | [google-cloud/container-linux/kubernetes](google-cloud.md) | beta |
 
 ## Usage
 
@@ -77,9 +77,9 @@ In 5-10 minutes (varies by platform), the cluster will be ready. This Google Clo
 $ KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig
 $ kubectl get nodes
 NAME                                          STATUS   AGE    VERSION
-yavin-controller-1682.c.example-com.internal  Ready    6m     v1.7.7+coreos.0
-yavin-worker-jrbf.c.example-com.internal      Ready    5m     v1.7.7+coreos.0
-yavin-worker-mzdm.c.example-com.internal      Ready    5m     v1.7.7+coreos.0
+yavin-controller-1682.c.example-com.internal  Ready    6m     v1.8.2
+yavin-worker-jrbf.c.example-com.internal      Ready    5m     v1.8.2
+yavin-worker-mzdm.c.example-com.internal      Ready    5m     v1.8.2
 ```
 
 List the pods.

docs/topics/hardware.md

@@ -0,0 +1,175 @@
+# Hardware
+
+While bare-metal Kubernetes clusters have no special hardware requirements (beyond the [min reqs](/bare-metal.md#requirements)), Typhoon does ensure certain router and server hardware integrates well with Kubernetes.
+
+## Ubiquitiy
+
+Ubiquity EdgeRouters work well with bare-metal Kubernetes clusters. Knowledge about how to setup an EdgeRouter and use the CLI is required.
+
+### PXE
+
+Ubiquiti EdgeRouters can provide a PXE-enabled network boot environment for client machines.
+
+#### ISC DHCP
+
+Add a subnet parameter to the LAN DHCP server to include an ISC DHCP config file.
+
+```
+configure
+show service dhcp-server shared-network-name NAME subnet SUBNET
+set service dhcp-server shared-network-name NAME subnet SUBNET subnet-parameters "include "/config/scripts/ipxe.conf";"
+commit-confirm
+```
+
+Switch to root (i.e. `sudo -i`) and write the ISC DHCP config `/config/scripts/ipxe.conf`. iPXE client machines will chainload to `matchbox.example.com`, while non-iPXE clients will chainload to `undionly.kpxe` (requires TFTP to be enabled).
+
+```
+allow bootp;
+allow booting;
+next-server ADD_ROUTER_IP_HERE;
+
+if exists user-class and option user-class = "iPXE" {
+  filename "http://matchbox.example.com/boot.ipxe";
+} else {
+  filename "undionly.kpxe";
+}
+```
+
+### TFTP
+
+Use `dnsmasq` as a TFTP server to serve [undionly.kpxe](http://boot.ipxe.org/undionly.kpxe).
+
+```
+sudo -i
+mkdir /var/lib/tftpboot
+cd /var/lib/tftpboot
+curl http://boot.ipxe.org/undionly.kpxe -o undionly.kpxe
+```
+
+Add `dnsmasq` command line options to enable the TFTP file server.
+
+```
+configure
+show servce dns forwarding
+set service dns forwarding options enable-tftp
+set service dns forwarding options tftp-root=/var/lib/tftpboot
+commit-confirm
+```
+
+!!! warning
+    After firmware upgrades, the `/var/lib/tftpboot` directory will not exist and dnsmasq will not start properly. Repeat this process following an upgrade.
+
+### DHCP
+
+Assign static IPs to clients with known MAC addresses. This is called a static mapping by EdgeOS. Configure the router with the commands based on region inventory.
+
+```
+configure
+show service dhcp-server shared-network
+set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME mac-address MACADDR
+set service dhcp-server shared-network-name LAN subnet SUBNET static-mapping NAME ip-address 10.0.0.20
+```
+
+### DNS
+
+Assign DNS A records to nodes as options to `dnsmasq`.
+
+```
+configure
+set service dns forwarding options host-record=node.example.com,10.0.0.20
+```
+
+Restart `dnsmasq`.
+
+```
+sudo /etc/init.d/dnsmasq restart
+```
+
+Configure queries for `*.svc.cluster.local` to be forwarded to a Kubernetes `kube-dns` service IP to allow hosts to resolve cluster-local Kubernetes names.
+
+```
+configure
+show service dns forwarding
+set service dns forwarding options server=/svc.cluster.local/10.3.0.10
+commit-confirm
+```
+
+### Kubernetes Services
+
+Add static routes for the Kubernetes IPv4 service range to Kubernetes node(s) so hosts can route to Kubernetes services (default: 10.3.0.0/16).
+
+```
+configure
+show protocols static route
+set protocols static route 10.3.0.0/16 next-hop NODE_IP
+...
+commit-confirm
+```
+
+### Port Forwarding
+
+Expose the [Ingress Controller](/addons/ingress.md#bare-metal) by adding `port-forward` rules that DNAT a port on the router's WAN interface to an internal IP and port. By convention, a public Ingress controller is assigned a fixed service IP like kube-dns (e.g. 10.3.0.12).
+
+```
+configure
+set port-forward wan-interface eth0
+set port-forward lan-interface eth1
+set port-forward auto-firewall enable
+set port-forward hairpin-nat enable
+set port-forward rule 1 description 'ingress http'
+set port-forward rule 1 forward-to address 10.3.0.12
+set port-forward rule 1 forward-to port 80
+set port-forward rule 1 original-port 80
+set port-forward rule 1 protocol tcp_udp
+set port-forward rule 2 description 'ingress https'
+set port-forward rule 2 forward-to address 10.3.0.12
+set port-forward rule 2 forward-to port 443
+set port-forward rule 2 original-port 443
+set port-forward rule 2 protocol tcp_udp
+commit-confirm
+```
+
+### Web UI
+
+The web UI is often accessible from the LAN on ports 80/443 by default. Edit the ports to 8080 and 4443 to avoid a conflict.
+
+```
+configure
+show service gui
+set service gui http-port 8080
+set service gui https-port 4443
+commit-confirm
+```
+
+### BGP
+
+Add the EdgeRouter as a global BGP peer for nodes in a Kubernetes cluster (requires Calico). Neighbors will exchange `podCIDR` routes and individual pods will become routeable on the LAN.
+
+Configure node(s) as BGP neighbors.
+
+```
+show protocols bgp 1
+set protocols bgp 1 parameters router-id LAN_IP
+set protocols bgp 1 neighbor NODE1_IP remote-as 64512
+set protocols bgp 1 neighbor NODE2_IP remote-as 64512
+set protocols bgp 1 neighbor NODE3_IP remote-as 64512
+```
+
+View the neighbors and exchanged routes.
+
+```
+show ip bgp neighbors
+show ip route bgp
+```
+
+Be sure to register the peer by creating a Calico `bgpPeer` CRD with `kubectl apply`.
+
+```
+apiVersion: v1
+kind: bgpPeer
+metadata:
+  peerIP: LAN_IP
+  scope: global
+spec:
+  asNumber: 64512
+```

docs/topics/performance.md

@@ -0,0 +1,46 @@
+# Performance
+
+## Provision Time
+
+Provisioning times vary based on the platform. Sampling the time to create (apply) and destroy clusters with 1 controller and 2 workers shows (roughly) what to expect.
+
+| Platform      | Apply | Destroy |
+|---------------|-------|---------|
+| AWS           | 20 min | 8 min 10 sec |
+| Bare-Metal    | 10-14 min | NA  |
+| Digital Ocean | 3 min 30 sec | 20 sec |
+| Google Cloud  | 6 min 10 sec | 4 min 30 sec |
+
+Notes:
+
+* AWS is alpha
+* DNS propagation times have a large impact on provision time
+* Platforms with auto-scaling take more time to provision (AWS, Google)
+* Bare-metal provision times vary depending on the time for machines to POST and network bandwidth to download images.
+
+## Network Performance
+
+Network performance varies based on the platform and CNI plugin. `iperf` was used to measture the bandwidth between different hosts and different pods. Host-to-host indicates the typical bandwidth offered by the provider. Pod-to-pod shows the bandwidth between two `iperf` containers. The difference provides some idea about the overhead.
+
+| Platform / Plugin          | Theory | Host to Host | Pod to Pod   |
+|----------------------------|-------:|-------------:|-------------:|
+| AWS (flannel)              | ?      | 976 MB/s     | 900-999 MB/s |
+| AWS (calico, MTU 1480)     | ?      | 976 MB/s     | 100-350 MB/s |
+| AWS (calico, MTU 8991)     | ?      | 976 MB/s     | 900-999 MB/s |
+| Bare-Metal (flannel)       | 1 GB/s | 934 MB/s     | 903 MB/s     | 
+| Bare-Metal (calico)        | 1 GB/s | 941 MB/s     | 931 MB/s     |
+| Bare-Metal (flannel, bond) | 3 GB/s |  2.3 GB/s    | 1.17 GB/s    | 
+| Bare-Metal (calico, bond)  | 3 GB/s |  2.3 GB/s    | 1.17 GB/s    |
+| Digital Ocean              | ?      | 938 MB/s     | 820-880 MB/s |
+| Google Cloud (flannel)     | ?      | 1.94 GB/s    | 1.76 GB/s    |
+| Google Cloud (calico)      | ?      | 1.94 GB/s    | 1.81 GB/s    |
+
+Notes:
+
+* AWS is alpha
+* AWS instances are located in the same region. Google instances are located in the same zone (helps bandwidth at the expense of fault tolerance).
+* Network bandwidth fluctuates on AWS and Digital Ocean.
+* Only [certain AWS EC2 instance types](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html#jumbo_frame_instances) allow jumbo frames. This is why the default MTU on AWS must be 1480.
+* Between Flannel and Calico, performance differences are usually minimal. Platform and configuration differenes dominate.
+* Pods do not seem to be able to leverage the hosts' bonded NIC setup. Possibly a testing artifact.
+* Observing the same bonded NIC pod-to-pod limit suggests the bottleneck lies below flannel and calico.

google-cloud/container-linux/kubernetes/README.md

@@ -11,7 +11,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster
 
 ## Features
 
-* Kubernetes v1.7.7 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
+* Kubernetes v1.8.2 (upstream, via [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube))
 * Single or multi-master, workloads isolated on workers, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking
 * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
 * Ready for Ingress, Dashboards, Metrics, and other optional [addons](https://typhoon.psdn.io/addons/overview/)

google-cloud/container-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.7.0"
+  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.8.1"
 
   cluster_name                  = "${var.cluster_name}"
   api_servers                   = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]

google-cloud/container-linux/kubernetes/controllers/cl/controller.yaml.tmpl

@@ -34,7 +34,9 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --hosts-entry=host \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -104,8 +106,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -128,7 +130,7 @@ storage:
           [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
           [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
           BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
-          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.7.0}"
+          BOOTKUBE_VERSION="$${BOOTKUBE_VERSION:-v0.8.1}"
           BOOTKUBE_ASSETS="$${BOOTKUBE_ASSETS:-/opt/bootkube/assets}"
           exec /usr/bin/rkt run \
             --trust-keys-from-https \

google-cloud/container-linux/kubernetes/controllers/controllers.tf

@@ -55,6 +55,10 @@ resource "google_compute_instance_template" "controller" {
     boot         = true
     source_image = "${var.os_image}"
     disk_size_gb = "${var.disk_size}"
+
+    // Set explicit name to match the new default name set by the API.
+    // https://github.com/terraform-providers/terraform-provider-google/issues/574
+    device_name = "persistent-disk-0"
   }
 
   network_interface {

google-cloud/container-linux/kubernetes/require.tf

@@ -0,0 +1,25 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.10.4"
+}
+
+provider "google" {
+  version = "~> 1.0"
+}
+
+provider "local" {
+  version = "~> 1.0"
+}
+
+provider "null" {
+  version = "~> 1.0"
+}
+
+provider "template" {
+  version = "~> 1.0"
+}
+
+provider "tls" {
+  version = "~> 1.0"
+}

google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -34,7 +34,9 @@ systemd:
           --volume opt-cni-bin,kind=host,source=/opt/cni/bin \
           --mount volume=opt-cni-bin,target=/opt/cni/bin \
           --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log"
+          --mount volume=var-log,target=/var/log \
+          --hosts-entry=host \
+          --insecure-options=image"
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
@@ -102,8 +104,8 @@ storage:
       mode: 0644
       contents:
         inline: |
-          KUBELET_IMAGE_URL=quay.io/coreos/hyperkube
-          KUBELET_IMAGE_TAG=v1.7.7_coreos.0
+          KUBELET_IMAGE_URL=docker://gcr.io/google_containers/hyperkube
+          KUBELET_IMAGE_TAG=v1.8.2
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:
@@ -120,7 +122,8 @@ storage:
             --trust-keys-from-https \
             --volume config,kind=host,source=/etc/kubernetes \
             --mount volume=config,target=/etc/kubernetes \
-            quay.io/coreos/hyperkube:v1.7.7_coreos.0 \
+            --insecure-options=image \
+            docker://gcr.io/google_containers/hyperkube:v1.8.2 \
             --net=host \
             --dns=host \
             --exec=/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)

google-cloud/container-linux/kubernetes/workers/workers.tf

@@ -55,6 +55,10 @@ resource "google_compute_instance_template" "worker" {
     boot         = true
     source_image = "${var.os_image}"
     disk_size_gb = "${var.disk_size}"
+
+    // Set explicit name to match the new default name set by the API.
+    // https://github.com/terraform-providers/terraform-provider-google/issues/574
+    device_name = "persistent-disk-0"
   }
 
   network_interface {

mkdocs.yml

@@ -46,12 +46,15 @@ pages:
   - 'Google Cloud': 'google-cloud.md'
   - 'Addons':
     - 'Overview': 'addons/overview.md'
-    - 'Nginx Ingress': 'addons/ingress.md'
-    - 'Heapster': 'addons/heapster.md'
-    - 'Dashboard': 'addons/dashboard.md'
     - 'CLUO': 'addons/cluo.md'
+    - 'Heapster': 'addons/heapster.md'
+    - 'Nginx Ingress': 'addons/ingress.md'
+    - 'Prometheus': 'addons/prometheus.md'
+    - 'Dashboard': 'addons/dashboard.md'
   - 'Topics':
+    - 'Hardware': 'topics/hardware.md'
     - 'Security': 'topics/security.md'
+    - 'Performance': 'topics/performance.md'
   - 'FAQ': 'faq.md'
   - 'Advanced':
     - 'Customization': 'advanced/customization.md'