From 7c733bd314ed3d463208e6b85d52a608b1b9853c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 31 Aug 2017 09:28:57 -0700 Subject: [PATCH] Add Nginx Ingress controller addons and docs --- .../digital-ocean/daemonset.yaml | 61 +++++++++++++ .../default-backend/deployment.yaml | 36 ++++++++ .../default-backend/service.yaml | 15 ++++ .../digital-ocean/namespace.yaml | 4 + .../rbac/cluster-role-binding.yaml | 12 +++ .../digital-ocean/rbac/cluster-role.yaml | 51 +++++++++++ .../digital-ocean/rbac/role-binding.yaml | 13 +++ .../digital-ocean/rbac/role.yaml | 41 +++++++++ .../nginx-ingress/digital-ocean/service.yaml | 19 ++++ .../default-backend/deployment.yaml | 36 ++++++++ .../google-cloud/default-backend/service.yaml | 15 ++++ .../google-cloud/deployment.yaml | 61 +++++++++++++ .../nginx-ingress/google-cloud/namespace.yaml | 4 + .../rbac/cluster-role-binding.yaml | 12 +++ .../google-cloud/rbac/cluster-role.yaml | 51 +++++++++++ .../google-cloud/rbac/role-binding.yaml | 13 +++ .../nginx-ingress/google-cloud/rbac/role.yaml | 41 +++++++++ .../nginx-ingress/google-cloud/service.yaml | 19 ++++ docs/addons/calico.md | 4 - docs/addons/ingress.md | 90 ++++++++++++++++++- docs/addons/overview.md | 1 - mkdocs.yml | 3 +- 22 files changed, 592 insertions(+), 10 deletions(-) create mode 100644 addons/nginx-ingress/digital-ocean/daemonset.yaml create mode 100644 addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml create mode 100644 addons/nginx-ingress/digital-ocean/default-backend/service.yaml create mode 100644 addons/nginx-ingress/digital-ocean/namespace.yaml create mode 100644 addons/nginx-ingress/digital-ocean/rbac/cluster-role-binding.yaml create mode 100644 addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml create mode 100644 addons/nginx-ingress/digital-ocean/rbac/role-binding.yaml create mode 100644 addons/nginx-ingress/digital-ocean/rbac/role.yaml create mode 100644 addons/nginx-ingress/digital-ocean/service.yaml create mode 100644 addons/nginx-ingress/google-cloud/default-backend/deployment.yaml create mode 100644 addons/nginx-ingress/google-cloud/default-backend/service.yaml create mode 100644 addons/nginx-ingress/google-cloud/deployment.yaml create mode 100644 addons/nginx-ingress/google-cloud/namespace.yaml create mode 100644 addons/nginx-ingress/google-cloud/rbac/cluster-role-binding.yaml create mode 100644 addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml create mode 100644 addons/nginx-ingress/google-cloud/rbac/role-binding.yaml create mode 100644 addons/nginx-ingress/google-cloud/rbac/role.yaml create mode 100644 addons/nginx-ingress/google-cloud/service.yaml delete mode 100644 docs/addons/calico.md diff --git a/addons/nginx-ingress/digital-ocean/daemonset.yaml b/addons/nginx-ingress/digital-ocean/daemonset.yaml new file mode 100644 index 00000000..4a54d7d9 --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/daemonset.yaml @@ -0,0 +1,61 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: nginx-ingress-controller + namespace: ingress +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + name: nginx-ingress-controller + phase: prod + spec: + nodeSelector: + node-role.kubernetes.io/node: "" + containers: + - name: nginx-ingress-controller + image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11 + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-backend + - --ingress-class=public + # use downward API + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + hostPort: 80 + - name: https + containerPort: 443 + hostPort: 443 + - name: health + containerPort: 10254 + hostPort: 10254 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 1 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + hostNetwork: true + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml b/addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml new file mode 100644 index 00000000..f25afe40 --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/default-backend/deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-backend + namespace: ingress +spec: + replicas: 1 + template: + metadata: + labels: + name: default-backend + phase: prod + spec: + containers: + - name: default-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.0 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/digital-ocean/default-backend/service.yaml b/addons/nginx-ingress/digital-ocean/default-backend/service.yaml new file mode 100644 index 00000000..87997aba --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/default-backend/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: default-backend + namespace: ingress +spec: + type: ClusterIP + selector: + name: default-backend + phase: prod + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8080 diff --git a/addons/nginx-ingress/digital-ocean/namespace.yaml b/addons/nginx-ingress/digital-ocean/namespace.yaml new file mode 100644 index 00000000..b40cd8b2 --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ingress diff --git a/addons/nginx-ingress/digital-ocean/rbac/cluster-role-binding.yaml b/addons/nginx-ingress/digital-ocean/rbac/cluster-role-binding.yaml new file mode 100644 index 00000000..20494d2f --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/rbac/cluster-role-binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress +subjects: + - kind: ServiceAccount + namespace: ingress + name: default diff --git a/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml new file mode 100644 index 00000000..696a6920 --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/rbac/cluster-role.yaml @@ -0,0 +1,51 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update diff --git a/addons/nginx-ingress/digital-ocean/rbac/role-binding.yaml b/addons/nginx-ingress/digital-ocean/rbac/role-binding.yaml new file mode 100644 index 00000000..ecd04049 --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/rbac/role-binding.yaml @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: ingress + namespace: ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress +subjects: + - kind: ServiceAccount + namespace: ingress + name: default diff --git a/addons/nginx-ingress/digital-ocean/rbac/role.yaml b/addons/nginx-ingress/digital-ocean/rbac/role.yaml new file mode 100644 index 00000000..366251ae --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/rbac/role.yaml @@ -0,0 +1,41 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: ingress + namespace: ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-public" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - create + - update diff --git a/addons/nginx-ingress/digital-ocean/service.yaml b/addons/nginx-ingress/digital-ocean/service.yaml new file mode 100644 index 00000000..88afc467 --- /dev/null +++ b/addons/nginx-ingress/digital-ocean/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-ingress-controller + namespace: ingress +spec: + type: ClusterIP + selector: + name: nginx-ingress-controller + phase: prod + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 80 + - name: https + protocol: TCP + port: 443 + targetPort: 443 diff --git a/addons/nginx-ingress/google-cloud/default-backend/deployment.yaml b/addons/nginx-ingress/google-cloud/default-backend/deployment.yaml new file mode 100644 index 00000000..f25afe40 --- /dev/null +++ b/addons/nginx-ingress/google-cloud/default-backend/deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-backend + namespace: ingress +spec: + replicas: 1 + template: + metadata: + labels: + name: default-backend + phase: prod + spec: + containers: + - name: default-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.0 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/google-cloud/default-backend/service.yaml b/addons/nginx-ingress/google-cloud/default-backend/service.yaml new file mode 100644 index 00000000..87997aba --- /dev/null +++ b/addons/nginx-ingress/google-cloud/default-backend/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: default-backend + namespace: ingress +spec: + type: ClusterIP + selector: + name: default-backend + phase: prod + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8080 diff --git a/addons/nginx-ingress/google-cloud/deployment.yaml b/addons/nginx-ingress/google-cloud/deployment.yaml new file mode 100644 index 00000000..0ebf685b --- /dev/null +++ b/addons/nginx-ingress/google-cloud/deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + namespace: ingress +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + name: nginx-ingress-controller + phase: prod + spec: + nodeSelector: + node-role.kubernetes.io/node: "" + containers: + - name: nginx-ingress-controller + image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.11 + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-backend + - --ingress-class=public + # use downward API + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + hostPort: 80 + - name: https + containerPort: 443 + hostPort: 443 + - name: health + containerPort: 10254 + hostPort: 10254 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 1 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + hostNetwork: true + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/google-cloud/namespace.yaml b/addons/nginx-ingress/google-cloud/namespace.yaml new file mode 100644 index 00000000..b40cd8b2 --- /dev/null +++ b/addons/nginx-ingress/google-cloud/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ingress diff --git a/addons/nginx-ingress/google-cloud/rbac/cluster-role-binding.yaml b/addons/nginx-ingress/google-cloud/rbac/cluster-role-binding.yaml new file mode 100644 index 00000000..20494d2f --- /dev/null +++ b/addons/nginx-ingress/google-cloud/rbac/cluster-role-binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress +subjects: + - kind: ServiceAccount + namespace: ingress + name: default diff --git a/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml new file mode 100644 index 00000000..696a6920 --- /dev/null +++ b/addons/nginx-ingress/google-cloud/rbac/cluster-role.yaml @@ -0,0 +1,51 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update diff --git a/addons/nginx-ingress/google-cloud/rbac/role-binding.yaml b/addons/nginx-ingress/google-cloud/rbac/role-binding.yaml new file mode 100644 index 00000000..ecd04049 --- /dev/null +++ b/addons/nginx-ingress/google-cloud/rbac/role-binding.yaml @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: ingress + namespace: ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress +subjects: + - kind: ServiceAccount + namespace: ingress + name: default diff --git a/addons/nginx-ingress/google-cloud/rbac/role.yaml b/addons/nginx-ingress/google-cloud/rbac/role.yaml new file mode 100644 index 00000000..366251ae --- /dev/null +++ b/addons/nginx-ingress/google-cloud/rbac/role.yaml @@ -0,0 +1,41 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: ingress + namespace: ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-public" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - create + - update diff --git a/addons/nginx-ingress/google-cloud/service.yaml b/addons/nginx-ingress/google-cloud/service.yaml new file mode 100644 index 00000000..88afc467 --- /dev/null +++ b/addons/nginx-ingress/google-cloud/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-ingress-controller + namespace: ingress +spec: + type: ClusterIP + selector: + name: nginx-ingress-controller + phase: prod + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 80 + - name: https + protocol: TCP + port: 443 + targetPort: 443 diff --git a/docs/addons/calico.md b/docs/addons/calico.md deleted file mode 100644 index 9a62ba06..00000000 --- a/docs/addons/calico.md +++ /dev/null @@ -1,4 +0,0 @@ -# Calico Policy - -!!! bug "In Progress" - These docs haven't been moved over yet. diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index 32362ea4..d6bc5abe 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -1,4 +1,88 @@ -# Ingress Controller +# Nginx Ingress Controller -!!! bug "In Progress" - These docs haven't been moved over yet. +Nginx Ingress controller pods accept and demultiplex HTTP, HTTPS, TCP, or UDP traffic to backend services. Ingress controllers watch the Kubernetes API for Ingress resources and update their configuration accordingly. Ingress resources for HTTP(S) applications support virtual hosts (FQDNs), path rules, TLS termination, and SNI. + +## Google Cloud + +On Google Cloud, a network load balancer distributes traffic across worker nodes (i.e. a target pool of backends) running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure the target pool only includes worker nodes with a healthy Nginx Ingress controller. + +Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace. + +``` +kubectl apply -R addons/nginx-ingress/google-cloud +``` + +Add a DNS record resolving to the network load balancer's IPv4 address for each application. + +``` +app1.example.com -> 11.22.33.44 +aap2.example.com -> 11.22.33.44 +app3.example.com -> 11.22.33.44 +``` + +Find the IPv4 address with `gcloud compute addresses list` or use the Typhoon module's output `ingress_static_ip`. For example, you might use Terraform to manage a Google Cloud DNS record: + +```tf +resource "google_dns_record_set" "some-application" { + # DNS zone name + managed_zone = "example-zone" + + # DNS record + name = "app.example.com." + type = "A" + ttl = 300 + rrdatas = ["${module.google-cloud-yavin.ingress_static_ip}"] +} +``` + +## Digital Ocean + +On Digital Ocean, a DNS A record (e.g. `nemo-workers.example.com`) resolves to each worker[^1] running an Ingress controller DaemonSet on host ports 80 and 443. Firewall rules allow IPv4 and IPv6 traffic to ports 80 and 443. + +Create the Ingress controller daemonset, service, RBAC roles, RBAC bindings, default backend, and namespace. + +``` +kubectl apply -R addons/nginx-ingress/digital-ocean +``` + +Add a CNAME record to the worker DNS record for each application. Use the Typhoon module's output `workers_dns` to find the worker DNS value. For example, you might use Terraform to manage a Google Cloud DNS record: + +```tf +resource "google_dns_record_set" "some-application" { + # DNS zone name + managed_zone = "example-zone" + + # DNS record + name = "app.example.com." + type = "CNAME" + ttl = 300 + rrdatas = ["${module.digital-ocean-nemo.workers_dns}."] +} +``` + +[^1]: Digital Ocean does offers load balancers. We've opted not to use them to keep the Digital Ocean setup simple and cheap for developers. + +## Bare-Metal + +On bare-metal, routing traffic to Ingress controller pods can be done in number of ways. + +### Equal-Cost Multi-Path + +Deploy the Nginx Ingress Controller as a deployment. Deploy the service with a fixed ClusterIP (e.g. 10.3.0.12) in the Kubernetes service IPv4 CIDR range. There is no need for a NodePort or for pods to bind host ports. Any node can proxy packets destined for the service's ClusterIP to a node which has a pod endpoint. + +Configure the network router or load balancer with a static route for the Kubernetes service range and set the next hop to a node. Repeat for each node and set the metric (i.e. cost) of each. Finally, DNAT traffic destined for the WAN on ports 80 or 443 to the service's fixed ClusterIP. + +Add a DNS record resolving to the WAN for each application. + +```tf +resource "google_dns_record_set" "some-application" { + # Managed DNS Zone name + managed_zone = "dghubble-io" + + # Name of the DNS record + name = "app.example.com." + type = "A" + ttl = 300 + rrdatas = ["SOME-WAN-IP"] +} +``` diff --git a/docs/addons/overview.md b/docs/addons/overview.md index 0e782d26..68aa29ba 100644 --- a/docs/addons/overview.md +++ b/docs/addons/overview.md @@ -3,7 +3,6 @@ Every Typhoon cluster is verified to work well with several post-install addons. * Nginx [Ingress Controller](ingress.md) -* Calico [Network Policy](calico.md) * [Heapster](heapster.md) * Kubernetes [Dashboard](dashboard.md) * [CLUO](cluo.md) (Container Linux only) diff --git a/mkdocs.yml b/mkdocs.yml index e5f6b9fa..f5aabd82 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,8 +46,7 @@ pages: - 'Google Cloud': 'google-cloud.md' - 'Addons': - 'Overview': 'addons/overview.md' - - 'Ingress Controller': 'addons/ingress.md' - - 'Network Policy': 'addons/calico.md' + - 'Nginx Ingress': 'addons/ingress.md' - 'Heapster': 'addons/heapster.md' - 'Dashboard': 'addons/dashboard.md' - 'CLUO': 'addons/cluo.md'