From dbdc3fc8504edfded907f58e654614ee049280b0 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 11 Aug 2018 12:14:23 -0700 Subject: [PATCH] Add nginx-ingress addon manifests for bare-metal --- CHANGES.md | 1 + .../nginx-ingress/bare-metal/0-namespace.yaml | 6 ++ .../default-backend/deployment.yaml | 40 ++++++++++ .../bare-metal/default-backend/service.yaml | 15 ++++ .../nginx-ingress/bare-metal/deployment.yaml | 73 +++++++++++++++++++ .../bare-metal/rbac/cluster-role-binding.yaml | 12 +++ .../bare-metal/rbac/cluster-role.yaml | 51 +++++++++++++ .../bare-metal/rbac/role-binding.yaml | 13 ++++ .../nginx-ingress/bare-metal/rbac/role.yaml | 41 +++++++++++ addons/nginx-ingress/bare-metal/service.yaml | 23 ++++++ docs/addons/ingress.md | 12 ++- 11 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 addons/nginx-ingress/bare-metal/0-namespace.yaml create mode 100644 addons/nginx-ingress/bare-metal/default-backend/deployment.yaml create mode 100644 addons/nginx-ingress/bare-metal/default-backend/service.yaml create mode 100644 addons/nginx-ingress/bare-metal/deployment.yaml create mode 100644 addons/nginx-ingress/bare-metal/rbac/cluster-role-binding.yaml create mode 100644 addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml create mode 100644 addons/nginx-ingress/bare-metal/rbac/role-binding.yaml create mode 100644 addons/nginx-ingress/bare-metal/rbac/role.yaml create mode 100644 addons/nginx-ingress/bare-metal/service.yaml diff --git a/CHANGES.md b/CHANGES.md index 39dd67c9..ff4d7a23 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,7 @@ Notable changes between versions. #### Addons * Update nginx-ingress from 0.16.2 to 0.17.1 +* Add nginx-ingress manifests for bare-metal * Update Grafana from 5.2.1 to 5.2.2 * Update heapster from v1.5.3 to v1.5.4 diff --git a/addons/nginx-ingress/bare-metal/0-namespace.yaml b/addons/nginx-ingress/bare-metal/0-namespace.yaml new file mode 100644 index 00000000..56903e1f --- /dev/null +++ b/addons/nginx-ingress/bare-metal/0-namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ingress + labels: + name: ingress diff --git a/addons/nginx-ingress/bare-metal/default-backend/deployment.yaml b/addons/nginx-ingress/bare-metal/default-backend/deployment.yaml new file mode 100644 index 00000000..786968e0 --- /dev/null +++ b/addons/nginx-ingress/bare-metal/default-backend/deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: default-backend + namespace: ingress +spec: + replicas: 1 + selector: + matchLabels: + name: default-backend + phase: prod + template: + metadata: + labels: + name: default-backend + phase: prod + spec: + containers: + - name: default-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: k8s.gcr.io/defaultbackend:1.4 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + terminationGracePeriodSeconds: 60 diff --git a/addons/nginx-ingress/bare-metal/default-backend/service.yaml b/addons/nginx-ingress/bare-metal/default-backend/service.yaml new file mode 100644 index 00000000..87997aba --- /dev/null +++ b/addons/nginx-ingress/bare-metal/default-backend/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: default-backend + namespace: ingress +spec: + type: ClusterIP + selector: + name: default-backend + phase: prod + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8080 diff --git a/addons/nginx-ingress/bare-metal/deployment.yaml b/addons/nginx-ingress/bare-metal/deployment.yaml new file mode 100644 index 00000000..814df769 --- /dev/null +++ b/addons/nginx-ingress/bare-metal/deployment.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingress-controller-public + namespace: ingress +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + name: ingress-controller-public + phase: prod + template: + metadata: + labels: + name: ingress-controller-public + phase: prod + spec: + containers: + - name: nginx-ingress-controller + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.17.1 + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-backend + - --ingress-class=public + # use downward API + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + - name: health + containerPort: 10254 + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 1 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 1 + securityContext: + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsUser: 33 # www-data + restartPolicy: Always + terminationGracePeriodSeconds: 60 + diff --git a/addons/nginx-ingress/bare-metal/rbac/cluster-role-binding.yaml b/addons/nginx-ingress/bare-metal/rbac/cluster-role-binding.yaml new file mode 100644 index 00000000..3be57109 --- /dev/null +++ b/addons/nginx-ingress/bare-metal/rbac/cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress +subjects: + - kind: ServiceAccount + namespace: ingress + name: default diff --git a/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml new file mode 100644 index 00000000..9fee9fde --- /dev/null +++ b/addons/nginx-ingress/bare-metal/rbac/cluster-role.yaml @@ -0,0 +1,51 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update diff --git a/addons/nginx-ingress/bare-metal/rbac/role-binding.yaml b/addons/nginx-ingress/bare-metal/rbac/role-binding.yaml new file mode 100644 index 00000000..46b5cece --- /dev/null +++ b/addons/nginx-ingress/bare-metal/rbac/role-binding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress + namespace: ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress +subjects: + - kind: ServiceAccount + namespace: ingress + name: default diff --git a/addons/nginx-ingress/bare-metal/rbac/role.yaml b/addons/nginx-ingress/bare-metal/rbac/role.yaml new file mode 100644 index 00000000..84a6065f --- /dev/null +++ b/addons/nginx-ingress/bare-metal/rbac/role.yaml @@ -0,0 +1,41 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress + namespace: ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-public" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - create + - update diff --git a/addons/nginx-ingress/bare-metal/service.yaml b/addons/nginx-ingress/bare-metal/service.yaml new file mode 100644 index 00000000..cca30291 --- /dev/null +++ b/addons/nginx-ingress/bare-metal/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: ingress-controller-public + namespace: ingress + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '10254' +spec: + type: ClusterIP + clusterIP: 10.3.0.12 + selector: + name: ingress-controller-public + phase: prod + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 80 + - name: https + protocol: TCP + port: 443 + targetPort: 443 diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index ec8e4cf9..a2474aba 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -101,11 +101,17 @@ On bare-metal, routing traffic to Ingress controller pods can be done in number ### Equal-Cost Multi-Path -Deploy the Nginx Ingress Controller as a deployment. Deploy the service with a fixed ClusterIP (e.g. 10.3.0.12) in the Kubernetes service IPv4 CIDR range. There is no need for a NodePort or for pods to bind host ports. Any node can proxy packets destined for the service's ClusterIP to a node which has a pod endpoint. +Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, and default backend. The service should use a fixed ClusterIP (e.g. 10.3.0.12) in the Kubernetes service IPv4 CIDR range. -Configure the network router or load balancer with a static route for the Kubernetes service range and set the next hop to a node. Repeat for each node and set the metric (i.e. cost) of each. Finally, DNAT traffic destined for the WAN on ports 80 or 443 to the service's fixed ClusterIP. +``` +kubectl apply -R -f addons/nginx-ingress/bare-metal +``` -Add a DNS record resolving to the WAN for each application. +There is no need for pods to use host networking or for the ingress service to use NodePort or LoadBalancer. Nodes already proxy packets destined for the service's ClusterIP to node(s) with a pod endpoint. + +Configure the network router or load balancer with a static route for the Kubernetes service range and set the next hop to a node. Repeat for each node, as desired, and set the metric (i.e. cost) of each. Finally, DNAT traffic destined for the WAN on ports 80 or 443 to the service's fixed ClusterIP. + +For each application, add a DNS record resolving to the WAN(s). ```tf resource "google_dns_record_set" "some-application" {