From 0611cc9f70449faca7d75eccc29eccf6909c2b84 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 10 Jul 2023 16:55:45 +0200 Subject: [PATCH] feat(k8s): adding kubernetes support Now we can use skaffold and deploy bouncer in a kubernetes cluster ref #10 --- .gitignore | 3 +- misc/images/bouncer/Dockerfile | 49 +++++++++++++++++++ .../k8s/kustomization/base/kustomization.yaml | 10 ++++ .../bouncer-admin/files/admin-key.json | 1 + .../resources/bouncer-admin/files/config.yml | 36 ++++++++++++++ .../bouncer-admin/kustomization.yaml | 17 +++++++ .../bouncer-admin/resources/deployment.yaml | 38 ++++++++++++++ .../bouncer-admin/resources/service.yaml | 14 ++++++ .../resources/bouncer-server/files/config.yml | 22 +++++++++ .../bouncer-server/kustomization.yaml | 11 +++++ .../bouncer-server/resources/deployment.yaml | 31 ++++++++++++ .../bouncer-server/resources/service.yaml | 14 ++++++ .../base/resources/namespace.yaml | 4 ++ .../base/resources/redis/kustomization.yaml | 15 ++++++ .../redis/resources/redis-cluster.yaml | 21 ++++++++ .../overlays/dev/kustomization.yaml | 18 +++++++ .../add-registry-pull-secret.patch.yaml | 4 ++ skaffold.yaml | 42 ++++++++++++++++ 18 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 misc/images/bouncer/Dockerfile create mode 100644 misc/k8s/kustomization/base/kustomization.yaml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-admin/files/admin-key.json create mode 100644 misc/k8s/kustomization/base/resources/bouncer-admin/files/config.yml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-admin/kustomization.yaml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-admin/resources/deployment.yaml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-admin/resources/service.yaml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-server/files/config.yml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-server/kustomization.yaml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-server/resources/deployment.yaml create mode 100644 misc/k8s/kustomization/base/resources/bouncer-server/resources/service.yaml create mode 100644 misc/k8s/kustomization/base/resources/namespace.yaml create mode 100644 misc/k8s/kustomization/base/resources/redis/kustomization.yaml create mode 100644 misc/k8s/kustomization/base/resources/redis/resources/redis-cluster.yaml create mode 100644 misc/k8s/kustomization/overlays/dev/kustomization.yaml create mode 100644 misc/k8s/kustomization/overlays/dev/patches/add-registry-pull-secret.patch.yaml create mode 100644 skaffold.yaml diff --git a/.gitignore b/.gitignore index 607f345..219ca77 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ /admin-key.json /.bouncer-token /data -/out \ No newline at end of file +/out +.dockerconfigjson diff --git a/misc/images/bouncer/Dockerfile b/misc/images/bouncer/Dockerfile new file mode 100644 index 0000000..9e8607d --- /dev/null +++ b/misc/images/bouncer/Dockerfile @@ -0,0 +1,49 @@ +FROM golang:1.20 AS BUILD + +RUN apt-get update \ + && apt-get install -y make + +ARG YQ_VERSION=4.34.1 + +RUN mkdir -p /usr/local/bin \ + && wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64 \ + && chmod +x /usr/local/bin/yq + +COPY . /src + +WORKDIR /src + +RUN make GORELEASER_ARGS='build --rm-dist --single-target --snapshot' goreleaser + +# Patch config +RUN /src/dist/bouncer_linux_amd64_v1/bouncer -c '' config dump > /src/dist/bouncer_linux_amd64_v1/config.yml \ + && yq -i '.layers.queue.templateDir = "/usr/share/bouncer/layers/queue/templates"' /src/dist/bouncer_linux_amd64_v1/config.yml \ + && yq -i '.admin.auth.privateKey = "/etc/bouncer/admin-key.json"' /src/dist/bouncer_linux_amd64_v1/config.yml \ + && yq -i '.redis.adresses = ["redis:6379"]' /src/dist/bouncer_linux_amd64_v1/config.yml + +FROM alpine:3.18 AS RUNTIME + +ARG DUMB_INIT_VERSION=1.2.5 + +RUN apk add --no-cache ca-certificates + +RUN mkdir -p /usr/local/bin \ + && wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_x86_64 \ + && chmod +x /usr/local/bin/dumb-init + +ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] + +RUN mkdir -p /usr/local/bin /usr/share/bouncer/bin /etc/bouncer + +COPY --from=BUILD /src/dist/bouncer_linux_amd64_v1/bouncer /usr/share/bouncer/bin/bouncer +COPY --from=BUILD /src/layers /usr/share/bouncer/layers +COPY --from=BUILD /src/dist/bouncer_linux_amd64_v1/config.yml /etc/bouncer/config.yml + +RUN ln -s /usr/share/bouncer/bin/bouncer /usr/local/bin/bouncer + +EXPOSE 8080 +EXPOSE 8081 + +ENV BOUNCER_CONFIG=/etc/bouncer/config.yml + +CMD ["bouncer"] \ No newline at end of file diff --git a/misc/k8s/kustomization/base/kustomization.yaml b/misc/k8s/kustomization/base/kustomization.yaml new file mode 100644 index 0000000..9bb8fed --- /dev/null +++ b/misc/k8s/kustomization/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: bouncer + +resources: +- ./resources/namespace.yaml +- ./resources/bouncer-server +- ./resources/bouncer-admin +- ./resources/redis + diff --git a/misc/k8s/kustomization/base/resources/bouncer-admin/files/admin-key.json b/misc/k8s/kustomization/base/resources/bouncer-admin/files/admin-key.json new file mode 100644 index 0000000..59025df --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-admin/files/admin-key.json @@ -0,0 +1 @@ +{"d":"JuBw5OsGv3rPgVczxUgtJ6iUQ41LQu4Xpu-t8IKI_z8r-BZBlbndxidPmRlGZASLGL3rhY4qw6_ScFxakrMpCreO1RMU0kqtz--N48BXFnW5tEgr1voyyKP__bPssQNn6PgkoyAd11es7MEKlBff_DtGrcSkVRgU0zDZB-vIU0aNEIZPNw0icbYqc1u_QQNPpBU9cw6P33WHhzvfCVAkZKRszwznhiPM08n1vjpiA7e1kQ8a6OC4IFZBvohkmpmyOq1g1OLRABQ83YPCjGjCAejO-jEWkbLksp6rAl_YYpCvfBAjFV76JuZq4eh5IU82LsSfi3PGYBkhxWuLY779XQ","dp":"gljHOQowGK7fVn2DJizWtgRIDJuKpKnoX2PWNJUbm2WZwcEPZalAkxn7Y-w_reLVJZuRpfKEUMS-Tn3-CwI1ZjCHPqMPTXcoG0Pe2E-Z88jOs9lW4XSOASiiM980VIvkV1xCxDJkN3NsDFQ9j9kRGnKuMnsucCW3AKaU917hXNU","dq":"mqY19JcEBDnzS70_XkAsOKqPzemOScax66b-4N6zrsgeLVlRjHffY9uCAgBWzlxOidRdQN8q23ZJB4fqsKB2w00Iw7Jxx94IoAKGjKDT5iB48Y_kdKLAwSHRTXsqA9GG3po_H_JpP_EqX4TDBYtqQZuBD_tACP9HbLYMi_V2YU8","e":"AQAB","kty":"RSA","n":"sam0X0BGcuFwX8z3Wde8cv2o_zl6A9ghpkT0tCjw8qH3GNWrbAqzncSWdHBzoChBgAbuTOVs-ixYC0KeUhwFdc8Ul-jmKJWFaS8kIr3y4EH62-vLgMuIKfaxbsyUG6KMkJfnftge1jPO4ccddNej9msxcqTxu37dcgstutwtd6QkS9p5RrNbDBc8-Z7SQ4TuxJfP8msXRnCPJ-I44yszGdQF1Np2DXakJHVn8PBrDh3iSFwORw8jxNS4oS0OlBl5aSc0t5XkkaNcSU2a50SKts290w54fl6MPJ1sLnnznLy4uu37-nrfEUvqRLDZL9B1F82RM1dtLIIiN4gnSrMlpQ","p":"wOmFPhAT_wXWzMuwtEdYIer3-CiOWxFKpFL09eEJkJ29MIUchEaoiJaUAghqPxM48llfOVaUaLbFVxmo5U3fyjNMaP-nHMUBwojutykMK-gC2R3J4bQgFWfKbGSL7M7UsextAvpq9iiOuR0LNE-xTfCgPIxHVdPZskO3yx0DkjM","q":"68OGRb0tLRjb_PpkGctcSjEz_vvcyjzxGL-fn4_h4GCw98Xrj6Y4rZ4lfWWRSeDohSvdd-ICSlxvxkQOIOcA0H7jyJcBC0KDs4hX5BRGJNDri3QX0ry4_F1ptAdbfiFgQGqCfMRCr7L60Tfd_6tLczvny7eEBKQNGdj6dLfhgMc","qi":"DFwixyxUDf0REPLLa8hOKieRL95_AH9rbYWzStBOdSjKWra5l0reD6a4bbvAYvl0e8qCcRI6S8Nzpz0BYm4sJL7poVOnjxqvBY3Q9Ppf4Mq8lW39pOCJcqOHIvvYHsMjTC5uwp7Yg2p0GvxuUibbyNL1PXf6WZ_szVP_oSMrCXA"} \ No newline at end of file diff --git a/misc/k8s/kustomization/base/resources/bouncer-admin/files/config.yml b/misc/k8s/kustomization/base/resources/bouncer-admin/files/config.yml new file mode 100644 index 0000000..25ceccf --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-admin/files/config.yml @@ -0,0 +1,36 @@ +admin: + http: + host: 127.0.0.1 + port: 8081 + cors: + allowedOrigins: + - http://localhost:3001 + allowCredentials: true + allowMethods: + - POST + - GET + - PUT + - DELETE + allowedHeaders: + - Origin + - Accept + - Content-Type + - Authorization + - Sentry-Trace + debug: false + auth: + issuer: http://127.0.0.1:8081 + privateKey: /etc/bouncer/admin-key.json + metrics: + enabled: true + endpoint: /.bouncer/metrics + basicAuth: null + +redis: + addresses: + - ${REDIS_SENTINEL_HOST}:${REDIS_SENTINEL_PORT} + master: "${REDIS_MASTER_NAME}" + +logger: + level: 3 + format: human diff --git a/misc/k8s/kustomization/base/resources/bouncer-admin/kustomization.yaml b/misc/k8s/kustomization/base/resources/bouncer-admin/kustomization.yaml new file mode 100644 index 0000000..edec116 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-admin/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ./resources/service.yaml +- ./resources/deployment.yaml + +configMapGenerator: +- name: bouncer-admin-config + files: + - ./files/config.yml + - ./files/admin-key.json +- name: bouncer-admin-env + literals: + - REDIS_SENTINEL_HOST="rfs-$(REDIS_SERVICE_NAME)" + - REDIS_SENTINEL_PORT="26379" + - REDIS_MASTER_NAME="mymaster" diff --git a/misc/k8s/kustomization/base/resources/bouncer-admin/resources/deployment.yaml b/misc/k8s/kustomization/base/resources/bouncer-admin/resources/deployment.yaml new file mode 100644 index 0000000..12bc421 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-admin/resources/deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bouncer-admin + labels: + app: bouncer-admin +spec: + replicas: 3 + selector: + matchLabels: + app: bouncer-admin + template: + metadata: + labels: + app: bouncer-admin + spec: + containers: + - name: bouncer-admin + image: reg.cadoles.com/cadoles/bouncer:v2023.7.8-b44ff2a + command: ["bouncer"] + args: ["--debug", "-c", "/etc/bouncer/config.yml", "server", "admin", "run"] + imagePullPolicy: Always + envFrom: + - configMapRef: + name: bouncer-admin-env + env: + - name: REDIS_SENTINEL_HOST + value: "rfs-$(REDIS_SERVICE_NAME)" + ports: + - name: bouncer-admin + containerPort: 8081 + volumeMounts: + - mountPath: /etc/bouncer/ + name: bouncer-admin-config + volumes: + - name: bouncer-admin-config + configMap: + name: bouncer-admin-config diff --git a/misc/k8s/kustomization/base/resources/bouncer-admin/resources/service.yaml b/misc/k8s/kustomization/base/resources/bouncer-admin/resources/service.yaml new file mode 100644 index 0000000..dd1aa9d --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-admin/resources/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + io.kompose.service: bouncer-admin + name: bouncer-admin +spec: + type: ClusterIP + ports: + - name: bouncer-admin + port: 8081 + targetPort: 8080 + selector: + io.kompose.service: bouncer-admin diff --git a/misc/k8s/kustomization/base/resources/bouncer-server/files/config.yml b/misc/k8s/kustomization/base/resources/bouncer-server/files/config.yml new file mode 100644 index 0000000..7940727 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-server/files/config.yml @@ -0,0 +1,22 @@ +proxy: + http: + host: 0.0.0.0 + port: 8080 + metrics: + enabled: true + endpoint: /.bouncer/metrics + basicAuth: null + +layers: + queue: + templateDir: /usr/share/bouncer/layers/queue/templates + defaultKeepAlive: 1m0s + +redis: + addresses: + - ${RFS_BOUNCER_REDIS_SERVICE_HOST}:${RFS_BOUNCER_REDIS_SERVICE_PORT} + master: "" + +logger: + level: 3 + format: human diff --git a/misc/k8s/kustomization/base/resources/bouncer-server/kustomization.yaml b/misc/k8s/kustomization/base/resources/bouncer-server/kustomization.yaml new file mode 100644 index 0000000..59d9407 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-server/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ./resources/service.yaml +- ./resources/deployment.yaml + +configMapGenerator: +- name: bouncer-server-config + files: + - ./files/config.yml diff --git a/misc/k8s/kustomization/base/resources/bouncer-server/resources/deployment.yaml b/misc/k8s/kustomization/base/resources/bouncer-server/resources/deployment.yaml new file mode 100644 index 0000000..d4eb356 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-server/resources/deployment.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bouncer-server + labels: + app: bouncer-server +spec: + replicas: 3 + selector: + matchLabels: + app: bouncer-server + template: + metadata: + labels: + app: bouncer-server + spec: + containers: + - name: bouncer-server + image: reg.cadoles.com/cadoles/bouncer:v2023.7.8-b44ff2a + command: ["bouncer", "-c", "/etc/bouncer/config.yml", "server", "proxy", "run"] + imagePullPolicy: Always + ports: + - name: bouncer-server + containerPort: 8080 + volumeMounts: + - mountPath: /etc/bouncer/ + name: bouncer-server-config + volumes: + - name: bouncer-server-config + configMap: + name: bouncer-server-config diff --git a/misc/k8s/kustomization/base/resources/bouncer-server/resources/service.yaml b/misc/k8s/kustomization/base/resources/bouncer-server/resources/service.yaml new file mode 100644 index 0000000..691984f --- /dev/null +++ b/misc/k8s/kustomization/base/resources/bouncer-server/resources/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + io.kompose.service: bouncer-server + name: bouncer-server +spec: + type: ClusterIP + ports: + - name: bouncer-server + port: 8080 + targetPort: 8080 + selector: + io.kompose.service: bouncer-server diff --git a/misc/k8s/kustomization/base/resources/namespace.yaml b/misc/k8s/kustomization/base/resources/namespace.yaml new file mode 100644 index 0000000..bd46fb4 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bouncer diff --git a/misc/k8s/kustomization/base/resources/redis/kustomization.yaml b/misc/k8s/kustomization/base/resources/redis/kustomization.yaml new file mode 100644 index 0000000..9efdd93 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/redis/kustomization.yaml @@ -0,0 +1,15 @@ + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ./resources/redis-cluster.yaml + +vars: +- name: REDIS_SERVICE_NAME + objref: + name: bouncer-redis + apiVersion: databases.spotahome.com/v1 + kind: RedisFailover + fieldref: + fieldpath: metadata.name diff --git a/misc/k8s/kustomization/base/resources/redis/resources/redis-cluster.yaml b/misc/k8s/kustomization/base/resources/redis/resources/redis-cluster.yaml new file mode 100644 index 0000000..d647ee2 --- /dev/null +++ b/misc/k8s/kustomization/base/resources/redis/resources/redis-cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: databases.spotahome.com/v1 +kind: RedisFailover +metadata: + name: bouncer-redis +spec: + sentinel: + replicas: 3 + resources: + requests: + cpu: 100m + limits: + memory: 100Mi + redis: + replicas: 3 + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 400m + memory: 500Mi diff --git a/misc/k8s/kustomization/overlays/dev/kustomization.yaml b/misc/k8s/kustomization/overlays/dev/kustomization.yaml new file mode 100644 index 0000000..d692dc9 --- /dev/null +++ b/misc/k8s/kustomization/overlays/dev/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: bouncer-dev + +resources: +- ../../base + +secretGenerator: +- files: + - secrets/dockerconfig/.dockerconfigjson + name: regcred-dev + type: kubernetes.io/dockerconfigjson + +patches: +- path: patches/add-registry-pull-secret.patch.yaml + target: + kind: Deployment + version: v1 diff --git a/misc/k8s/kustomization/overlays/dev/patches/add-registry-pull-secret.patch.yaml b/misc/k8s/kustomization/overlays/dev/patches/add-registry-pull-secret.patch.yaml new file mode 100644 index 0000000..1e4b8d2 --- /dev/null +++ b/misc/k8s/kustomization/overlays/dev/patches/add-registry-pull-secret.patch.yaml @@ -0,0 +1,4 @@ +- op: add + path: "/spec/template/spec/imagePullSecrets" + value: + - name: regcred-dev diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..3da20a1 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,42 @@ +apiVersion: skaffold/v3 +kind: Config + +metadata: + name: bouncer + +manifests: + kustomize: + paths: + - misc/k8s/kustomization/base + +profiles: +- name: dev + manifests: + kustomize: + paths: + - misc/k8s/kustomization/overlays/dev + activation: + - command: dev + +build: + local: + push: true + + tagPolicy: + sha256: {} + + artifacts: + - image: reg.cadoles.com/cadoles/bouncer + context: . + sync: + infer: + - cmd/** + - internal/** + - layers/** + - tools/** + - data/** + docker: + dockerfile: misc/images/bouncer/Dockerfile + +deploy: + statusCheckDeadlineSeconds: 600