From 4ec8490c558de6de64098d778d91cc1f79035caf Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 00:04:13 +0100 Subject: [PATCH 01/52] Create Dockerfile --- docker/Dockerfile | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..60aa473e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,66 @@ +FROM alpine:3.8 as builder + +ARG VERSION + +ENV VERSION master +#ENV VERSION 0.7.0 + +RUN apk --update add python3 + +RUN apk --update add --virtual build-dependencies \ + git \ + tar \ + curl \ + python3-dev \ + npm \ + bash \ + musl-dev \ + gcc \ + autoconf \ + automake \ + make \ + nasm \ + zlib-dev \ + postgresql-dev \ + libressl-dev \ + libffi-dev \ + cyrus-sasl-dev \ + openldap-dev + +#RUN git clone https://github.com/Netflix/lemur + +RUN mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 + +RUN ls -lha /opt/lemur/ + +WORKDIR /opt/lemur + +RUN pip3 install --upgrade pip + +RUN npm install --unsafe-perm +RUN pip3 install setuptools +RUN pip3 install -e . +RUN node_modules/.bin/gulp build +RUN node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) + +RUN apk del build-dependencies + +##################### + +RUN apk add --update libldap postgresql-client bash nginx supervisor + +RUN mkdir -p /run/nginx/ + +WORKDIR / + +COPY entrypoint / + +RUN chmod +x /entrypoint + +COPY lemur.py /root/.lemur/lemur.conf.py +COPY supervisor.conf / +COPY default.conf /etc/nginx/conf.d/ + +ENTRYPOINT ["/entrypoint"] + +CMD ["/usr/bin/supervisord","-c","supervisor.conf"] From fc6caecc0bbf93bf8b7614111ecb757f2a6eca51 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:37:09 +0100 Subject: [PATCH 02/52] Update Dockerfile --- docker/Dockerfile | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 60aa473e..54b517b8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,17 @@ FROM alpine:3.8 as builder -ARG VERSION - ENV VERSION master #ENV VERSION 0.7.0 +ENV uid 1337 +ENV gid 1337 +ENV user lemur +ENV group lemur + +#RUN adduser -D -S -u ${uid} ${user} -G ${group} + +RUN addgroup -S ${group} -g ${gid} && adduser -D -S ${user} -G ${group} -u ${uid} + RUN apk --update add python3 RUN apk --update add --virtual build-dependencies \ @@ -35,19 +42,29 @@ RUN ls -lha /opt/lemur/ WORKDIR /opt/lemur +RUN npm install --unsafe-perm + RUN pip3 install --upgrade pip -RUN npm install --unsafe-perm RUN pip3 install setuptools RUN pip3 install -e . + +#RUN node_modules/.bin/gulp build --urlContextPath=/arnold/foo + RUN node_modules/.bin/gulp build + +#RUN node_modules/.bin/gulp build -h + RUN node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) RUN apk del build-dependencies + ##################### -RUN apk add --update libldap postgresql-client bash nginx supervisor +RUN apk add --update libldap postgresql-client bash nginx supervisor curl + +#RUN python3 /opt/lemur/lemur/manage.py reset_password -u lemur RUN mkdir -p /run/nginx/ @@ -57,10 +74,18 @@ COPY entrypoint / RUN chmod +x /entrypoint -COPY lemur.py /root/.lemur/lemur.conf.py +#RUN mkdir -p /conf + +COPY lemur.py /conf/lemur.conf.py + COPY supervisor.conf / COPY default.conf /etc/nginx/conf.d/ +HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ + CMD curl --fail http://localhost:80/api/1/healthcheck |grep -q ok || exit 1 + ENTRYPOINT ["/entrypoint"] +#CMD ["python3","/lemur/lemur/manage.py","start","-b","0.0.0.0:8000"] + CMD ["/usr/bin/supervisord","-c","supervisor.conf"] From 7eb6617a2801bfccbe290898e64d16b7aba345be Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:37:30 +0100 Subject: [PATCH 03/52] Create supervisor.conf --- docker/supervisor.conf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docker/supervisor.conf diff --git a/docker/supervisor.conf b/docker/supervisor.conf new file mode 100644 index 00000000..e04e4002 --- /dev/null +++ b/docker/supervisor.conf @@ -0,0 +1,31 @@ +[supervisord] +nodaemon=true +user=root +logfile=/dev/stdout +logfile_maxbytes=0 +pidfile = /tmp/supervisord.pid + + +[program:lemur] +command=python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py start -b 0.0.0.0:8000 +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:nginx] +command=nginx -g "daemon off;" +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:dcron] +command=crond -f +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 From c25c703723a2eca125230570aa6ce406aa508d85 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:37:46 +0100 Subject: [PATCH 04/52] Create entrypoint --- docker/entrypoint | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docker/entrypoint diff --git a/docker/entrypoint b/docker/entrypoint new file mode 100644 index 00000000..386cdc08 --- /dev/null +++ b/docker/entrypoint @@ -0,0 +1,32 @@ +#!/bin/sh + +#echo $POSTGRES_USER +#echo $POSTGRES_PASSWORD +#echo $POSTGRES_HOST +#echo $POSTGRES_PORT +#echo $POSTGRES_DB + +export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" + +#echo $SQLALCHEMY_DATABASE_URI + +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;;' +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' + +# echo "from django.contrib.auth.models import User; User.objects.create_superuser('ronald', 'koko', 'koko')" | python /opt/lemur/lemur/manage.py shell + + +echo "running init" +python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init -p password +echo "done" + + +cron="${custom_cron:-"*/5 * * * *"}" + +echo "${cron} /opt/check/exec.sh" >> /etc/crontabs/root + +#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py notify expirations +#*/15 * * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py source sync -s all +#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py certificate check_revoked + +exec "$@" From 6d5782b44c832bfe5858cc5caab1c3b7d2315ae3 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:38:05 +0100 Subject: [PATCH 05/52] Create lemur.conf.py --- docker/lemur.conf.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docker/lemur.conf.py diff --git a/docker/lemur.conf.py b/docker/lemur.conf.py new file mode 100644 index 00000000..753b39af --- /dev/null +++ b/docker/lemur.conf.py @@ -0,0 +1,31 @@ +import os +_basedir = os.path.abspath(os.path.dirname(__file__)) + +CORS = os.environ.get("CORS") == "True" +debug = os.environ.get("DEBUG") == "True" + +SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw==')) + +LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ==')) +LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo=')) + +LEMUR_WHITELISTED_DOMAINS = [] + +LEMUR_EMAIL = '' +LEMUR_SECURITY_TEAM_EMAIL = [] + + +LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY','')) +LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE','')) +LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION','')) +LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION','')) +LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT','')) + +ACTIVE_PROVIDERS = [] + +METRIC_PROVIDERS = [] + +LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) +LOG_FILE = str(os.environ.get('LOG_FILE','lemur.log')) + +SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') From 5567bb2eaafc5678bb14d88508e371cad1efd188 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:43:04 +0100 Subject: [PATCH 06/52] Update Dockerfile --- docker/Dockerfile | 80 +++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 55 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 54b517b8..d665da0e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,22 @@ -FROM alpine:3.8 as builder +FROM alpine:3.8 +ARG VERSION ENV VERSION master -#ENV VERSION 0.7.0 ENV uid 1337 ENV gid 1337 ENV user lemur ENV group lemur -#RUN adduser -D -S -u ${uid} ${user} -G ${group} +COPY entrypoint / +COPY lemur.conf.py /conf/lemur.conf.py +COPY supervisor.conf / +COPY default.conf /etc/nginx/conf.d/ -RUN addgroup -S ${group} -g ${gid} && adduser -D -S ${user} -G ${group} -u ${uid} - -RUN apk --update add python3 - -RUN apk --update add --virtual build-dependencies \ +RUN addgroup -S ${group} -g ${gid} && \ + adduser -D -S ${user} -G ${group} -u ${uid} && \ + apk --update add python3 libldap postgresql-client bash nginx supervisor curl && \ + apk --update add --virtual build-dependencies \ git \ tar \ curl \ @@ -32,60 +34,28 @@ RUN apk --update add --virtual build-dependencies \ libressl-dev \ libffi-dev \ cyrus-sasl-dev \ - openldap-dev - -#RUN git clone https://github.com/Netflix/lemur - -RUN mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 - -RUN ls -lha /opt/lemur/ - + openldap-dev && \ + mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ + pip3 install --upgrade pip && \ + pip3 install --upgrade setuptools && \ + chmod +x /entrypoint && \ + mkdir -p /run/nginx/ + WORKDIR /opt/lemur -RUN npm install --unsafe-perm - -RUN pip3 install --upgrade pip - -RUN pip3 install setuptools -RUN pip3 install -e . - -#RUN node_modules/.bin/gulp build --urlContextPath=/arnold/foo - -RUN node_modules/.bin/gulp build - -#RUN node_modules/.bin/gulp build -h - -RUN node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) - -RUN apk del build-dependencies - - -##################### - -RUN apk add --update libldap postgresql-client bash nginx supervisor curl - -#RUN python3 /opt/lemur/lemur/manage.py reset_password -u lemur - -RUN mkdir -p /run/nginx/ +RUN npm install --unsafe-perm && \ + pip3 install -e . && \ + node_modules/.bin/gulp build && \ + node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \ + apk del build-dependencies WORKDIR / -COPY entrypoint / - -RUN chmod +x /entrypoint - -#RUN mkdir -p /conf - -COPY lemur.py /conf/lemur.conf.py - -COPY supervisor.conf / -COPY default.conf /etc/nginx/conf.d/ - HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ - CMD curl --fail http://localhost:80/api/1/healthcheck |grep -q ok || exit 1 + CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1 + +USER lemur ENTRYPOINT ["/entrypoint"] -#CMD ["python3","/lemur/lemur/manage.py","start","-b","0.0.0.0:8000"] - CMD ["/usr/bin/supervisord","-c","supervisor.conf"] From 390157168546c2c0b32f69eba7ff786eee55448e Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:44:05 +0100 Subject: [PATCH 07/52] Update Dockerfile --- docker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d665da0e..0953b230 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,6 +8,7 @@ ENV gid 1337 ENV user lemur ENV group lemur + COPY entrypoint / COPY lemur.conf.py /conf/lemur.conf.py COPY supervisor.conf / @@ -39,7 +40,8 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ - mkdir -p /run/nginx/ + mkdir -p /run/nginx/ && \ + chown -R $user:$group /opt/lemur/ WORKDIR /opt/lemur From d8377ffc57c6a9e281223a72a775e0024d5b09bd Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:44:27 +0100 Subject: [PATCH 08/52] Update supervisor.conf --- docker/supervisor.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index e04e4002..b6355b6c 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -5,10 +5,9 @@ logfile=/dev/stdout logfile_maxbytes=0 pidfile = /tmp/supervisord.pid - [program:lemur] -command=python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py start -b 0.0.0.0:8000 -user=root +command=python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 +user=lemur stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 stderr_logfile=/dev/stderr From 4edda34e2dfb6868db4aa7053daea029a3cbcca2 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:47:27 +0100 Subject: [PATCH 09/52] Update entrypoint --- docker/entrypoint | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index 386cdc08..a3b4e20c 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,32 +1,20 @@ #!/bin/sh -#echo $POSTGRES_USER -#echo $POSTGRES_PASSWORD -#echo $POSTGRES_HOST -#echo $POSTGRES_PORT -#echo $POSTGRES_DB - export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" -#echo $SQLALCHEMY_DATABASE_URI - PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;;' PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' -# echo "from django.contrib.auth.models import User; User.objects.create_superuser('ronald', 'koko', 'koko')" | python /opt/lemur/lemur/manage.py shell - - -echo "running init" +echo "Running init" python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init -p password -echo "done" +echo "Done" +cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" +cron_sync="${CRON_SYNC:-"*/15 * * * *"}" +cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" -cron="${custom_cron:-"*/5 * * * *"}" - -echo "${cron} /opt/check/exec.sh" >> /etc/crontabs/root - -#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py notify expirations -#*/15 * * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py source sync -s all -#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py certificate check_revoked +echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root +echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root +echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root exec "$@" From ce634bfd08d91069699a3f1f208cf5899ab3f4f3 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:49:03 +0100 Subject: [PATCH 10/52] Create default.conf --- docker/default.conf | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docker/default.conf diff --git a/docker/default.conf b/docker/default.conf new file mode 100644 index 00000000..d71a93d3 --- /dev/null +++ b/docker/default.conf @@ -0,0 +1,26 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 80; + access_log /dev/stdout; + error_log /dev/stderr; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} From f8008e8614cdc35f62f42de00ba1c356b29999f0 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:01:28 +0100 Subject: [PATCH 11/52] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0953b230..0befdc57 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -56,7 +56,7 @@ WORKDIR / HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1 -USER lemur +USER root ENTRYPOINT ["/entrypoint"] From 58296cff5aa3b0d75a353a9c95c735678db2a4b6 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:25:11 +0100 Subject: [PATCH 12/52] Update entrypoint --- docker/entrypoint | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index a3b4e20c..eced8695 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -2,19 +2,28 @@ export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" -PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;;' +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' + +echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' +echo "Done" echo "Running init" -python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init -p password +python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init +echo "Done" + +echo "Creating user" +echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell echo "Done" cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" +echo "Populating crontab" echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root +echo "Done" exec "$@" From 60b84a29b515639bf076a60d5e345adea5f84aaa Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:28:02 +0100 Subject: [PATCH 13/52] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0befdc57..e3bb4552 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ ENV group lemur COPY entrypoint / -COPY lemur.conf.py /conf/lemur.conf.py +COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / COPY default.conf /etc/nginx/conf.d/ From 692671a5431d2db17d2cf8d8f7b1c0503f0ed604 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:43:55 +0100 Subject: [PATCH 14/52] Update entrypoint --- docker/entrypoint | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index eced8695..2b275e60 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -8,13 +8,21 @@ echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo "Done" +# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then +# echo "Creating config" +# https://github.com/Netflix/lemur/issues/2257 +# python3 /opt/lemur/lemur/manage.py create_config +# echo "Done" +# fi + echo "Running init" python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init echo "Done" -echo "Creating user" -echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell -echo "Done" +# echo "Creating user" +# https://github.com/Netflix/lemur/issues/ +# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell +# echo "Done" cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" From a4ce379bced46a095f95c29c03ff9aae832afa05 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:46:41 +0100 Subject: [PATCH 15/52] Update lemur.conf.py --- docker/lemur.conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/lemur.conf.py b/docker/lemur.conf.py index 753b39af..a5f7e8b6 100644 --- a/docker/lemur.conf.py +++ b/docker/lemur.conf.py @@ -26,6 +26,6 @@ ACTIVE_PROVIDERS = [] METRIC_PROVIDERS = [] LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) -LOG_FILE = str(os.environ.get('LOG_FILE','lemur.log')) +LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log')) SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') From 2ae6c3a7147bcd23175932ac7bcd057d99ed48b2 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:48:28 +0100 Subject: [PATCH 16/52] Update Dockerfile --- docker/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e3bb4552..c2cc805f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,12 +36,15 @@ RUN addgroup -S ${group} -g ${gid} && \ libffi-dev \ cyrus-sasl-dev \ openldap-dev && \ - mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ + mkdir -p /opt/lemur /home/lemur/.lemur/ && \ + curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ mkdir -p /run/nginx/ && \ - chown -R $user:$group /opt/lemur/ + touch /home/lemur/.lemur/lemur.log && \ + chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ + ln -s /home/lemur/.lemur/lemur.log /dev/stdout WORKDIR /opt/lemur From 7348fd37e86e5276cfe67a31f8693deafdf672d3 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:50:22 +0100 Subject: [PATCH 17/52] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c2cc805f..8305cdd5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,7 +44,7 @@ RUN addgroup -S ${group} -g ${gid} && \ mkdir -p /run/nginx/ && \ touch /home/lemur/.lemur/lemur.log && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ - ln -s /home/lemur/.lemur/lemur.log /dev/stdout + ln -s /dev/stdout /home/lemur/.lemur/lemur.log WORKDIR /opt/lemur From 97f6cdccfcd84848f9ca1f2de8df9bf03645010a Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:58:06 +0100 Subject: [PATCH 18/52] Update Dockerfile --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8305cdd5..d3d0d78b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,7 +42,6 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ mkdir -p /run/nginx/ && \ - touch /home/lemur/.lemur/lemur.log && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ ln -s /dev/stdout /home/lemur/.lemur/lemur.log From d5d42415013f52322f54d632cf11474ad356af7f Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:20:29 +0100 Subject: [PATCH 19/52] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 2b275e60..3604fce5 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -16,7 +16,7 @@ echo "Done" # fi echo "Running init" -python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init +su lemur -c "python3 /opt/lemur/lemur/manage.py init" echo "Done" # echo "Creating user" From abd29f8462211f1f48e7b34991fc6ebc671973b1 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:53:39 +0100 Subject: [PATCH 20/52] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 3604fce5..0b39bfed 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,6 +1,6 @@ #!/bin/sh -export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" +echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB' >> /etc/environment PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' From ba20c0742083a4de25a319ad0387a8e40c604a0e Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:54:31 +0100 Subject: [PATCH 21/52] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 0b39bfed..3604fce5 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,6 +1,6 @@ #!/bin/sh -echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB' >> /etc/environment +export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' From e488c0ddcf8c4ff4c7a126e661673758c0132ea8 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:57:14 +0100 Subject: [PATCH 22/52] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d3d0d78b..546e325e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,7 +16,7 @@ COPY default.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ - apk --update add python3 libldap postgresql-client bash nginx supervisor curl && \ + apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata bash && \ apk --update add --virtual build-dependencies \ git \ tar \ From aefdead50a95b35a7b852f5e7cd1a4b7befe3e67 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 00:04:58 +0100 Subject: [PATCH 23/52] Update entrypoint --- docker/entrypoint | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/entrypoint b/docker/entrypoint index 3604fce5..d0d8ab8b 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -17,6 +17,7 @@ echo "Done" echo "Running init" su lemur -c "python3 /opt/lemur/lemur/manage.py init" +#export LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py ; python3 /opt/lemur/lemur/manage.py init echo "Done" # echo "Creating user" From 25c4672845088e1324caa23e577796b5cd763842 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 10:41:19 +0100 Subject: [PATCH 24/52] Update supervisor.conf --- docker/supervisor.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index b6355b6c..311d997f 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -6,7 +6,7 @@ logfile_maxbytes=0 pidfile = /tmp/supervisord.pid [program:lemur] -command=python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 +command=/usr/bin/python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 user=lemur stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 @@ -14,7 +14,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:nginx] -command=nginx -g "daemon off;" +command=/usr/sbin/nginx -g "daemon off;" user=root stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 @@ -22,7 +22,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:dcron] -command=crond -f +command=/usr/sbin/crond -f user=root stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 From 239acb5f95a2b0fc6a4e7ffeb4bb514f6f3ac401 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 12:49:21 +0100 Subject: [PATCH 25/52] Update supervisor.conf --- docker/supervisor.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index 311d997f..185b07d1 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -1,4 +1,5 @@ [supervisord] +environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py nodaemon=true user=root logfile=/dev/stdout @@ -6,8 +7,9 @@ logfile_maxbytes=0 pidfile = /tmp/supervisord.pid [program:lemur] -command=/usr/bin/python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 +command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000 user=lemur +directory=/opt/lemur/lemur stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 stderr_logfile=/dev/stderr @@ -21,7 +23,7 @@ stdout_logfile_maxbytes = 0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:dcron] +[program:cron] command=/usr/sbin/crond -f user=root stdout_logfile=/dev/stdout From ca6f2b782b03f8c1f8a65a1b73507108d6a222de Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 12:52:07 +0100 Subject: [PATCH 26/52] Update supervisor.conf --- docker/supervisor.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index 185b07d1..fed01581 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -1,5 +1,4 @@ [supervisord] -environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py nodaemon=true user=root logfile=/dev/stdout @@ -7,6 +6,7 @@ logfile_maxbytes=0 pidfile = /tmp/supervisord.pid [program:lemur] +environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000 user=lemur directory=/opt/lemur/lemur From c94557f2edd8ddb006618e8095532c090aa1c10c Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 13:21:13 +0100 Subject: [PATCH 27/52] Update entrypoint --- docker/entrypoint | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/entrypoint b/docker/entrypoint index d0d8ab8b..dce3773d 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -8,6 +8,11 @@ echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo "Done" + +# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then +# openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE +# fi + # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" # https://github.com/Netflix/lemur/issues/2257 From 666f180482b17a578925566d118401d1390e63ae Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 13:21:30 +0100 Subject: [PATCH 28/52] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 546e325e..d2ae56a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,7 +16,7 @@ COPY default.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ - apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata bash && \ + apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \ apk --update add --virtual build-dependencies \ git \ tar \ From d6a374130cb033929c4c834b690af7a6d4fef229 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 13:33:58 +0100 Subject: [PATCH 29/52] Update entrypoint --- docker/entrypoint | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index dce3773d..82fe1780 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -9,9 +9,9 @@ PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTG echo "Done" -# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then -# openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE -# fi +if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then + openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" +fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" From 341756d7c0fde73c58e9970393067fc1d79b74de Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 14:07:56 +0100 Subject: [PATCH 30/52] Update entrypoint --- docker/entrypoint | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index 82fe1780..1c895b16 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -8,10 +8,12 @@ echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo "Done" - -if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" -fi +if [ -z ${SKIP_SSL} ]; then + if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then + openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" + fi + cp default.conf default_ssl.conf +then # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" From 6b1d2bfb60578dabbc390a64b0f7efc74834b475 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 14:55:13 +0100 Subject: [PATCH 31/52] Create default-ssl.conf --- docker/default-ssl.conf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docker/default-ssl.conf diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf new file mode 100644 index 00000000..8b791c45 --- /dev/null +++ b/docker/default-ssl.conf @@ -0,0 +1,31 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 443; + server_name _; + access_log /dev/stdout; + error_log /dev/stderr; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} From 7fb0631ff025ebd09b7f95a8c68b90010cd32e23 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 15:37:19 +0100 Subject: [PATCH 32/52] Update entrypoint --- docker/entrypoint | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index 1c895b16..ebfa9bfa 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,18 +1,27 @@ #!/bin/sh -export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" +if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];the + echo " # Vars not set" + exit 1 +fi + +export POSTGRES_PORT="${POSTGRES_PORT:-5432}" + +echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile + +source /etc/profile PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' -echo "Create Postgres trgm extension" +echo " # Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' -echo "Done" +echo " # Done" if [ -z ${SKIP_SSL} ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" + openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=FAKE/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi - cp default.conf default_ssl.conf + mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf then # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then @@ -22,10 +31,9 @@ then # echo "Done" # fi -echo "Running init" +echo " # Running init" su lemur -c "python3 /opt/lemur/lemur/manage.py init" -#export LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py ; python3 /opt/lemur/lemur/manage.py init -echo "Done" +echo " # Done" # echo "Creating user" # https://github.com/Netflix/lemur/issues/ @@ -36,10 +44,10 @@ cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" -echo "Populating crontab" +echo " # Populating crontab" echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root -echo "Done" +echo " # Done" exec "$@" From 728be37de9a969f164de3f750efece77e9c43938 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 15:37:48 +0100 Subject: [PATCH 33/52] Update Dockerfile --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d2ae56a3..b105b1fb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,6 +13,7 @@ COPY entrypoint / COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / COPY default.conf /etc/nginx/conf.d/ +COPY default-ssl.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ @@ -41,7 +42,7 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ - mkdir -p /run/nginx/ && \ + mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ ln -s /dev/stdout /home/lemur/.lemur/lemur.log From 4faedf3e5b8280161169c488e89337fcc3ee2683 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 16:58:51 +0100 Subject: [PATCH 34/52] Update entrypoint --- docker/entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index ebfa9bfa..f97e2cdb 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];the +if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then echo " # Vars not set" exit 1 fi @@ -22,7 +22,7 @@ if [ -z ${SKIP_SSL} ]; then openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=FAKE/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf -then +fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" From 809ca0fcfe28198aae8b28f521fd0a2ee88b5494 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 17:13:31 +0100 Subject: [PATCH 35/52] Update Dockerfile --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b105b1fb..8ebb5241 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,8 +43,7 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ - chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ - ln -s /dev/stdout /home/lemur/.lemur/lemur.log + chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ WORKDIR /opt/lemur From 628aaf2748a46fc302fc73a61149ec4c2c9629a5 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 17:36:52 +0100 Subject: [PATCH 36/52] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index f97e2cdb..b2850963 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -19,7 +19,7 @@ echo " # Done" if [ -z ${SKIP_SSL} ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=FAKE/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" + openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf fi From c0f6e5a134274a3fa329645738755c29a27e2e04 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 18:03:39 +0100 Subject: [PATCH 37/52] Update default-ssl.conf --- docker/default-ssl.conf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf index 8b791c45..2235b88d 100644 --- a/docker/default-ssl.conf +++ b/docker/default-ssl.conf @@ -2,6 +2,30 @@ add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; +server { + listen 80; + server_name _; + access_log /dev/stdout; + error_log /dev/stderr; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} + server { listen 443; server_name _; From 918af0873f8ba4102b0a5283f4c2f140e7a2508b Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 18:35:17 +0100 Subject: [PATCH 38/52] Update default-ssl.conf --- docker/default-ssl.conf | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf index 2235b88d..8b791c45 100644 --- a/docker/default-ssl.conf +++ b/docker/default-ssl.conf @@ -2,30 +2,6 @@ add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; -server { - listen 80; - server_name _; - access_log /dev/stdout; - error_log /dev/stderr; - - location /api { - proxy_pass http://127.0.0.1:8000; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - proxy_redirect off; - proxy_buffering off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location / { - root /opt/lemur/lemur/static/dist; - include mime.types; - index index.html; - } - -} - server { listen 443; server_name _; From ff0dbdcc5a1b1f2fefcb2fceab3dd6f695ab0dff Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 18:36:02 +0100 Subject: [PATCH 39/52] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index b2850963..565c0fd6 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,7 +1,7 @@ #!/bin/sh if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then - echo " # Vars not set" + echo "Database vars not set" exit 1 fi From 3cc63c6618846bc1e15b56458c8ce5aeca247641 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:05:45 +0100 Subject: [PATCH 40/52] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 565c0fd6..d7ace70a 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -17,7 +17,7 @@ echo " # Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo " # Done" -if [ -z ${SKIP_SSL} ]; then +if [ -z "${SKIP_SSL}" ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi From 0d0c295f82705a8173a4530f3b9393898bfe9c37 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:33:49 +0100 Subject: [PATCH 41/52] Update entrypoint --- docker/entrypoint | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index d7ace70a..18ab0da5 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -42,12 +42,12 @@ echo " # Done" cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" -cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" +cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" echo " # Populating crontab" -echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root -echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root -echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root +echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify +echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync +echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked echo " # Done" exec "$@" From bb4b781d246297e298143c9153e10088d0d8660d Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:46:56 +0100 Subject: [PATCH 42/52] Update entrypoint --- docker/entrypoint | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/entrypoint b/docker/entrypoint index 18ab0da5..ad1d310c 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -22,6 +22,7 @@ if [ -z "${SKIP_SSL}" ]; then openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf + mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then From 28382ce728d25c190d5dca14d88a65d69d0c6802 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:48:42 +0100 Subject: [PATCH 43/52] Update default-ssl.conf --- docker/default-ssl.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf index 8b791c45..86c770df 100644 --- a/docker/default-ssl.conf +++ b/docker/default-ssl.conf @@ -2,6 +2,12 @@ add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; +server { + listen 80; + server_name _; + return 301 https://$host$request_uri; +} + server { listen 443; server_name _; From 4570fcf7fa07cd42b249e67926f1a4bfc5e24990 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:49:24 +0100 Subject: [PATCH 44/52] Rename docker/default-ssl.conf to docker/nginx/default-ssl.conf --- docker/{ => nginx}/default-ssl.conf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{ => nginx}/default-ssl.conf (100%) diff --git a/docker/default-ssl.conf b/docker/nginx/default-ssl.conf similarity index 100% rename from docker/default-ssl.conf rename to docker/nginx/default-ssl.conf From 248c0d226f827e0c612450baacf27100670079ad Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:49:36 +0100 Subject: [PATCH 45/52] Rename docker/default.conf to docker/nginx/default.conf --- docker/{ => nginx}/default.conf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{ => nginx}/default.conf (100%) diff --git a/docker/default.conf b/docker/nginx/default.conf similarity index 100% rename from docker/default.conf rename to docker/nginx/default.conf From 949ebfa2850f02f1e2f875706192fe9dddb8f299 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:49:49 +0100 Subject: [PATCH 46/52] Update Dockerfile --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8ebb5241..7fa61700 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,8 +12,8 @@ ENV group lemur COPY entrypoint / COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / -COPY default.conf /etc/nginx/conf.d/ -COPY default-ssl.conf /etc/nginx/conf.d/ +COPY nginx/default.conf /etc/nginx/conf.d/ +COPY nginx/default-ssl.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ From 6c1129c946a4b47bf966e9c003335122995dc6c6 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:50:14 +0100 Subject: [PATCH 47/52] Rename docker/lemur.conf.py to docker/src/lemur.conf.py --- docker/{ => src}/lemur.conf.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{ => src}/lemur.conf.py (100%) diff --git a/docker/lemur.conf.py b/docker/src/lemur.conf.py similarity index 100% rename from docker/lemur.conf.py rename to docker/src/lemur.conf.py From 125a885742a19c0eb2f821007d168b0b22b98f45 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:50:48 +0100 Subject: [PATCH 48/52] Update Dockerfile --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7fa61700..f7d1caf7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,9 +8,8 @@ ENV gid 1337 ENV user lemur ENV group lemur - COPY entrypoint / -COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py +COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / COPY nginx/default.conf /etc/nginx/conf.d/ COPY nginx/default-ssl.conf /etc/nginx/conf.d/ From 7cbdc09055a04c747b2ab190b7e4d5b3e2144761 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 12:09:06 +0100 Subject: [PATCH 49/52] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index ad1d310c..6077167a 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -19,7 +19,7 @@ echo " # Done" if [ -z "${SKIP_SSL}" ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" + openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a From 3ac5361cb2b22775c7bd2f2fe5989c919919d9af Mon Sep 17 00:00:00 2001 From: bby-bishopclark <30503374+bby-bishopclark@users.noreply.github.com> Date: Thu, 3 Jan 2019 07:58:42 -0800 Subject: [PATCH 50/52] Update index.rst Simple English gaffes noticed while perusing docs -- Setup vs set up, it's vs English, etc. --- docs/quickstart/index.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 70ca1312..adeadd7c 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur: Installing Build Dependencies ----------------------------- -If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies: +If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies: .. code-block:: bash @@ -117,7 +117,7 @@ Simply run: .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command. -You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment. +You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment. Update your configuration @@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config LEMUR_DEFAULT_ORGANIZATION LEMUR_DEFAULT_ORGANIZATIONAL_UNIT -Setup Postgres +Set Up Postgres -------------- For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on. @@ -193,10 +193,10 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. -Setup a Reverse Proxy +Set Up a Reverse Proxy --------------------- -By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. +By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. Proxying with Nginx From faa91ef2a71aac12cbf68910e172dd7beec96ad5 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 8 Jan 2019 09:47:46 -0800 Subject: [PATCH 51/52] Update requirements with Kombu fix --- requirements-dev.txt | 16 +++++++++------- requirements-docs.txt | 40 ++++++++++++++++++++-------------------- requirements-tests.txt | 10 +++++----- requirements.in | 1 + requirements.txt | 24 ++++++++++++------------ 5 files changed, 47 insertions(+), 44 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b427b20..e9e47ed5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,18 +8,19 @@ aspy.yaml==1.1.1 # via pre-commit bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.11.29 # via requests -cfgv==1.1.0 # via pre-commit +cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.1.7 # via pre-commit +identify==1.1.8 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.7 # via pre-commit +importlib-metadata==0.8 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 -pkginfo==1.4.2 # via twine -pre-commit==1.12.0 +pkginfo==1.5.0 # via twine +pre-commit==1.13.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer @@ -29,8 +30,9 @@ requests-toolbelt==0.8.0 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.28.1 # via twine +tqdm==4.29.0 # via twine twine==1.12.1 urllib3==1.24.1 # via requests -virtualenv==16.1.0 # via pre-commit +virtualenv==16.2.0 # via pre-commit webencodings==0.5.1 # via bleach +zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index 3f036915..bb1fe767 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,21 +4,21 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.29.1 +acme==0.30.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.5 amqp==2.3.2 aniso8601==4.0.1 -arrow==0.12.1 +arrow==0.13.0 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx -bcrypt==3.1.4 +bcrypt==3.1.5 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.60 -botocore==1.12.60 +boto3==1.9.75 +botocore==1.12.75 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 @@ -35,13 +35,13 @@ flask-cors==3.0.7 flask-mail==0.9.1 flask-migrate==2.3.1 flask-principal==0.4.0 -flask-restful==0.3.6 +flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -idna==2.7 +idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 @@ -49,12 +49,12 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.2 +kombu==4.2.1 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 marshmallow-sqlalchemy==0.15.0 -marshmallow==2.16.3 +marshmallow==2.17.0 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==18.0 # via sphinx @@ -62,35 +62,35 @@ paramiko==2.4.2 pbr==5.1.1 pem==18.2.0 psycopg2==2.7.6.1 -pyasn1-modules==0.2.2 -pyasn1==0.4.4 +pyasn1-modules==0.2.3 +pyasn1==0.4.5 pycparser==2.19 pygments==2.3.1 # via sphinx -pyjwt==1.7.0 +pyjwt==1.7.1 pynacl==1.3.0 pyopenssl==18.0.0 pyparsing==2.3.0 # via packaging pyrfc3339==1.1 python-dateutil==2.7.5 python-editor==1.0.3 -pytz==2018.7 +pytz==2018.9 pyyaml==3.13 -raven[flask]==6.9.0 +raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.8.0 -requests[security]==2.20.1 +requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 -six==1.11.0 +six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.2 -sphinx==1.8.2 +sphinx==1.8.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx -sqlalchemy-utils==0.33.9 -sqlalchemy==1.2.14 +sqlalchemy-utils==0.33.10 +sqlalchemy==1.2.15 tabulate==0.8.2 urllib3==1.24.1 -vine==1.1.4 +vine==1.2.0 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 59c626f7..a11de6ec 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.67 # via moto +boto3==1.9.75 # via moto boto==2.49.0 # via moto -botocore==1.12.67 # via boto3, moto, s3transfer +botocore==1.12.75 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -34,7 +34,7 @@ jsondiff==1.1.1 # via moto jsonpickle==1.0 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==4.3.0 # via pytest +more-itertools==5.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.1 # via mock @@ -46,10 +46,10 @@ pycryptodome==3.7.2 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==4.0.2 +pytest==4.1.0 python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.7 # via moto +pytz==2018.9 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses diff --git a/requirements.in b/requirements.in index 9824650b..e427c9a2 100644 --- a/requirements.in +++ b/requirements.in @@ -25,6 +25,7 @@ future gunicorn inflection jinja2 +kombu<=4.2.2 # Kombu 4.2.2 breaks requirements lockfile marshmallow-sqlalchemy marshmallow diff --git a/requirements.txt b/requirements.txt index 7ee9a167..e3918631 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,19 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.29.1 +acme==0.30.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.5 # via flask-migrate amqp==2.3.2 # via kombu aniso8601==4.0.1 # via flask-restful -arrow==0.12.1 +arrow==0.13.0 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.5 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.67 -botocore==1.12.67 +boto3==1.9.75 +botocore==1.12.75 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl @@ -46,20 +46,20 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.2 # via celery +kombu==4.2.1 lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako marshmallow-sqlalchemy==0.15.0 -marshmallow==2.16.3 +marshmallow==2.17.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.1 # via mock pem==18.2.0 psycopg2==2.7.6.1 -pyasn1-modules==0.2.2 # via python-ldap -pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap +pyasn1-modules==0.2.3 # via python-ldap +pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi pyjwt==1.7.1 pynacl==1.3.0 # via paramiko @@ -68,19 +68,19 @@ pyrfc3339==1.1 # via acme python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339 +pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare -raven[flask]==6.9.0 +raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.8.0 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.12.0 -sqlalchemy-utils==0.33.9 +sqlalchemy-utils==0.33.10 sqlalchemy==1.2.15 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests -vine==1.1.4 # via amqp +vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0 From c95fde702376cd99d8cdb4d8b1bbaf89f0913666 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 8 Jan 2019 09:55:53 -0800 Subject: [PATCH 52/52] Better fix for kombu is to unpin it and modify makefile --- Makefile | 2 +- requirements-docs.txt | 2 +- requirements.in | 1 - requirements.txt | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 19a69236..f859f554 100644 --- a/Makefile +++ b/Makefile @@ -113,10 +113,10 @@ endif @echo "--> Updating Python requirements" pip install --upgrade pip pip install --upgrade pip-tools + pip-compile --output-file requirements.txt requirements.in -U --no-index pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index - pip-compile --output-file requirements.txt requirements.in -U --no-index @echo "--> Done updating Python requirements" @echo "--> Removing python-ldap from requirements-docs.txt" grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt diff --git a/requirements-docs.txt b/requirements-docs.txt index bb1fe767..19ebb0ea 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -49,7 +49,7 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.1 +kombu==4.2.2.post1 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 diff --git a/requirements.in b/requirements.in index e427c9a2..9824650b 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,6 @@ future gunicorn inflection jinja2 -kombu<=4.2.2 # Kombu 4.2.2 breaks requirements lockfile marshmallow-sqlalchemy marshmallow diff --git a/requirements.txt b/requirements.txt index e3918631..59871284 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.1 +kombu==4.2.2.post1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako