diff --git a/Makefile b/Makefile index 19a69236..f859f554 100644 --- a/Makefile +++ b/Makefile @@ -113,10 +113,10 @@ endif @echo "--> Updating Python requirements" pip install --upgrade pip pip install --upgrade pip-tools + pip-compile --output-file requirements.txt requirements.in -U --no-index pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index - pip-compile --output-file requirements.txt requirements.in -U --no-index @echo "--> Done updating Python requirements" @echo "--> Removing python-ldap from requirements-docs.txt" grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..f7d1caf7 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,64 @@ +FROM alpine:3.8 + +ARG VERSION +ENV VERSION master + +ENV uid 1337 +ENV gid 1337 +ENV user lemur +ENV group lemur + +COPY entrypoint / +COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py +COPY supervisor.conf / +COPY nginx/default.conf /etc/nginx/conf.d/ +COPY nginx/default-ssl.conf /etc/nginx/conf.d/ + +RUN addgroup -S ${group} -g ${gid} && \ + adduser -D -S ${user} -G ${group} -u ${uid} && \ + apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \ + apk --update add --virtual build-dependencies \ + git \ + tar \ + curl \ + python3-dev \ + npm \ + bash \ + musl-dev \ + gcc \ + autoconf \ + automake \ + make \ + nasm \ + zlib-dev \ + postgresql-dev \ + libressl-dev \ + libffi-dev \ + cyrus-sasl-dev \ + openldap-dev && \ + mkdir -p /opt/lemur /home/lemur/.lemur/ && \ + curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ + pip3 install --upgrade pip && \ + pip3 install --upgrade setuptools && \ + chmod +x /entrypoint && \ + mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ + chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ + +WORKDIR /opt/lemur + +RUN npm install --unsafe-perm && \ + pip3 install -e . && \ + node_modules/.bin/gulp build && \ + node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \ + apk del build-dependencies + +WORKDIR / + +HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ + CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1 + +USER root + +ENTRYPOINT ["/entrypoint"] + +CMD ["/usr/bin/supervisord","-c","supervisor.conf"] diff --git a/docker/entrypoint b/docker/entrypoint new file mode 100644 index 00000000..6077167a --- /dev/null +++ b/docker/entrypoint @@ -0,0 +1,54 @@ +#!/bin/sh + +if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then + echo "Database vars not set" + exit 1 +fi + +export POSTGRES_PORT="${POSTGRES_PORT:-5432}" + +echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile + +source /etc/profile + +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' + +echo " # Create Postgres trgm extension" +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' +echo " # Done" + +if [ -z "${SKIP_SSL}" ]; then + if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then + openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" + fi + mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf + mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a +fi + +# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then +# echo "Creating config" +# https://github.com/Netflix/lemur/issues/2257 +# python3 /opt/lemur/lemur/manage.py create_config +# echo "Done" +# fi + +echo " # Running init" +su lemur -c "python3 /opt/lemur/lemur/manage.py init" +echo " # Done" + +# echo "Creating user" +# https://github.com/Netflix/lemur/issues/ +# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell +# echo "Done" + +cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" +cron_sync="${CRON_SYNC:-"*/15 * * * *"}" +cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" + +echo " # Populating crontab" +echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify +echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync +echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked +echo " # Done" + +exec "$@" diff --git a/docker/nginx/default-ssl.conf b/docker/nginx/default-ssl.conf new file mode 100644 index 00000000..86c770df --- /dev/null +++ b/docker/nginx/default-ssl.conf @@ -0,0 +1,37 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 80; + server_name _; + return 301 https://$host$request_uri; +} + +server { + listen 443; + server_name _; + access_log /dev/stdout; + error_log /dev/stderr; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf new file mode 100644 index 00000000..d71a93d3 --- /dev/null +++ b/docker/nginx/default.conf @@ -0,0 +1,26 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 80; + access_log /dev/stdout; + error_log /dev/stderr; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} diff --git a/docker/src/lemur.conf.py b/docker/src/lemur.conf.py new file mode 100644 index 00000000..a5f7e8b6 --- /dev/null +++ b/docker/src/lemur.conf.py @@ -0,0 +1,31 @@ +import os +_basedir = os.path.abspath(os.path.dirname(__file__)) + +CORS = os.environ.get("CORS") == "True" +debug = os.environ.get("DEBUG") == "True" + +SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw==')) + +LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ==')) +LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo=')) + +LEMUR_WHITELISTED_DOMAINS = [] + +LEMUR_EMAIL = '' +LEMUR_SECURITY_TEAM_EMAIL = [] + + +LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY','')) +LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE','')) +LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION','')) +LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION','')) +LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT','')) + +ACTIVE_PROVIDERS = [] + +METRIC_PROVIDERS = [] + +LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) +LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log')) + +SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') diff --git a/docker/supervisor.conf b/docker/supervisor.conf new file mode 100644 index 00000000..fed01581 --- /dev/null +++ b/docker/supervisor.conf @@ -0,0 +1,32 @@ +[supervisord] +nodaemon=true +user=root +logfile=/dev/stdout +logfile_maxbytes=0 +pidfile = /tmp/supervisord.pid + +[program:lemur] +environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py +command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000 +user=lemur +directory=/opt/lemur/lemur +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:cron] +command=/usr/sbin/crond -f +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b427b20..e9e47ed5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,18 +8,19 @@ aspy.yaml==1.1.1 # via pre-commit bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.11.29 # via requests -cfgv==1.1.0 # via pre-commit +cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.1.7 # via pre-commit +identify==1.1.8 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.7 # via pre-commit +importlib-metadata==0.8 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 -pkginfo==1.4.2 # via twine -pre-commit==1.12.0 +pkginfo==1.5.0 # via twine +pre-commit==1.13.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer @@ -29,8 +30,9 @@ requests-toolbelt==0.8.0 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.28.1 # via twine +tqdm==4.29.0 # via twine twine==1.12.1 urllib3==1.24.1 # via requests -virtualenv==16.1.0 # via pre-commit +virtualenv==16.2.0 # via pre-commit webencodings==0.5.1 # via bleach +zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index 3f036915..19ebb0ea 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,21 +4,21 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.29.1 +acme==0.30.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.5 amqp==2.3.2 aniso8601==4.0.1 -arrow==0.12.1 +arrow==0.13.0 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx -bcrypt==3.1.4 +bcrypt==3.1.5 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.60 -botocore==1.12.60 +boto3==1.9.75 +botocore==1.12.75 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 @@ -35,13 +35,13 @@ flask-cors==3.0.7 flask-mail==0.9.1 flask-migrate==2.3.1 flask-principal==0.4.0 -flask-restful==0.3.6 +flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -idna==2.7 +idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 @@ -49,12 +49,12 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.2 +kombu==4.2.2.post1 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 marshmallow-sqlalchemy==0.15.0 -marshmallow==2.16.3 +marshmallow==2.17.0 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==18.0 # via sphinx @@ -62,35 +62,35 @@ paramiko==2.4.2 pbr==5.1.1 pem==18.2.0 psycopg2==2.7.6.1 -pyasn1-modules==0.2.2 -pyasn1==0.4.4 +pyasn1-modules==0.2.3 +pyasn1==0.4.5 pycparser==2.19 pygments==2.3.1 # via sphinx -pyjwt==1.7.0 +pyjwt==1.7.1 pynacl==1.3.0 pyopenssl==18.0.0 pyparsing==2.3.0 # via packaging pyrfc3339==1.1 python-dateutil==2.7.5 python-editor==1.0.3 -pytz==2018.7 +pytz==2018.9 pyyaml==3.13 -raven[flask]==6.9.0 +raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.8.0 -requests[security]==2.20.1 +requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 -six==1.11.0 +six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.2 -sphinx==1.8.2 +sphinx==1.8.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx -sqlalchemy-utils==0.33.9 -sqlalchemy==1.2.14 +sqlalchemy-utils==0.33.10 +sqlalchemy==1.2.15 tabulate==0.8.2 urllib3==1.24.1 -vine==1.1.4 +vine==1.2.0 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 59c626f7..a11de6ec 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.67 # via moto +boto3==1.9.75 # via moto boto==2.49.0 # via moto -botocore==1.12.67 # via boto3, moto, s3transfer +botocore==1.12.75 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -34,7 +34,7 @@ jsondiff==1.1.1 # via moto jsonpickle==1.0 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==4.3.0 # via pytest +more-itertools==5.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.1 # via mock @@ -46,10 +46,10 @@ pycryptodome==3.7.2 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==4.0.2 +pytest==4.1.0 python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.7 # via moto +pytz==2018.9 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses diff --git a/requirements.txt b/requirements.txt index 7ee9a167..59871284 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,19 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.29.1 +acme==0.30.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.5 # via flask-migrate amqp==2.3.2 # via kombu aniso8601==4.0.1 # via flask-restful -arrow==0.12.1 +arrow==0.13.0 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.5 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.67 -botocore==1.12.67 +boto3==1.9.75 +botocore==1.12.75 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl @@ -46,20 +46,20 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.2 # via celery +kombu==4.2.2.post1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako marshmallow-sqlalchemy==0.15.0 -marshmallow==2.16.3 +marshmallow==2.17.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.1 # via mock pem==18.2.0 psycopg2==2.7.6.1 -pyasn1-modules==0.2.2 # via python-ldap -pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap +pyasn1-modules==0.2.3 # via python-ldap +pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi pyjwt==1.7.1 pynacl==1.3.0 # via paramiko @@ -68,19 +68,19 @@ pyrfc3339==1.1 # via acme python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339 +pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare -raven[flask]==6.9.0 +raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.8.0 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.12.0 -sqlalchemy-utils==0.33.9 +sqlalchemy-utils==0.33.10 sqlalchemy==1.2.15 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests -vine==1.1.4 # via amqp +vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0