Merge branch 'master' into verify-cert-private-key

Makefile

@@ -113,10 +113,10 @@ endif
 	@echo "--> Updating Python requirements"
 	pip install --upgrade pip
 	pip install --upgrade pip-tools
+	pip-compile --output-file requirements.txt requirements.in -U --no-index
 	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
 	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
 	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
-	pip-compile --output-file requirements.txt requirements.in -U --no-index
 	@echo "--> Done updating Python requirements"
 	@echo "--> Removing python-ldap from requirements-docs.txt"
 	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

docker/Dockerfile

@@ -0,0 +1,64 @@
+FROM alpine:3.8
+
+ARG VERSION
+ENV VERSION master
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    mkdir -p /opt/lemur /home/lemur/.lemur/ && \
+    curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    chmod +x /entrypoint && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
+    chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
+    
+WORKDIR /opt/lemur
+
+RUN npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
+    apk del build-dependencies
+
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/entrypoint

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
+  echo "Database vars not set"
+  exit 1
+fi
+
+export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
+
+echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
+
+source /etc/profile
+
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
+
+echo " # Create Postgres trgm extension"
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
+echo " # Done"
+
+if [ -z "${SKIP_SSL}" ]; then
+  if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
+    openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
+  fi
+  mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+fi
+
+# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
+# echo "Creating config"
+# https://github.com/Netflix/lemur/issues/2257
+# python3 /opt/lemur/lemur/manage.py create_config
+# echo "Done"
+# fi
+
+echo " # Running init"
+su lemur -c "python3 /opt/lemur/lemur/manage.py init"
+echo " # Done"
+
+# echo "Creating user"
+# https://github.com/Netflix/lemur/issues/
+# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
+# echo "Done"
+
+cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
+cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
+cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
+
+echo " # Populating crontab"
+echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify
+echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync
+echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked
+echo " # Done"
+
+exec "$@"

docker/nginx/default-ssl.conf

@@ -0,0 +1,37 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+    listen 80;
+    server_name  _;
+    return 301 https://$host$request_uri;
+}
+
+server {
+   listen 443;
+   server_name  _;
+   access_log /dev/stdout;
+   error_log /dev/stderr;
+   ssl_certificate /etc/nginx/ssl/server.crt;
+   ssl_certificate_key /etc/nginx/ssl/server.key;
+   ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+   ssl_ciphers HIGH:!aNULL:!MD5;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/nginx/default.conf

@@ -0,0 +1,26 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+   listen      80;
+   access_log  /dev/stdout;
+   error_log   /dev/stderr;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/src/lemur.conf.py

@@ -0,0 +1,31 @@
+import os
+_basedir = os.path.abspath(os.path.dirname(__file__))
+
+CORS = os.environ.get("CORS") == "True"
+debug = os.environ.get("DEBUG") == "True"
+
+SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw=='))
+
+LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ=='))
+LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo='))
+
+LEMUR_WHITELISTED_DOMAINS = []
+
+LEMUR_EMAIL = ''
+LEMUR_SECURITY_TEAM_EMAIL = []
+
+
+LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
+LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE',''))
+LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
+LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
+
+ACTIVE_PROVIDERS = []
+
+METRIC_PROVIDERS = []
+
+LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
+LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
+
+SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')

docker/supervisor.conf

@@ -0,0 +1,32 @@
+[supervisord]
+nodaemon=true
+user=root
+logfile=/dev/stdout
+logfile_maxbytes=0
+pidfile = /tmp/supervisord.pid
+
+[program:lemur]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
+command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000
+user=lemur
+directory=/opt/lemur/lemur
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:cron]
+command=/usr/sbin/crond -f
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docs/quickstart/index.rst

@@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur:
 Installing Build Dependencies
 -----------------------------
 
-If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
+If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
 
 .. code-block:: bash
 
@@ -117,7 +117,7 @@ Simply run:
 
 .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command.
 
-You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment.
+You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment.
 
 
 Update your configuration
@@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config
     LEMUR_DEFAULT_ORGANIZATION
     LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
 
-Setup Postgres
+Set Up Postgres
 --------------
 
 For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
@@ -193,10 +193,10 @@ Additional notifications can be created through the UI or API.  See :ref:`Creati
 .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 
-Setup a Reverse Proxy
+Set Up a Reverse Proxy
 ---------------------
 
-By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
+By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
 
 
 Proxying with Nginx

requirements-dev.txt

@@ -8,18 +8,19 @@ aspy.yaml==1.1.1          # via pre-commit
 bleach==3.0.2             # via readme-renderer
 cached-property==1.5.1    # via pre-commit
 certifi==2018.11.29       # via requests
-cfgv==1.1.0               # via pre-commit
+cfgv==1.4.0               # via pre-commit
 chardet==3.0.4            # via requests
 docutils==0.14            # via readme-renderer
 flake8==3.5.0
-identify==1.1.7           # via pre-commit
+identify==1.1.8           # via pre-commit
 idna==2.8                 # via requests
-importlib-metadata==0.7   # via pre-commit
+importlib-metadata==0.8   # via pre-commit
+importlib-resources==1.0.2  # via pre-commit
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.3
-pkginfo==1.4.2            # via twine
+pkginfo==1.5.0            # via twine
-pre-commit==1.12.0
+pre-commit==1.13.0
 pycodestyle==2.3.1        # via flake8
 pyflakes==1.6.0           # via flake8
 pygments==2.3.1           # via readme-renderer
@@ -29,8 +30,9 @@ requests-toolbelt==0.8.0  # via twine
 requests==2.21.0          # via requests-toolbelt, twine
 six==1.12.0               # via bleach, cfgv, pre-commit, readme-renderer
 toml==0.10.0              # via pre-commit
-tqdm==4.28.1              # via twine
+tqdm==4.29.0              # via twine
 twine==1.12.1
 urllib3==1.24.1           # via requests
-virtualenv==16.1.0        # via pre-commit
+virtualenv==16.2.0        # via pre-commit
 webencodings==0.5.1       # via bleach
+zipp==0.3.3               # via importlib-metadata

requirements-docs.txt

@@ -4,21 +4,21 @@
 #
 #    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
-acme==0.29.1
+acme==0.30.0
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.5
 amqp==2.3.2
 aniso8601==4.0.1
-arrow==0.12.1
+arrow==0.13.0
 asn1crypto==0.24.0
 asyncpool==1.0
 babel==2.6.0              # via sphinx
-bcrypt==3.1.4
+bcrypt==3.1.5
 billiard==3.5.0.5
 blinker==1.4
-boto3==1.9.60
+boto3==1.9.75
-botocore==1.12.60
+botocore==1.12.75
 celery[redis]==4.2.1
 certifi==2018.11.29
 cffi==1.11.5
@@ -35,13 +35,13 @@ flask-cors==3.0.7
 flask-mail==0.9.1
 flask-migrate==2.3.1
 flask-principal==0.4.0
-flask-restful==0.3.6
+flask-restful==0.3.7
 flask-script==2.0.6
 flask-sqlalchemy==2.3.2
 flask==1.0.2
 future==0.17.1
 gunicorn==19.9.0
-idna==2.7
+idna==2.8
 imagesize==1.1.0          # via sphinx
 inflection==0.3.1
 itsdangerous==1.1.0
@@ -49,12 +49,12 @@ jinja2==2.10
 jmespath==0.9.3
 josepy==1.1.0
 jsonlines==1.2.0
-kombu==4.2.2
+kombu==4.2.2.post1
 lockfile==0.12.2
 mako==1.0.7
 markupsafe==1.1.0
 marshmallow-sqlalchemy==0.15.0
-marshmallow==2.16.3
+marshmallow==2.17.0
 mock==2.0.0
 ndg-httpsclient==0.5.1
 packaging==18.0           # via sphinx
@@ -62,35 +62,35 @@ paramiko==2.4.2
 pbr==5.1.1
 pem==18.2.0
 psycopg2==2.7.6.1
-pyasn1-modules==0.2.2
+pyasn1-modules==0.2.3
-pyasn1==0.4.4
+pyasn1==0.4.5
 pycparser==2.19
 pygments==2.3.1           # via sphinx
-pyjwt==1.7.0
+pyjwt==1.7.1
 pynacl==1.3.0
 pyopenssl==18.0.0
 pyparsing==2.3.0          # via packaging
 pyrfc3339==1.1
 python-dateutil==2.7.5
 python-editor==1.0.3
-pytz==2018.7
+pytz==2018.9
 pyyaml==3.13
-raven[flask]==6.9.0
+raven[flask]==6.10.0
 redis==2.10.6
 requests-toolbelt==0.8.0
-requests[security]==2.20.1
+requests[security]==2.21.0
 retrying==1.3.3
 s3transfer==0.1.13
-six==1.11.0
+six==1.12.0
 snowballstemmer==1.2.1    # via sphinx
 sphinx-rtd-theme==0.4.2
-sphinx==1.8.2
+sphinx==1.8.3
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx
-sqlalchemy-utils==0.33.9
+sqlalchemy-utils==0.33.10
-sqlalchemy==1.2.14
+sqlalchemy==1.2.15
 tabulate==0.8.2
 urllib3==1.24.1
-vine==1.1.4
+vine==1.2.0
 werkzeug==0.14.1
 xmltodict==0.11.0

requirements-tests.txt

@@ -8,9 +8,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.9.67             # via moto
+boto3==1.9.75             # via moto
 boto==2.49.0              # via moto
-botocore==1.12.67         # via boto3, moto, s3transfer
+botocore==1.12.75         # via boto3, moto, s3transfer
 certifi==2018.11.29       # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -34,7 +34,7 @@ jsondiff==1.1.1           # via moto
 jsonpickle==1.0           # via aws-xray-sdk
 markupsafe==1.1.0         # via jinja2
 mock==2.0.0               # via moto
-more-itertools==4.3.0     # via pytest
+more-itertools==5.0.0     # via pytest
 moto==1.3.7
 nose==1.3.7
 pbr==5.1.1                # via mock
@@ -46,10 +46,10 @@ pycryptodome==3.7.2       # via python-jose
 pyflakes==2.0.0
 pytest-flask==0.14.0
 pytest-mock==1.10.0
-pytest==4.0.2
+pytest==4.1.0
 python-dateutil==2.7.5    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
-pytz==2018.7              # via moto
+pytz==2018.9              # via moto
 pyyaml==3.13              # via pyaml
 requests-mock==1.5.2
 requests==2.21.0          # via aws-xray-sdk, docker, moto, requests-mock, responses

requirements.txt

@@ -4,19 +4,19 @@
 #
 #    pip-compile --no-index --output-file requirements.txt requirements.in
 #
-acme==0.29.1
+acme==0.30.0
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.5            # via flask-migrate
 amqp==2.3.2               # via kombu
 aniso8601==4.0.1          # via flask-restful
-arrow==0.12.1
+arrow==0.13.0
 asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.5             # via flask-bcrypt, paramiko
 billiard==3.5.0.5         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.9.67
+boto3==1.9.75
-botocore==1.12.67
+botocore==1.12.75
 celery[redis]==4.2.1
 certifi==2018.11.29
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
@@ -46,20 +46,20 @@ jinja2==2.10
 jmespath==0.9.3           # via boto3, botocore
 josepy==1.1.0             # via acme
 jsonlines==1.2.0          # via cloudflare
-kombu==4.2.2              # via celery
+kombu==4.2.2.post1        # via celery
 lockfile==0.12.2
 mako==1.0.7               # via alembic
 markupsafe==1.1.0         # via jinja2, mako
 marshmallow-sqlalchemy==0.15.0
-marshmallow==2.16.3
+marshmallow==2.17.0
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
 paramiko==2.4.2
 pbr==5.1.1                # via mock
 pem==18.2.0
 psycopg2==2.7.6.1
-pyasn1-modules==0.2.2     # via python-ldap
+pyasn1-modules==0.2.3     # via python-ldap
-pyasn1==0.4.4             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
+pyasn1==0.4.5             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
 pycparser==2.19           # via cffi
 pyjwt==1.7.1
 pynacl==1.3.0             # via paramiko
@@ -68,19 +68,19 @@ pyrfc3339==1.1            # via acme
 python-dateutil==2.7.5    # via alembic, arrow, botocore
 python-editor==1.0.3      # via alembic
 python-ldap==3.1.0
-pytz==2018.7              # via acme, celery, flask-restful, pyrfc3339
+pytz==2018.9              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==3.13              # via cloudflare
-raven[flask]==6.9.0
+raven[flask]==6.10.0
 redis==2.10.6
 requests-toolbelt==0.8.0  # via acme
 requests[security]==2.21.0
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.12.0
-sqlalchemy-utils==0.33.9
+sqlalchemy-utils==0.33.10
 sqlalchemy==1.2.15        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
 urllib3==1.24.1           # via botocore, requests
-vine==1.1.4               # via amqp
+vine==1.2.0               # via amqp
 werkzeug==0.14.1          # via flask
 xmltodict==0.11.0