Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into verify-cert-private-key

This commit is contained in:
Curtis 2019-01-08 10:53:28 -08:00 committed by GitHub
parent 542e953919 3637410327
commit cdf9a53a40
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 296 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -113,10 +113,10 @@ endif
@echo "--> Updating Python requirements"
pip install --upgrade pip
pip install --upgrade pip-tools
pip-compile --output-file requirements.txt requirements.in -U --no-index
pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
pip-compile --output-file requirements.txt requirements.in -U --no-index
@echo "--> Done updating Python requirements"
@echo "--> Removing python-ldap from requirements-docs.txt"
grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

64
docker/Dockerfile Normal file
View File

@ -0,0 +1,64 @@
FROM alpine:3.8
ARG VERSION
ENV VERSION master
ENV uid 1337
ENV gid 1337
ENV user lemur
ENV group lemur
COPY entrypoint /
COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
COPY supervisor.conf /
COPY nginx/default.conf /etc/nginx/conf.d/
COPY nginx/default-ssl.conf /etc/nginx/conf.d/
RUN addgroup -S ${group} -g ${gid} && \
adduser -D -S ${user} -G ${group} -u ${uid} && \
apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
apk --update add --virtual build-dependencies \
git \
tar \
curl \
python3-dev \
npm \
bash \
musl-dev \
gcc \
autoconf \
automake \
make \
nasm \
zlib-dev \
postgresql-dev \
libressl-dev \
libffi-dev \
cyrus-sasl-dev \
openldap-dev && \
mkdir -p /opt/lemur /home/lemur/.lemur/ && \
curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
pip3 install --upgrade pip && \
pip3 install --upgrade setuptools && \
chmod +x /entrypoint && \
mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
WORKDIR /opt/lemur
RUN npm install --unsafe-perm && \
pip3 install -e . && \
node_modules/.bin/gulp build && \
node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
apk del build-dependencies
WORKDIR /
HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \
CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
USER root
ENTRYPOINT ["/entrypoint"]
CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

54
docker/entrypoint Normal file
View File

@ -0,0 +1,54 @@
#!/bin/sh
if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
echo "Database vars not set"
exit 1
fi
export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
source /etc/profile
PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
echo " # Create Postgres trgm extension"
PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
echo " # Done"
if [ -z "${SKIP_SSL}" ]; then
if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
fi
mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
fi
# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
# echo "Creating config"
# https://github.com/Netflix/lemur/issues/2257
# python3 /opt/lemur/lemur/manage.py create_config
# echo "Done"
# fi
echo " # Running init"
su lemur -c "python3 /opt/lemur/lemur/manage.py init"
echo " # Done"
# echo "Creating user"
# https://github.com/Netflix/lemur/issues/
# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
# echo "Done"
cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
echo " # Populating crontab"
echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify
echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync
echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked
echo " # Done"
exec "$@"

37
docker/nginx/default-ssl.conf Normal file
View File

@ -0,0 +1,37 @@
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
server {
listen 80;
server_name _;
return 301 https://$host$request_uri;
}
server {
listen 443;
server_name _;
access_log /dev/stdout;
error_log /dev/stderr;
ssl_certificate /etc/nginx/ssl/server.crt;
ssl_certificate_key /etc/nginx/ssl/server.key;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
location /api {
proxy_pass http://127.0.0.1:8000;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
root /opt/lemur/lemur/static/dist;
include mime.types;
index index.html;
}
}

26
docker/nginx/default.conf Normal file
View File

@ -0,0 +1,26 @@
add_header X-Frame-Options DENY;
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
server {
listen 80;
access_log /dev/stdout;
error_log /dev/stderr;
location /api {
proxy_pass http://127.0.0.1:8000;
proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location / {
root /opt/lemur/lemur/static/dist;
include mime.types;
index index.html;
}
}

31
docker/src/lemur.conf.py Normal file
View File

@ -0,0 +1,31 @@
import os
_basedir = os.path.abspath(os.path.dirname(__file__))
CORS = os.environ.get("CORS") == "True"
debug = os.environ.get("DEBUG") == "True"
SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw=='))
LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ=='))
LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo='))
LEMUR_WHITELISTED_DOMAINS = []
LEMUR_EMAIL = ''
LEMUR_SECURITY_TEAM_EMAIL = []
LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE',''))
LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
ACTIVE_PROVIDERS = []
METRIC_PROVIDERS = []
LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')

32
docker/supervisor.conf Normal file
View File

@ -0,0 +1,32 @@
[supervisord]
nodaemon=true
user=root
logfile=/dev/stdout
logfile_maxbytes=0
pidfile = /tmp/supervisord.pid
[program:lemur]
environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000
user=lemur
directory=/opt/lemur/lemur
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:nginx]
command=/usr/sbin/nginx -g "daemon off;"
user=root
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:cron]
command=/usr/sbin/crond -f
user=root
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0

10
docs/quickstart/index.rst
View File

@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur:
Installing Build Dependencies
-----------------------------
If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
.. code-block:: bash
@ -117,7 +117,7 @@ Simply run:
.. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command.
You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment.
You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment.
Update your configuration
@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config
LEMUR_DEFAULT_ORGANIZATION
LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
Setup Postgres
Set Up Postgres
--------------
For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
@ -193,10 +193,10 @@ Additional notifications can be created through the UI or API. See :ref:`Creati
.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
Setup a Reverse Proxy
Set Up a Reverse Proxy
---------------------
By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
Proxying with Nginx

16
requirements-dev.txt
View File

@ -8,18 +8,19 @@ aspy.yaml==1.1.1 # via pre-commit
bleach==3.0.2 # via readme-renderer
cached-property==1.5.1 # via pre-commit
certifi==2018.11.29 # via requests
cfgv==1.1.0 # via pre-commit
cfgv==1.4.0 # via pre-commit
chardet==3.0.4 # via requests
docutils==0.14 # via readme-renderer
flake8==3.5.0
identify==1.1.7 # via pre-commit
identify==1.1.8 # via pre-commit
idna==2.8 # via requests
importlib-metadata==0.7 # via pre-commit
importlib-metadata==0.8 # via pre-commit
importlib-resources==1.0.2 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.3
pkginfo==1.4.2 # via twine
pre-commit==1.12.0
pkginfo==1.5.0 # via twine
pre-commit==1.13.0
pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8
pygments==2.3.1 # via readme-renderer
@ -29,8 +30,9 @@ requests-toolbelt==0.8.0 # via twine
requests==2.21.0 # via requests-toolbelt, twine
six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.28.1 # via twine
tqdm==4.29.0 # via twine
twine==1.12.1
urllib3==1.24.1 # via requests
virtualenv==16.1.0 # via pre-commit
virtualenv==16.2.0 # via pre-commit
webencodings==0.5.1 # via bleach
zipp==0.3.3 # via importlib-metadata

40
requirements-docs.txt
View File

@ -4,21 +4,21 @@
#
# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
#
acme==0.29.1
acme==0.30.0
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.5
amqp==2.3.2
aniso8601==4.0.1
arrow==0.12.1
arrow==0.13.0
asn1crypto==0.24.0
asyncpool==1.0
babel==2.6.0 # via sphinx
bcrypt==3.1.4
bcrypt==3.1.5
billiard==3.5.0.5
blinker==1.4
boto3==1.9.60
botocore==1.12.60
boto3==1.9.75
botocore==1.12.75
celery[redis]==4.2.1
certifi==2018.11.29
cffi==1.11.5
@ -35,13 +35,13 @@ flask-cors==3.0.7
flask-mail==0.9.1
flask-migrate==2.3.1
flask-principal==0.4.0
flask-restful==0.3.6
flask-restful==0.3.7
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
idna==2.7
idna==2.8
imagesize==1.1.0 # via sphinx
inflection==0.3.1
itsdangerous==1.1.0
@ -49,12 +49,12 @@ jinja2==2.10
jmespath==0.9.3
josepy==1.1.0
jsonlines==1.2.0
kombu==4.2.2
kombu==4.2.2.post1
lockfile==0.12.2
mako==1.0.7
markupsafe==1.1.0
marshmallow-sqlalchemy==0.15.0
marshmallow==2.16.3
marshmallow==2.17.0
mock==2.0.0
ndg-httpsclient==0.5.1
packaging==18.0 # via sphinx
@ -62,35 +62,35 @@ paramiko==2.4.2
pbr==5.1.1
pem==18.2.0
psycopg2==2.7.6.1
pyasn1-modules==0.2.2
pyasn1==0.4.4
pyasn1-modules==0.2.3
pyasn1==0.4.5
pycparser==2.19
pygments==2.3.1 # via sphinx
pyjwt==1.7.0
pyjwt==1.7.1
pynacl==1.3.0
pyopenssl==18.0.0
pyparsing==2.3.0 # via packaging
pyrfc3339==1.1
python-dateutil==2.7.5
python-editor==1.0.3
pytz==2018.7
pytz==2018.9
pyyaml==3.13
raven[flask]==6.9.0
raven[flask]==6.10.0
redis==2.10.6
requests-toolbelt==0.8.0
requests[security]==2.20.1
requests[security]==2.21.0
retrying==1.3.3
s3transfer==0.1.13
six==1.11.0
six==1.12.0
snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.4.2
sphinx==1.8.2
sphinx==1.8.3
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx
sqlalchemy-utils==0.33.9
sqlalchemy==1.2.14
sqlalchemy-utils==0.33.10
sqlalchemy==1.2.15
tabulate==0.8.2
urllib3==1.24.1
vine==1.1.4
vine==1.2.0
werkzeug==0.14.1
xmltodict==0.11.0

10
requirements-tests.txt
View File

@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.2.1 # via pytest
attrs==18.2.0 # via pytest
aws-xray-sdk==0.95 # via moto
boto3==1.9.67 # via moto
boto3==1.9.75 # via moto
boto==2.49.0 # via moto
botocore==1.12.67 # via boto3, moto, s3transfer
botocore==1.12.75 # via boto3, moto, s3transfer
certifi==2018.11.29 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
@ -34,7 +34,7 @@ jsondiff==1.1.1 # via moto
jsonpickle==1.0 # via aws-xray-sdk
markupsafe==1.1.0 # via jinja2
mock==2.0.0 # via moto
more-itertools==4.3.0 # via pytest
more-itertools==5.0.0 # via pytest
moto==1.3.7
nose==1.3.7
pbr==5.1.1 # via mock
@ -46,10 +46,10 @@ pycryptodome==3.7.2 # via python-jose
pyflakes==2.0.0
pytest-flask==0.14.0
pytest-mock==1.10.0
pytest==4.0.2
pytest==4.1.0
python-dateutil==2.7.5 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.7 # via moto
pytz==2018.9 # via moto
pyyaml==3.13 # via pyaml
requests-mock==1.5.2
requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses

24
requirements.txt
View File

@ -4,19 +4,19 @@
#
# pip-compile --no-index --output-file requirements.txt requirements.in
#
acme==0.29.1
acme==0.30.0
alembic-autogenerate-enums==0.0.2
alembic==1.0.5 # via flask-migrate
amqp==2.3.2 # via kombu
aniso8601==4.0.1 # via flask-restful
arrow==0.12.1
arrow==0.13.0
asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
bcrypt==3.1.5 # via flask-bcrypt, paramiko
billiard==3.5.0.5 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.9.67
botocore==1.12.67
boto3==1.9.75
botocore==1.12.75
celery[redis]==4.2.1
certifi==2018.11.29
cffi==1.11.5 # via bcrypt, cryptography, pynacl
@ -46,20 +46,20 @@ jinja2==2.10
jmespath==0.9.3 # via boto3, botocore
josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare
kombu==4.2.2 # via celery
kombu==4.2.2.post1 # via celery
lockfile==0.12.2
mako==1.0.7 # via alembic
markupsafe==1.1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.15.0
marshmallow==2.16.3
marshmallow==2.17.0
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.1.1 # via mock
pem==18.2.0
psycopg2==2.7.6.1
pyasn1-modules==0.2.2 # via python-ldap
pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
pyasn1-modules==0.2.3 # via python-ldap
pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
pycparser==2.19 # via cffi
pyjwt==1.7.1
pynacl==1.3.0 # via paramiko
@ -68,19 +68,19 @@ pyrfc3339==1.1 # via acme
python-dateutil==2.7.5 # via alembic, arrow, botocore
python-editor==1.0.3 # via alembic
python-ldap==3.1.0
pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339
pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339
pyyaml==3.13 # via cloudflare
raven[flask]==6.9.0
raven[flask]==6.10.0
redis==2.10.6
requests-toolbelt==0.8.0 # via acme
requests[security]==2.21.0
retrying==1.3.3
s3transfer==0.1.13 # via boto3
six==1.12.0
sqlalchemy-utils==0.33.9
sqlalchemy-utils==0.33.10
sqlalchemy==1.2.15 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2
urllib3==1.24.1 # via botocore, requests
vine==1.1.4 # via amqp
vine==1.2.0 # via amqp
werkzeug==0.14.1 # via flask
xmltodict==0.11.0