From 78f9c490ddedfd3b9a1efd8dfe9c8c9b92f752fc Mon Sep 17 00:00:00 2001 From: Ilya Labun Date: Mon, 13 Jan 2020 15:26:35 +0100 Subject: [PATCH 01/10] Fix Dockercompose for tests --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b9439be7..fc83a034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apt-get update RUN apt-get install -y make software-properties-common curl RUN curl -sL https://deb.nodesource.com/setup_7.x | bash - RUN apt-get update -RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev +RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev RUN pip install -U setuptools RUN pip install coveralls bandit WORKDIR /app From 58d8a145c30e117218edef6363f1c48d84004dce Mon Sep 17 00:00:00 2001 From: pmelse Date: Mon, 13 Jan 2020 22:13:30 -0500 Subject: [PATCH 02/10] update for #2857 workaround update for #2857 workaround --- docs/quickstart/index.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 01a5c7ca..82bfc357 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -180,6 +180,13 @@ Lemur provides a helpful command that will initialize your database for you. It In addition to creating a new user, Lemur also creates a few default email notifications. These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``. They basically guarantee that every certificate within Lemur will send one expiration notification to the security team. +Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user. + +.. code-block:: bash + sudo -u postgres -i + psql + postgres=# ALTER USER lemur WITH SUPERUSER + Additional notifications can be created through the UI or API. See :ref:`Creating Notifications ` and :ref:`Command Line Interface ` for details. **Make note of the password used as this will be used during first login to the Lemur UI.** @@ -189,10 +196,16 @@ Additional notifications can be created through the UI or API. See :ref:`Creati cd /www/lemur/lemur lemur init +.. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now. + +.. code-block:: bash + sudo -u postgres -i + psql + postgres=# ALTER USER lemur WITH NOSUPERUSER + .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. - Set Up a Reverse Proxy --------------------- From d6f41b6a99d8fedfc0b7505a95a921a2add22466 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 16 Jan 2020 13:45:13 -0800 Subject: [PATCH 03/10] improving string formatting to avoid dangling white spaces and new lines --- lemur/plugins/lemur_aws/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 98b01672..6669f641 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -212,7 +212,7 @@ class AWSSourcePlugin(SourcePlugin): if not regions: regions = ec2.get_regions(account_number=account_number) else: - regions = regions.split(",") + regions = "".join(regions.split()).split(",") for region in regions: elbs = elb.get_all_elbs(account_number=account_number, region=region) From 71f43dfcc13bb68d488a5b322bbf5f45e1447fa4 Mon Sep 17 00:00:00 2001 From: Gutttlt <43376523+Gutttlt@users.noreply.github.com> Date: Tue, 21 Jan 2020 08:40:54 +0100 Subject: [PATCH 04/10] Fixing "'Role' object has no attribute 'set_third_party'" error. --- lemur/auth/ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index f4ceab03..ed87b76c 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -105,7 +105,7 @@ class LdapPrincipal: role = role_service.get_by_name(self.ldap_default_role) if role: if not role.third_party: - role = role.set_third_party(role.id, third_party_status=True) + role = role_service.set_third_party(role.id, third_party_status=True) roles.add(role) # update their 'roles' From 9984470b5846fbbe956e9b4432b1b01ca905d4c9 Mon Sep 17 00:00:00 2001 From: rajatsharma94 Date: Thu, 23 Jan 2020 12:35:57 +0100 Subject: [PATCH 05/10] fix fatal error in schema validator --- lemur/certificates/schemas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index c987e5fa..8f15542d 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -119,6 +119,9 @@ class CertificateInputSchema(CertificateCreationSchema): @validates_schema def validate_authority(self, data): + if 'authority' not in data: + raise ValidationError("Missing Authority.") + if isinstance(data["authority"], str): raise ValidationError("Authority not found.") From 620f972635b9b16be68c0d3ba0c2199640fa030a Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 27 Jan 2020 11:04:49 +0100 Subject: [PATCH 06/10] Fixed an error Found out that I introduced an error when I changed code up for publishig. The certserv.py I use does not return the ID of the certificate created. For now I just leave the field empty. I will create another issue , so that the ID is filled up. --- lemur/plugins/lemur_adcs/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_adcs/plugin.py b/lemur/plugins/lemur_adcs/plugin.py index bc07ede3..a69afc90 100644 --- a/lemur/plugins/lemur_adcs/plugin.py +++ b/lemur/plugins/lemur_adcs/plugin.py @@ -46,7 +46,7 @@ class ADCSIssuerPlugin(IssuerPlugin): ) current_app.logger.info("Requesting CSR: {0}".format(csr)) current_app.logger.info("Issuer options: {0}".format(issuer_options)) - cert, req_id = ( + cert = ( ca_server.get_cert(csr, adcs_template, encoding="b64") .decode("utf-8") .replace("\r\n", "\n") @@ -54,7 +54,7 @@ class ADCSIssuerPlugin(IssuerPlugin): chain = ( ca_server.get_ca_cert(encoding="b64").decode("utf-8").replace("\r\n", "\n") ) - return cert, chain, req_id + return cert, chain, None def revoke_certificate(self, certificate, comments): raise NotImplementedError("Not implemented\n", self, certificate, comments) From 192ecb3ce0391267b6e43b6bc3e19ebd3aeecaf7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 28 Jan 2020 16:24:50 -0800 Subject: [PATCH 07/10] DNS provider: adding more logging --- lemur/dns_providers/cli.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py index 72f9c874..aa0d97f5 100644 --- a/lemur/dns_providers/cli.py +++ b/lemur/dns_providers/cli.py @@ -1,8 +1,10 @@ from flask_script import Manager +import sys + from lemur.constants import SUCCESS_METRIC_STATUS from lemur.dns_providers.service import get_all_dns_providers, set_domains -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.plugins.base import plugins manager = Manager( @@ -19,12 +21,20 @@ def get_all_zones(): dns_providers = get_all_dns_providers() acme_plugin = plugins.get("acme-issuer") + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "", + } + for dns_provider in dns_providers: try: zones = acme_plugin.get_all_zones(dns_provider) set_domains(dns_provider, zones) except Exception as e: print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e)) + log_data["message"] = f"get all zones failed for {dns_provider} {e}." + sentry.captureException(extra=log_data) set_domains(dns_provider, []) status = SUCCESS_METRIC_STATUS From b885244aa77d571911121493c8700b35b3d868c2 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 29 Jan 2020 11:26:53 -0800 Subject: [PATCH 08/10] fixing issue where set_domains() is still called when get_all_zones() throws an exception --- lemur/dns_providers/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py index aa0d97f5..b14e0339 100644 --- a/lemur/dns_providers/cli.py +++ b/lemur/dns_providers/cli.py @@ -35,7 +35,6 @@ def get_all_zones(): print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e)) log_data["message"] = f"get all zones failed for {dns_provider} {e}." sentry.captureException(extra=log_data) - set_domains(dns_provider, []) status = SUCCESS_METRIC_STATUS From 48d8e1d235e04fb4370006e7f7da02a15d1a30cd Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 29 Jan 2020 15:30:08 -0800 Subject: [PATCH 09/10] adding support for pip version >=19.3 by supporting change to PipSession() location in setup.py --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90c0b2f8..fa5a23bc 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,11 @@ from setuptools import setup, find_packages from subprocess import check_output import pip -if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0): +if tuple(map(int, pip.__version__.split('.'))) >= (19, 3, 0): + from pip._internal.network.session import PipSession + from pip._internal.req import parse_requirements + +elif tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0): from pip._internal.download import PipSession from pip._internal.req import parse_requirements else: From 75d4699c7a48f1e755a68b009ed7a3607b10a966 Mon Sep 17 00:00:00 2001 From: Ilya Makarov Date: Fri, 31 Jan 2020 22:52:59 +0300 Subject: [PATCH 10/10] Fix nginx ssl. Add env vars. Opt docker --- docker/.dockerignore | 3 +++ docker/Dockerfile | 14 +++++++------- docker/docker-compose.yml | 29 +++++++++++++++++++++++++++++ docker/entrypoint | 17 ++++++++++------- docker/lemur-env | 25 +++++++++++++++++++++++++ docker/nginx/default-ssl.conf | 2 +- docker/pgsql-env | 4 ++++ docker/src/lemur.conf.py | 12 ++++++++++++ 8 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 docker/.dockerignore create mode 100644 docker/docker-compose.yml create mode 100644 docker/lemur-env create mode 100644 docker/pgsql-env diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 00000000..2199292b --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1,3 @@ +*-env +docker-compose.yml +Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile index f7d1caf7..5c80606f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,12 +8,6 @@ ENV gid 1337 ENV user lemur ENV group lemur -COPY entrypoint / -COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py -COPY supervisor.conf / -COPY nginx/default.conf /etc/nginx/conf.d/ -COPY nginx/default-ssl.conf /etc/nginx/conf.d/ - RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \ @@ -40,7 +34,6 @@ RUN addgroup -S ${group} -g ${gid} && \ curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ - chmod +x /entrypoint && \ mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ @@ -52,6 +45,13 @@ RUN npm install --unsafe-perm && \ node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \ apk del build-dependencies +COPY entrypoint / +COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py +COPY supervisor.conf / +COPY nginx/default.conf /etc/nginx/conf.d/ +COPY nginx/default-ssl.conf /etc/nginx/conf.d/ + +RUN chmod +x /entrypoint WORKDIR / HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..77293f43 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,29 @@ +version: '3' + +services: + postgres: + image: "postgres:10" + restart: always + volumes: + - pg_data:/var/lib/postgresql/data + env_file: + - pgsql-env + + lemur: + # image: "netlix-lemur:latest" + build: . + depends_on: + - postgres + - redis + env_file: + - lemur-env + - pgsql-env + ports: + - 80:80 + - 443:443 + + redis: + image: "redis:alpine" + +volumes: + pg_data: {} diff --git a/docker/entrypoint b/docker/entrypoint index 6077167a..2a3a84e3 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +set -eo pipefail if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then echo "Database vars not set" @@ -7,22 +9,23 @@ fi export POSTGRES_PORT="${POSTGRES_PORT:-5432}" -echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile +export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}" + +export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" -source /etc/profile PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' echo " # Create Postgres trgm extension" -PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;' echo " # Done" if [ -z "${SKIP_SSL}" ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi - mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf - mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a + [ -f "/etc/nginx/conf.d/default-ssl.conf.a" ] && mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf + [ -f "/etc/nginx/conf.d/default.conf" ] && mv -f /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then @@ -33,7 +36,7 @@ fi # fi echo " # Running init" -su lemur -c "python3 /opt/lemur/lemur/manage.py init" +su lemur -s /bin/bash -c "cd /opt/lemur/lemur; python3 /opt/lemur/lemur/manage.py init -p ${LEMUR_ADMIN_PASSWORD}" echo " # Done" # echo "Creating user" diff --git a/docker/lemur-env b/docker/lemur-env new file mode 100644 index 00000000..419a9858 --- /dev/null +++ b/docker/lemur-env @@ -0,0 +1,25 @@ +# SKIP_SSL=1 +# LEMUR_TOKEN_SECRET= +# LEMUR_DEFAULT_COUNTRY= +# LEMUR_DEFAULT_STATE= +# LEMUR_DEFAULT_LOCATION= +# LEMUR_DEFAULT_ORGANIZATION= +# LEMUR_DEFAULT_ORGANIZATIONAL_UNIT= +# LEMUR_DEFAULT_ISSUER_PLUGIN=cryptography-issuer +# LEMUR_DEFAULT_AUTHORITY=cryptography +# MAIL_SERVER=mail.example.com +# MAIL_PORT=25 +# LEMUR_EMAIL=lemur@example.com +# LEMUR_SECURITY_TEAM_EMAIL=['team@example.com'] +# LEMUR_TOKEN_SECRET= +# LEMUR_ENCRYPTION_KEYS=[''] +# DEBUG=True +# LDAP_DEBUG=True +# LDAP_AUTH=True +# LDAP_BIND_URI=ldap://example.com +# LDAP_BASE_DN=DC=example,DC=com +# LDAP_EMAIL_DOMAIN=example.com +# LDAP_USE_TLS=False +# LDAP_REQUIRED_GROUP=certificate-management-admins +# LDAP_GROUPS_TO_ROLES={'certificate-management-admins': 'admin', 'Team': 'team@example.com'} +# LDAP_IS_ACTIVE_DIRECTORY=False diff --git a/docker/nginx/default-ssl.conf b/docker/nginx/default-ssl.conf index 86c770df..43d40f38 100644 --- a/docker/nginx/default-ssl.conf +++ b/docker/nginx/default-ssl.conf @@ -9,7 +9,7 @@ server { } server { - listen 443; + listen 443 ssl; server_name _; access_log /dev/stdout; error_log /dev/stderr; diff --git a/docker/pgsql-env b/docker/pgsql-env new file mode 100644 index 00000000..70d73fcb --- /dev/null +++ b/docker/pgsql-env @@ -0,0 +1,4 @@ +POSTGRES_USER=lemur +POSTGRES_PASSWORD=12345 +POSTGRES_DB=lemur +POSTGRES_HOST=postgres diff --git a/docker/src/lemur.conf.py b/docker/src/lemur.conf.py index a5f7e8b6..0f294b28 100644 --- a/docker/src/lemur.conf.py +++ b/docker/src/lemur.conf.py @@ -1,4 +1,6 @@ import os +from ast import literal_eval + _basedir = os.path.abspath(os.path.dirname(__file__)) CORS = os.environ.get("CORS") == "True" @@ -29,3 +31,13 @@ LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log')) SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') + +LDAP_DEBUG = os.environ.get('LDAP_DEBUG') == "True" +LDAP_AUTH = os.environ.get('LDAP_AUTH') == "True" +LDAP_IS_ACTIVE_DIRECTORY = os.environ.get('LDAP_IS_ACTIVE_DIRECTORY') == "True" +LDAP_BIND_URI = str(os.environ.get('LDAP_BIND_URI','')) +LDAP_BASE_DN = str(os.environ.get('LDAP_BASE_DN','')) +LDAP_EMAIL_DOMAIN = str(os.environ.get('LDAP_EMAIL_DOMAIN','')) +LDAP_USE_TLS = str(os.environ.get('LDAP_USE_TLS','')) +LDAP_REQUIRED_GROUP = str(os.environ.get('LDAP_REQUIRED_GROUP','')) +LDAP_GROUPS_TO_ROLES = literal_eval(os.environ.get('LDAP_GROUPS_TO_ROLES') or "{}")