Merge branch 'master' into powerdnsplugin_01

Dockerfile

@@ -3,7 +3,7 @@ RUN apt-get update
 RUN apt-get install -y make software-properties-common curl
 RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
 RUN apt-get update
-RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev
 RUN pip install -U setuptools
 RUN pip install coveralls bandit
 WORKDIR /app

docker/.dockerignore

@@ -0,0 +1,3 @@
+*-env
+docker-compose.yml
+Dockerfile

docker/Dockerfile

@@ -8,12 +8,6 @@ ENV gid 1337
 ENV user lemur
 ENV group lemur
 
-COPY entrypoint /
-COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
-COPY supervisor.conf /
-COPY nginx/default.conf /etc/nginx/conf.d/
-COPY nginx/default-ssl.conf /etc/nginx/conf.d/
-
 RUN addgroup -S ${group} -g ${gid} && \
     adduser -D -S ${user} -G ${group} -u ${uid} && \
     apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
@@ -40,7 +34,6 @@ RUN addgroup -S ${group} -g ${gid} && \
     curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
     pip3 install --upgrade pip && \
     pip3 install --upgrade setuptools && \
-    chmod +x /entrypoint && \
     mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
     chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
     
@@ -52,6 +45,13 @@ RUN npm install --unsafe-perm && \
     node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
     apk del build-dependencies
 
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
 WORKDIR /
 
 HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  

docker/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  postgres:
+    image: "postgres:10"
+    restart: always
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    env_file:
+      - pgsql-env
+
+  lemur:
+    # image: "netlix-lemur:latest"
+    build: .
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - lemur-env
+      - pgsql-env
+    ports:
+      - 80:80
+      - 443:443
+
+  redis:
+    image: "redis:alpine"
+
+volumes:
+    pg_data: {}

docker/entrypoint

@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+set -eo pipefail
 
 if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
   echo "Database vars not set"
@@ -7,22 +9,23 @@ fi
 
 export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
 
-echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
+export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}"
+
+export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
 
-source /etc/profile
 
 PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
 
 echo " # Create Postgres trgm extension"
-PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 echo " # Done"
 
 if [ -z "${SKIP_SSL}" ]; then
   if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
     openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
   fi
-  mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
-  mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+  [ -f "/etc/nginx/conf.d/default-ssl.conf.a" ] && mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  [ -f "/etc/nginx/conf.d/default.conf" ] && mv -f /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
 fi
 
 # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
@@ -33,7 +36,7 @@ fi
 # fi
 
 echo " # Running init"
-su lemur -c "python3 /opt/lemur/lemur/manage.py init"
+su lemur -s /bin/bash -c "cd /opt/lemur/lemur; python3 /opt/lemur/lemur/manage.py init -p ${LEMUR_ADMIN_PASSWORD}"
 echo " # Done"
 
 # echo "Creating user"

docker/lemur-env

@@ -0,0 +1,25 @@
+# SKIP_SSL=1
+# LEMUR_TOKEN_SECRET=
+# LEMUR_DEFAULT_COUNTRY=
+# LEMUR_DEFAULT_STATE=
+# LEMUR_DEFAULT_LOCATION=
+# LEMUR_DEFAULT_ORGANIZATION=
+# LEMUR_DEFAULT_ORGANIZATIONAL_UNIT=
+# LEMUR_DEFAULT_ISSUER_PLUGIN=cryptography-issuer
+# LEMUR_DEFAULT_AUTHORITY=cryptography
+# MAIL_SERVER=mail.example.com
+# MAIL_PORT=25
+# LEMUR_EMAIL=lemur@example.com
+# LEMUR_SECURITY_TEAM_EMAIL=['team@example.com']
+# LEMUR_TOKEN_SECRET=
+# LEMUR_ENCRYPTION_KEYS=['']
+# DEBUG=True
+# LDAP_DEBUG=True
+# LDAP_AUTH=True
+# LDAP_BIND_URI=ldap://example.com
+# LDAP_BASE_DN=DC=example,DC=com
+# LDAP_EMAIL_DOMAIN=example.com
+# LDAP_USE_TLS=False
+# LDAP_REQUIRED_GROUP=certificate-management-admins
+# LDAP_GROUPS_TO_ROLES={'certificate-management-admins': 'admin', 'Team': 'team@example.com'}
+# LDAP_IS_ACTIVE_DIRECTORY=False

docker/nginx/default-ssl.conf

@@ -9,7 +9,7 @@ server {
 }
 
 server {
-   listen 443;
+   listen 443 ssl;
    server_name  _;
    access_log /dev/stdout;
    error_log /dev/stderr;

docker/pgsql-env

@@ -0,0 +1,4 @@
+POSTGRES_USER=lemur
+POSTGRES_PASSWORD=12345
+POSTGRES_DB=lemur
+POSTGRES_HOST=postgres

docker/src/lemur.conf.py

@@ -1,4 +1,6 @@
 import os
+from ast import literal_eval
+
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
 CORS = os.environ.get("CORS") == "True"
@@ -29,3 +31,13 @@ LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
 LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
 
 SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')
+
+LDAP_DEBUG = os.environ.get('LDAP_DEBUG') == "True"
+LDAP_AUTH = os.environ.get('LDAP_AUTH') == "True"
+LDAP_IS_ACTIVE_DIRECTORY = os.environ.get('LDAP_IS_ACTIVE_DIRECTORY') == "True"
+LDAP_BIND_URI = str(os.environ.get('LDAP_BIND_URI',''))
+LDAP_BASE_DN = str(os.environ.get('LDAP_BASE_DN',''))
+LDAP_EMAIL_DOMAIN = str(os.environ.get('LDAP_EMAIL_DOMAIN',''))
+LDAP_USE_TLS = str(os.environ.get('LDAP_USE_TLS',''))
+LDAP_REQUIRED_GROUP = str(os.environ.get('LDAP_REQUIRED_GROUP',''))
+LDAP_GROUPS_TO_ROLES = literal_eval(os.environ.get('LDAP_GROUPS_TO_ROLES') or "{}")

docs/quickstart/index.rst

@@ -180,6 +180,13 @@ Lemur provides a helpful command that will initialize your database for you. It
 
 In addition to creating a new user, Lemur also creates a few default email notifications.  These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``.  They basically guarantee that every certificate within Lemur will send one expiration notification to the security team.
 
+Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user.
+
+.. code-block:: bash
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH SUPERUSER
+
 Additional notifications can be created through the UI or API.  See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 **Make note of the password used as this will be used during first login to the Lemur UI.**
@@ -189,10 +196,16 @@ Additional notifications can be created through the UI or API.  See :ref:`Creati
     cd /www/lemur/lemur
     lemur init
 
+.. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now.
+
+.. code-block:: bash
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH NOSUPERUSER
+
 
 .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
-
 Set Up a Reverse Proxy
 ---------------------
 

lemur/auth/ldap.py

@@ -105,7 +105,7 @@ class LdapPrincipal:
             role = role_service.get_by_name(self.ldap_default_role)
             if role:
                 if not role.third_party:
-                    role = role.set_third_party(role.id, third_party_status=True)
+                    role = role_service.set_third_party(role.id, third_party_status=True)
                 roles.add(role)
 
         # update their 'roles'

lemur/certificates/schemas.py

@@ -119,6 +119,9 @@ class CertificateInputSchema(CertificateCreationSchema):
 
     @validates_schema
     def validate_authority(self, data):
+        if 'authority' not in data:
+            raise ValidationError("Missing Authority.")
+
         if isinstance(data["authority"], str):
             raise ValidationError("Authority not found.")
 

lemur/dns_providers/cli.py

@@ -1,8 +1,10 @@
 from flask_script import Manager
 
+import sys
+
 from lemur.constants import SUCCESS_METRIC_STATUS
 from lemur.dns_providers.service import get_all_dns_providers, set_domains
-from lemur.extensions import metrics
+from lemur.extensions import metrics, sentry
 from lemur.plugins.base import plugins
 
 manager = Manager(
@@ -19,13 +21,20 @@ def get_all_zones():
     dns_providers = get_all_dns_providers()
     acme_plugin = plugins.get("acme-issuer")
 
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    log_data = {
+        "function": function,
+        "message": "",
+    }
+
     for dns_provider in dns_providers:
         try:
             zones = acme_plugin.get_all_zones(dns_provider)
             set_domains(dns_provider, zones)
         except Exception as e:
             print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e))
-            set_domains(dns_provider, [])
+            log_data["message"] = f"get all zones failed for {dns_provider} {e}."
+            sentry.captureException(extra=log_data)
 
     status = SUCCESS_METRIC_STATUS
 

lemur/plugins/lemur_adcs/plugin.py

@@ -46,7 +46,7 @@ class ADCSIssuerPlugin(IssuerPlugin):
         )
         current_app.logger.info("Requesting CSR: {0}".format(csr))
         current_app.logger.info("Issuer options: {0}".format(issuer_options))
-        cert, req_id = (
+        cert = (
             ca_server.get_cert(csr, adcs_template, encoding="b64")
             .decode("utf-8")
             .replace("\r\n", "\n")
@@ -54,7 +54,7 @@ class ADCSIssuerPlugin(IssuerPlugin):
         chain = (
             ca_server.get_ca_cert(encoding="b64").decode("utf-8").replace("\r\n", "\n")
         )
-        return cert, chain, req_id
+        return cert, chain, None
 
     def revoke_certificate(self, certificate, comments):
         raise NotImplementedError("Not implemented\n", self, certificate, comments)

lemur/plugins/lemur_aws/plugin.py

@@ -212,7 +212,7 @@ class AWSSourcePlugin(SourcePlugin):
         if not regions:
             regions = ec2.get_regions(account_number=account_number)
         else:
-            regions = regions.split(",")
+            regions = "".join(regions.split()).split(",")
 
         for region in regions:
             elbs = elb.get_all_elbs(account_number=account_number, region=region)

setup.py

@@ -23,7 +23,11 @@ from setuptools import setup, find_packages
 from subprocess import check_output
 
 import pip
-if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0):
+if tuple(map(int, pip.__version__.split('.'))) >= (19, 3, 0):
+    from pip._internal.network.session import PipSession
+    from pip._internal.req import parse_requirements
+
+elif tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0):
     from pip._internal.download import PipSession
     from pip._internal.req import parse_requirements
 else: