Merge branch 'master' into ilabun/optimize-certificates-sql

docker/.dockerignore

@@ -0,0 +1,3 @@
+*-env
+docker-compose.yml
+Dockerfile

docker/Dockerfile

@@ -8,12 +8,6 @@ ENV gid 1337
 ENV user lemur
 ENV group lemur
 
-COPY entrypoint /
-COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
-COPY supervisor.conf /
-COPY nginx/default.conf /etc/nginx/conf.d/
-COPY nginx/default-ssl.conf /etc/nginx/conf.d/
-
 RUN addgroup -S ${group} -g ${gid} && \
     adduser -D -S ${user} -G ${group} -u ${uid} && \
     apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
@@ -40,7 +34,6 @@ RUN addgroup -S ${group} -g ${gid} && \
     curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
     pip3 install --upgrade pip && \
     pip3 install --upgrade setuptools && \
-    chmod +x /entrypoint && \
     mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
     chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
     
@@ -52,6 +45,13 @@ RUN npm install --unsafe-perm && \
     node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
     apk del build-dependencies
 
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
 WORKDIR /
 
 HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  

docker/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  postgres:
+    image: "postgres:10"
+    restart: always
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    env_file:
+      - pgsql-env
+
+  lemur:
+    # image: "netlix-lemur:latest"
+    build: .
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - lemur-env
+      - pgsql-env
+    ports:
+      - 80:80
+      - 443:443
+
+  redis:
+    image: "redis:alpine"
+
+volumes:
+    pg_data: {}

docker/entrypoint

@@ -1,4 +1,6 @@
-#!/bin/sh
+#!/bin/bash
+
+set -eo pipefail
 
 if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
   echo "Database vars not set"
@@ -7,22 +9,23 @@ fi
 
 export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
 
-echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
+export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}"
+
+export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
 
-source /etc/profile
 
 PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
 
 echo " # Create Postgres trgm extension"
-PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 echo " # Done"
 
 if [ -z "${SKIP_SSL}" ]; then
   if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
     openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
   fi
-  mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  [ -f "/etc/nginx/conf.d/default-ssl.conf.a" ] && mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
-  mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+  [ -f "/etc/nginx/conf.d/default.conf" ] && mv -f /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
 fi
 
 # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
@@ -33,7 +36,7 @@ fi
 # fi
 
 echo " # Running init"
-su lemur -c "python3 /opt/lemur/lemur/manage.py init"
+su lemur -s /bin/bash -c "cd /opt/lemur/lemur; python3 /opt/lemur/lemur/manage.py init -p ${LEMUR_ADMIN_PASSWORD}"
 echo " # Done"
 
 # echo "Creating user"

docker/lemur-env

@@ -0,0 +1,25 @@
+# SKIP_SSL=1
+# LEMUR_TOKEN_SECRET=
+# LEMUR_DEFAULT_COUNTRY=
+# LEMUR_DEFAULT_STATE=
+# LEMUR_DEFAULT_LOCATION=
+# LEMUR_DEFAULT_ORGANIZATION=
+# LEMUR_DEFAULT_ORGANIZATIONAL_UNIT=
+# LEMUR_DEFAULT_ISSUER_PLUGIN=cryptography-issuer
+# LEMUR_DEFAULT_AUTHORITY=cryptography
+# MAIL_SERVER=mail.example.com
+# MAIL_PORT=25
+# LEMUR_EMAIL=lemur@example.com
+# LEMUR_SECURITY_TEAM_EMAIL=['team@example.com']
+# LEMUR_TOKEN_SECRET=
+# LEMUR_ENCRYPTION_KEYS=['']
+# DEBUG=True
+# LDAP_DEBUG=True
+# LDAP_AUTH=True
+# LDAP_BIND_URI=ldap://example.com
+# LDAP_BASE_DN=DC=example,DC=com
+# LDAP_EMAIL_DOMAIN=example.com
+# LDAP_USE_TLS=False
+# LDAP_REQUIRED_GROUP=certificate-management-admins
+# LDAP_GROUPS_TO_ROLES={'certificate-management-admins': 'admin', 'Team': 'team@example.com'}
+# LDAP_IS_ACTIVE_DIRECTORY=False

docker/nginx/default-ssl.conf

@@ -9,7 +9,7 @@ server {
 }
 
 server {
-   listen 443;
+   listen 443 ssl;
    server_name  _;
    access_log /dev/stdout;
    error_log /dev/stderr;

docker/pgsql-env

@@ -0,0 +1,4 @@
+POSTGRES_USER=lemur
+POSTGRES_PASSWORD=12345
+POSTGRES_DB=lemur
+POSTGRES_HOST=postgres

docker/src/lemur.conf.py

@@ -1,4 +1,6 @@
 import os
+from ast import literal_eval
+
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
 CORS = os.environ.get("CORS") == "True"
@@ -29,3 +31,13 @@ LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
 LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
 
 SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')
+
+LDAP_DEBUG = os.environ.get('LDAP_DEBUG') == "True"
+LDAP_AUTH = os.environ.get('LDAP_AUTH') == "True"
+LDAP_IS_ACTIVE_DIRECTORY = os.environ.get('LDAP_IS_ACTIVE_DIRECTORY') == "True"
+LDAP_BIND_URI = str(os.environ.get('LDAP_BIND_URI',''))
+LDAP_BASE_DN = str(os.environ.get('LDAP_BASE_DN',''))
+LDAP_EMAIL_DOMAIN = str(os.environ.get('LDAP_EMAIL_DOMAIN',''))
+LDAP_USE_TLS = str(os.environ.get('LDAP_USE_TLS',''))
+LDAP_REQUIRED_GROUP = str(os.environ.get('LDAP_REQUIRED_GROUP',''))
+LDAP_GROUPS_TO_ROLES = literal_eval(os.environ.get('LDAP_GROUPS_TO_ROLES') or "{}")

lemur/dns_providers/cli.py

@@ -1,8 +1,10 @@
 from flask_script import Manager
 
+import sys
+
 from lemur.constants import SUCCESS_METRIC_STATUS
 from lemur.dns_providers.service import get_all_dns_providers, set_domains
-from lemur.extensions import metrics
+from lemur.extensions import metrics, sentry
 from lemur.plugins.base import plugins
 
 manager = Manager(
@@ -19,13 +21,20 @@ def get_all_zones():
     dns_providers = get_all_dns_providers()
     acme_plugin = plugins.get("acme-issuer")
 
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    log_data = {
+        "function": function,
+        "message": "",
+    }
+
     for dns_provider in dns_providers:
         try:
             zones = acme_plugin.get_all_zones(dns_provider)
             set_domains(dns_provider, zones)
         except Exception as e:
             print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e))
-            set_domains(dns_provider, [])
+            log_data["message"] = f"get all zones failed for {dns_provider} {e}."
+            sentry.captureException(extra=log_data)
 
     status = SUCCESS_METRIC_STATUS
 

lemur/plugins/lemur_adcs/plugin.py

@@ -46,7 +46,7 @@ class ADCSIssuerPlugin(IssuerPlugin):
         )
         current_app.logger.info("Requesting CSR: {0}".format(csr))
         current_app.logger.info("Issuer options: {0}".format(issuer_options))
-        cert, req_id = (
+        cert = (
             ca_server.get_cert(csr, adcs_template, encoding="b64")
             .decode("utf-8")
             .replace("\r\n", "\n")
@@ -54,7 +54,7 @@ class ADCSIssuerPlugin(IssuerPlugin):
         chain = (
             ca_server.get_ca_cert(encoding="b64").decode("utf-8").replace("\r\n", "\n")
         )
-        return cert, chain, req_id
+        return cert, chain, None
 
     def revoke_certificate(self, certificate, comments):
         raise NotImplementedError("Not implemented\n", self, certificate, comments)

setup.py

@@ -23,7 +23,11 @@ from setuptools import setup, find_packages
 from subprocess import check_output
 
 import pip
-if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0):
+if tuple(map(int, pip.__version__.split('.'))) >= (19, 3, 0):
+    from pip._internal.network.session import PipSession
+    from pip._internal.req import parse_requirements
+
+elif tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0):
     from pip._internal.download import PipSession
     from pip._internal.req import parse_requirements
 else: