Merge pull request #2 from Netflix/master

Fxing Issue #2831

.gitignore

@@ -26,6 +26,11 @@ package-lock.json
 /lemur/static/dist/
 /lemur/static/app/vendor/
 /wheelhouse
+/lemur/lib
+/lemur/bin
+/lemur/lib64
+/lemur/include
+
 docs/_build
 .editorconfig
 .idea

.pre-commit-config.yaml

@@ -8,3 +8,17 @@
     sha: v2.9.5
     hooks:
     -   id: jshint
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.7
+
+-   repo: local
+    hooks:
+    -   id: python-bandit-vulnerability-check
+        name: bandit
+        entry: bandit
+        args: ['--ini', 'tox.ini', '-r', 'consoleme']
+        language: system
+        pass_filenames: false

.travis.yml

@@ -1,6 +1,5 @@
 language: python
-sudo: required
-dist: trusty
+dist: xenial
 
 node_js:
   - "6.2.0"
@@ -10,8 +9,8 @@ addons:
 
 matrix:
   include:
-    - python: "3.5"
-      env: TOXENV=py35
+    - python: "3.7"
+      env: TOXENV=py37
 
 cache:
   directories:

Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.5
+FROM python:3.7
 RUN apt-get update
-RUN apt-get install -y make python-software-properties curl
+RUN apt-get install -y make software-properties-common curl
 RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
 RUN apt-get update
-RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev
 RUN pip install -U setuptools
 RUN pip install coveralls bandit
 WORKDIR /app

Makefile

@@ -36,7 +36,7 @@ endif
 	@echo ""
 
 dev-docs:
-	pip install -r docs/requirements.txt
+	pip install -r requirements-docs.txt
 
 reset-db:
 	@echo "--> Dropping existing 'lemur' database"
@@ -46,7 +46,7 @@ reset-db:
 	@echo "--> Enabling pg_trgm extension"
 	psql lemur -c "create extension IF NOT EXISTS pg_trgm;"
 	@echo "--> Applying migrations"
-	lemur db upgrade
+	cd lemur && lemur db upgrade
 
 setup-git:
 	@echo "--> Installing git hooks"
@@ -113,10 +113,10 @@ endif
 	@echo "--> Updating Python requirements"
 	pip install --upgrade pip
 	pip install --upgrade pip-tools
+	pip-compile --output-file requirements.txt requirements.in -U --no-index
 	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
 	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
 	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
-	pip-compile --output-file requirements.txt requirements.in -U --no-index
 	@echo "--> Done updating Python requirements"
 	@echo "--> Removing python-ldap from requirements-docs.txt"
 	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt
@@ -125,5 +125,9 @@ endif
 	@echo "--> Done installing new dependencies"
 	@echo ""
 
+# Execute with make checkout-pr pr=<pr number>
+checkout-pr:
+	git fetch upstream pull/$(pr)/head:pr-$(pr)
+
 
 .PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish release

docker-compose.yml

@@ -13,10 +13,13 @@ services:
       VIRTUAL_ENV: 'true'
 
   postgres:
-    image: postgres:9.4
+    image: postgres
+    restart: always
     environment:
       POSTGRES_USER: lemur
       POSTGRES_PASSWORD: lemur
+    ports:
+      - "5432:5432"
 
   redis:
     image: "redis:alpine"

docker/Dockerfile

@@ -0,0 +1,64 @@
+FROM alpine:3.8
+
+ARG VERSION
+ENV VERSION master
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    mkdir -p /opt/lemur /home/lemur/.lemur/ && \
+    curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    chmod +x /entrypoint && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
+    chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
+    
+WORKDIR /opt/lemur
+
+RUN npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
+    apk del build-dependencies
+
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/entrypoint

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
+  echo "Database vars not set"
+  exit 1
+fi
+
+export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
+
+echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile
+
+source /etc/profile
+
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
+
+echo " # Create Postgres trgm extension"
+PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;'
+echo " # Done"
+
+if [ -z "${SKIP_SSL}" ]; then
+  if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
+    openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
+  fi
+  mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+fi
+
+# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
+# echo "Creating config"
+# https://github.com/Netflix/lemur/issues/2257
+# python3 /opt/lemur/lemur/manage.py create_config
+# echo "Done"
+# fi
+
+echo " # Running init"
+su lemur -c "python3 /opt/lemur/lemur/manage.py init"
+echo " # Done"
+
+# echo "Creating user"
+# https://github.com/Netflix/lemur/issues/
+# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
+# echo "Done"
+
+cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
+cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
+cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
+
+echo " # Populating crontab"
+echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify
+echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync
+echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked
+echo " # Done"
+
+exec "$@"

docker/nginx/default-ssl.conf

@@ -0,0 +1,37 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+    listen 80;
+    server_name  _;
+    return 301 https://$host$request_uri;
+}
+
+server {
+   listen 443;
+   server_name  _;
+   access_log /dev/stdout;
+   error_log /dev/stderr;
+   ssl_certificate /etc/nginx/ssl/server.crt;
+   ssl_certificate_key /etc/nginx/ssl/server.key;
+   ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+   ssl_ciphers HIGH:!aNULL:!MD5;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/nginx/default.conf

@@ -0,0 +1,26 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+   listen      80;
+   access_log  /dev/stdout;
+   error_log   /dev/stderr;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/src/lemur.conf.py

@@ -0,0 +1,31 @@
+import os
+_basedir = os.path.abspath(os.path.dirname(__file__))
+
+CORS = os.environ.get("CORS") == "True"
+debug = os.environ.get("DEBUG") == "True"
+
+SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw=='))
+
+LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ=='))
+LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo='))
+
+LEMUR_WHITELISTED_DOMAINS = []
+
+LEMUR_EMAIL = ''
+LEMUR_SECURITY_TEAM_EMAIL = []
+
+
+LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
+LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE',''))
+LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
+LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
+
+ACTIVE_PROVIDERS = []
+
+METRIC_PROVIDERS = []
+
+LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
+LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
+
+SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')

docker/supervisor.conf

@@ -0,0 +1,32 @@
+[supervisord]
+nodaemon=true
+user=root
+logfile=/dev/stdout
+logfile_maxbytes=0
+pidfile = /tmp/supervisord.pid
+
+[program:lemur]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
+command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000
+user=lemur
+directory=/opt/lemur/lemur
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:cron]
+command=/usr/sbin/crond -f
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docs/administration.rst

@@ -161,6 +161,13 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
 
         Dump all imported or generated CSR and certificate details to stdout using OpenSSL. (default: `False`)
 
+.. data:: ALLOW_CERT_DELETION
+    :noindex:
+
+        When set to True, certificates can be marked as deleted via the API and deleted certificates will not be displayed
+        in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed"
+        and deleted certificates will always be visible in the UI. (default: `False`)
+
 
 Certificate Default Options
 ---------------------------
@@ -586,8 +593,60 @@ If you are not using a metric provider you do not need to configure any of these
 Plugin Specific Options
 -----------------------
 
+Active Directory Certificate Services Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+.. data:: ADCS_SERVER
+    :noindex:
+
+        FQDN of your ADCS Server
+
+
+.. data:: ADCS_AUTH_METHOD
+    :noindex:
+
+        The chosen authentication method. Either ‘basic’ (the default), ‘ntlm’ or ‘cert’ (SSL client certificate). The next 2 variables are interpreted differently for different methods.
+
+
+.. data:: ADCS_USER
+    :noindex:
+
+        The username (basic) or the path to the public cert (cert) of the user accessing PKI
+
+
+.. data:: ADCS_PWD
+    :noindex:
+
+        The passwd (basic) or the path to the private key (cert) of the user accessing PKI
+
+
+.. data:: ADCS_TEMPLATE
+    :noindex:
+
+        Template to be used for certificate issuing. Usually display name w/o spaces
+
+
+.. data:: ADCS_START
+    :noindex:
+
+.. data:: ADCS_STOP
+    :noindex:
+
+.. data:: ADCS_ISSUING
+    :noindex:
+
+        Contains the issuing cert of the CA
+
+
+.. data:: ADCS_ROOT
+    :noindex:
+
+        Contains the root cert of the CA
+
+
 Verisign Issuer Plugin
-^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~
 
 Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur,
 Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation
@@ -683,7 +742,7 @@ The following configuration properties are required to use the Digicert issuer p
 
 
 CFSSL Issuer Plugin
-^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~
 
 The following configuration properties are required to use the CFSSL issuer plugin.
 
@@ -702,9 +761,36 @@ The following configuration properties are required to use the CFSSL issuer plug
 
         This is the intermediate to be used for your CA chain
 
+.. data:: CFSSL_KEY
+    :noindex:
+
+        This is the hmac key to authenticate to the CFSSL service. (Optional)
+
+
+Hashicorp Vault Source/Destination Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Lemur can import and export certificate data to and from a Hashicorp Vault secrets store. Lemur can connect to a different Vault service per source/destination.
+
+.. note:: This plugin does not supersede or overlap the 3rd party Vault Issuer plugin.
+
+.. note:: Vault does not have any configuration properties however it does read from a file on disk for a vault access token. The Lemur service account needs read access to this file.
+
+Vault Source
+""""""""""""
+
+The Vault Source Plugin will read from one Vault object location per source defined. There is expected to be one or more certificates defined in each object in Vault.
+
+Vault Destination
+"""""""""""""""""
+
+A Vault destination can be one object in Vault or a directory where all certificates will be stored as their own object by CN.
+
+Vault Destination supports a regex filter to prevent certificates with SAN that do not match the regex filter from being deployed. This is an optional feature per destination defined.
+
 
 AWS Source/Destination Plugin
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions.
 
@@ -1056,7 +1142,9 @@ Verisign/Symantec
 -----------------
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Issuer
 :Description:
@@ -1082,6 +1170,8 @@ Acme
 
 :Authors:
     Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>,
     Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 :Type:
     Issuer
@@ -1093,7 +1183,9 @@ Atlas
 -----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Metric
 :Description:
@@ -1104,7 +1196,9 @@ Email
 -----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Notification
 :Description:
@@ -1126,7 +1220,9 @@ AWS
 ----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Source
 :Description:
@@ -1137,7 +1233,9 @@ AWS
 ----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Destination
 :Description:
@@ -1187,6 +1285,26 @@ CFSSL
 :Description:
     Basic support for generating certificates from the private certificate authority CFSSL
 
+Vault
+-----
+
+:Authors:
+    Christopher Jolley <chris@alwaysjolley.com>
+:Type:
+    Source
+:Description:
+    Source plugin imports certificates from Hashicorp Vault secret store.
+
+Vault
+-----
+
+:Authors:
+    Christopher Jolley <chris@alwaysjolley.com>
+:Type:
+    Destination
+:Description:
+    Destination plugin to deploy certificates to Hashicorp Vault secret store.
+
 
 3rd Party Plugins
 =================

docs/conf.py

@@ -18,17 +18,18 @@ from unittest.mock import MagicMock
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath(".."))
 
 # Mock packages that cannot be installed on rtd
-on_rtd = os.environ.get('READTHEDOCS') == 'True'
+on_rtd = os.environ.get("READTHEDOCS") == "True"
 if on_rtd:
+
     class Mock(MagicMock):
         @classmethod
         def __getattr__(cls, name):
             return MagicMock()
 
-    MOCK_MODULES = ['ldap']
+    MOCK_MODULES = ["ldap"]
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 # -- General configuration ------------------------------------------------
@@ -39,27 +40,23 @@ if on_rtd:
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinxcontrib.autohttp.flask',
-    'sphinx.ext.todo',
-]
+extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autohttp.flask", "sphinx.ext.todo"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'lemur'
-copyright = u'2018, Netflix Inc.'
+project = u"lemur"
+copyright = u"2018, Netflix Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -68,7 +65,7 @@ copyright = u'2018, Netflix Inc.'
 base_dir = os.path.join(os.path.dirname(__file__), os.pardir)
 about = {}
 with open(os.path.join(base_dir, "lemur", "__about__.py")) as f:
-    exec(f.read(), about)
+    exec(f.read(), about)  # nosec
 
 version = release = about["__version__"]
 
@@ -84,7 +81,7 @@ version = release = about["__version__"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ["_build"]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -102,7 +99,7 @@ exclude_patterns = ['_build']
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -114,11 +111,12 @@ pygments_style = 'sphinx'
 # -- Options for HTML output ----------------------------------------------
 
 # on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+on_rtd = os.environ.get("READTHEDOCS", None) == "True"
 
 if not on_rtd:  # only import and set the theme if we're building docs locally
     import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
+
+    html_theme = "sphinx_rtd_theme"
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -148,7 +146,7 @@ if not on_rtd:  # only import and set the theme if we're building docs locally
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -197,7 +195,7 @@ html_static_path = ['_static']
 # html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'lemurdoc'
+htmlhelp_basename = "lemurdoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -205,10 +203,8 @@ htmlhelp_basename = 'lemurdoc'
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #'preamble': '',
 }
@@ -217,8 +213,7 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'lemur.tex', u'Lemur Documentation',
-   u'Kevin Glisson', 'manual'),
+    ("index", "lemur.tex", u"Lemur Documentation", u"Netflix Security", "manual")
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -246,10 +241,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'Lemur', u'Lemur Documentation',
-     [u'Kevin Glisson'], 1)
-]
+man_pages = [("index", "Lemur", u"Lemur Documentation", [u"Netflix Security"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -261,9 +253,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Lemur', u'Lemur Documentation',
-   u'Kevin Glisson', 'Lemur', 'SSL Certificate Management',
-   'Miscellaneous'),
+    (
+        "index",
+        "Lemur",
+        u"Lemur Documentation",
+        u"Netflix Security",
+        "Lemur",
+        "SSL Certificate Management",
+        "Miscellaneous",
+    )
 ]
 
 # Documents to append as an appendix to all manuals.

docs/developer/index.rst

@@ -22,12 +22,18 @@ Once you've got all that, the rest is simple:
     # If you have a fork, you'll want to clone it instead
     git clone git://github.com/netflix/lemur.git
 
-    # Create a python virtualenv
-    mkvirtualenv lemur
+    # Create and activate python virtualenv from within the lemur repo
+    python3 -m venv env
+    . env/bin/activate
+
+    # Install doc requirements
 
-    # Make the magic happen
     make dev-docs
 
+    # Make the docs
+    cd docs
+    make html
+
 Running ``make dev-docs`` will install the basic requirements to get Sphinx running.
 
 
@@ -58,7 +64,7 @@ Once you've got all that, the rest is simple:
     git clone git://github.com/lemur/lemur.git
 
     # Create a python virtualenv
-    mkvirtualenv lemur
+    python3 -m venv env
 
     # Make the magic happen
     make
@@ -135,7 +141,7 @@ The test suite consists of multiple parts, testing both the Python and JavaScrip
 
     make test
 
-If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``test-js`` for the JavaScript tests.
+If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``make test-js`` for the JavaScript tests.
 
 
 You'll notice that the test suite is structured based on where the code lives, and strongly encourages using the mock library to drive more accurate individual tests.

docs/production/index.rst

@@ -318,7 +318,7 @@ Periodic Tasks
 ==============
 
 Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
-a cron job that runs the commands.
+celery tasks or cron jobs that run these commands.
 
 There are currently three commands that could/should be run on a periodic basis:
 
@@ -326,11 +326,124 @@ There are currently three commands that could/should be run on a periodic basis:
 - `check_revoked`
 - `sync`
 
+If you are using LetsEncrypt, you must also run the following:
+
+- `fetch_all_pending_acme_certs`
+- `remove_old_acme_certs`
+
 How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
-`sync` is typically run every 15 minutes.
+`sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
+`remove_old_acme_certs` can be ran more rarely, such as once every week.
 
 Example cron entries::
 
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
     */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
+
+
+Example Celery configuration (To be placed in your configuration file)::
+
+    CELERYBEAT_SCHEDULE = {
+        'fetch_all_pending_acme_certs': {
+            'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(minute="*"),
+        },
+        'remove_old_acme_certs': {
+            'task': 'lemur.common.celery.remove_old_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=7, minute=30, day_of_week=1),
+        },
+        'clean_all_sources': {
+            'task': 'lemur.common.celery.clean_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=1, minute=0, day_of_week=1),
+        },
+        'sync_all_sources': {
+            'task': 'lemur.common.celery.sync_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*/3", minute=5),
+        },
+        'sync_source_destination': {
+            'task': 'lemur.common.celery.sync_source_destination',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*"),
+        }
+    }
+
+To enable celery support, you must also have configuration values that tell Celery which broker and backend to use.
+Here are the Celery configuration variables that should be set::
+
+    CELERY_RESULT_BACKEND = 'redis://your_redis_url:6379'
+    CELERY_BROKER_URL = 'redis://your_redis_url:6379'
+    CELERY_IMPORTS = ('lemur.common.celery')
+    CELERY_TIMEZONE = 'UTC'
+
+You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks.
+The scheduler can be started with::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery beat
+
+And the worker can be started with desired options such as the following::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery worker --concurrency 10 -E -n lemurworker1@%%h
+
+supervisor or systemd configurations should be created for these in production environments as appropriate.
+
+Add support for LetsEncrypt
+===========================
+
+LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
+for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
+LetsEncrypt requires that we prove ownership of a domain before we're able to issue a certificate for that domain, each
+time we want a certificate.
+
+The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
+through the creation of DNS TXT records.
+
+In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
+token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
+prove ownership of all of these domains through this method. The token is typically written to a TXT record at
+-acme_challenge.domain.com. Once we create the appropriate TXT record(s), Lemur will try to validate propagation
+before requesting that LetsEncrypt finalize the certificate request and send us the certificate.
+
+.. figure:: letsencrypt_flow.png
+
+To start issuing certificates through LetsEncrypt, you must enable Celery support within Lemur first. After doing so,
+you need to create a LetsEncrypt authority. To do this, visit
+Authorities -> Create. Set the applicable attributes and click "More Options".
+
+.. figure:: letsencrypt_authority_1.png
+
+You will need to set "Certificate" to LetsEncrypt's active chain of trust for the authority you want to use. To find
+the active chain of trust at the time of writing, please visit `LetsEncrypt
+<https://letsencrypt.org/certificates/>`_.
+
+Under Acme_url, enter in the appropriate endpoint URL. Lemur supports LetsEncrypt's V2 API, and we recommend you to use
+this. At the time of writing, the staging and production URLs for LetsEncrypt V2 are
+https://acme-staging-v02.api.letsencrypt.org/directory and https://acme-v02.api.letsencrypt.org/directory.
+
+.. figure:: letsencrypt_authority_2.png
+
+After creating the authorities, we will need to create a DNS provider. Visit `Admin` -> `DNS Providers` and click
+`Create`. Lemur comes with a few provider plugins built in, with different options. Create a DNS provider with the
+appropriate choices.
+
+.. figure:: create_dns_provider.png
+
+By default, users will need to select the DNS provider that is authoritative over their domain in order for the
+LetsEncrypt flow to function. However, Lemur will attempt to automatically determine the appropriate provider if
+possible. To enable this functionality, periodically (or through Cron/Celery) run `lemur dns_providers get_all_zones`.
+This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
+Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.

docs/quickstart/index.rst

@@ -12,7 +12,7 @@ Dependencies
 Some basic prerequisites which you'll need in order to run Lemur:
 
 * A UNIX-based operating system (we test on Ubuntu, develop on OS X)
-* Python 3.5 or greater
+* Python 3.7 or greater
 * PostgreSQL 9.4 or greater
 * Nginx
 
@@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur:
 Installing Build Dependencies
 -----------------------------
 
-If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
+If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
 
 .. code-block:: bash
 
@@ -31,7 +31,7 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack
 
 .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver.  npm is needed if you're installing Lemur from the source (e.g., from git).
 
-.. note:: Installing node from a package manager may creat the nodejs bin at  /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
+.. note:: Installing node from a package manager may create the nodejs bin at  /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
     sudo ln -s /user/bin/nodejs /usr/bin/node
 
 Now, install Python ``virtualenv`` package:
@@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config
     LEMUR_DEFAULT_ORGANIZATION
     LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
 
-Setup Postgres
+Set Up Postgres
 --------------
 
 For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
@@ -180,6 +180,13 @@ Lemur provides a helpful command that will initialize your database for you. It
 
 In addition to creating a new user, Lemur also creates a few default email notifications.  These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``.  They basically guarantee that every certificate within Lemur will send one expiration notification to the security team.
 
+Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user.
+
+.. code-block:: bash
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH SUPERUSER
+
 Additional notifications can be created through the UI or API.  See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 **Make note of the password used as this will be used during first login to the Lemur UI.**
@@ -189,14 +196,20 @@ Additional notifications can be created through the UI or API.  See :ref:`Creati
     cd /www/lemur/lemur
     lemur init
 
+.. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now.
+
+.. code-block:: bash
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH NOSUPERUSER
+
 
 .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
-
-Setup a Reverse Proxy
+Set Up a Reverse Proxy
 ---------------------
 
-By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
+By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
 
 
 Proxying with Nginx

gulp/server.js

@@ -6,9 +6,9 @@ var browserSync = require('browser-sync');
 var httpProxy = require('http-proxy');
 
 /* This configuration allow you to configure browser sync to proxy your backend */
-/*
- var proxyTarget = 'http://localhost/context/'; // The location of your backend
- var proxyApiPrefix = 'api'; // The element in the URL which differentiate between API request and static file request
+
+ var proxyTarget = 'http://localhost:8000/'; // The location of your backend
+ var proxyApiPrefix = '/api/'; // The element in the URL which differentiate between API request and static file request
  var proxy = httpProxy.createProxyServer({
      target: proxyTarget
  });
@@ -19,7 +19,6 @@ var httpProxy = require('http-proxy');
          next();
      }
  }
- */
 
 function browserSyncInit(baseDir, files, browser) {
   browser = browser === undefined ? 'default' : browser;
@@ -27,6 +26,7 @@ function browserSyncInit(baseDir, files, browser) {
   browserSync.instance = browserSync.init(files, {
     startPath: '/index.html',
       server: {
+          middleware: [proxyMiddleware],
           baseDir: baseDir,
           routes: {
               '/bower_components': './bower_components'

lemur/__about__.py

@@ -1,12 +1,18 @@
 from __future__ import absolute_import, division, print_function
 
 __all__ = [
-    "__title__", "__summary__", "__uri__", "__version__", "__author__",
-    "__email__", "__license__", "__copyright__",
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
 ]
 
 __title__ = "lemur"
-__summary__ = ("Certificate management and orchestration service")
+__summary__ = "Certificate management and orchestration service"
 __uri__ = "https://github.com/Netflix/lemur"
 
 __version__ = "0.7.0"

lemur/__init__.py

@@ -5,7 +5,8 @@
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-
+.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
+.. moduleauthor:: Hossein Shafagh <hshafagh@netflix.com>
 
 """
 import time
@@ -32,14 +33,26 @@ from lemur.pending_certificates.views import mod as pending_certificates_bp
 from lemur.dns_providers.views import mod as dns_providers_bp
 
 from lemur.__about__ import (
-    __author__, __copyright__, __email__, __license__, __summary__, __title__,
-    __uri__, __version__
+    __author__,
+    __copyright__,
+    __email__,
+    __license__,
+    __summary__,
+    __title__,
+    __uri__,
+    __version__,
 )
 
 
 __all__ = [
-    "__title__", "__summary__", "__uri__", "__version__", "__author__",
-    "__email__", "__license__", "__copyright__",
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
 ]
 
 LEMUR_BLUEPRINTS = (
@@ -62,8 +75,10 @@ LEMUR_BLUEPRINTS = (
 )
 
 
-def create_app(config=None):
-    app = factory.create_app(app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config)
+def create_app(config_path=None):
+    app = factory.create_app(
+        app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config_path
+    )
     configure_hook(app)
     return app
 
@@ -93,7 +108,7 @@ def configure_hook(app):
     @app.after_request
     def after_request(response):
         # Return early if we don't have the start time
-        if not hasattr(g, 'request_start_time'):
+        if not hasattr(g, "request_start_time"):
             return response
 
         # Get elapsed time in milliseconds
@@ -102,12 +117,12 @@ def configure_hook(app):
 
         # Collect request/response tags
         tags = {
-            'endpoint': request.endpoint,
-            'request_method': request.method.lower(),
-            'status_code': response.status_code
+            "endpoint": request.endpoint,
+            "request_method": request.method.lower(),
+            "status_code": response.status_code,
         }
 
         # Record our response time metric
-        metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags)
-        metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
+        metrics.send("response_time", "TIMER", elapsed, metric_tags=tags)
+        metrics.send("status_code_{}".format(response.status_code), "counter", 1)
         return response

lemur/api_keys/cli.py

@@ -14,23 +14,32 @@ from datetime import datetime
 manager = Manager(usage="Handles all api key related tasks.")
 
 
-@manager.option('-u', '--user-id', dest='uid', help='The User ID this access key belongs too.')
-@manager.option('-n', '--name', dest='name', help='The name of this API Key.')
-@manager.option('-t', '--ttl', dest='ttl', help='The TTL of this API Key. -1 for forever.')
+@manager.option(
+    "-u", "--user-id", dest="uid", help="The User ID this access key belongs too."
+)
+@manager.option("-n", "--name", dest="name", help="The name of this API Key.")
+@manager.option(
+    "-t", "--ttl", dest="ttl", help="The TTL of this API Key. -1 for forever."
+)
 def create(uid, name, ttl):
     """
     Create a new api key for a user.
     :return:
     """
     print("[+] Creating a new api key.")
-    key = api_key_service.create(user_id=uid, name=name,
-        ttl=ttl, issued_at=int(datetime.utcnow().timestamp()), revoked=False)
+    key = api_key_service.create(
+        user_id=uid,
+        name=name,
+        ttl=ttl,
+        issued_at=int(datetime.utcnow().timestamp()),
+        revoked=False,
+    )
     print("[+] Successfully created a new api key. Generating a JWT...")
     jwt = create_token(uid, key.id, key.ttl)
     print("[+] Your JWT is: {jwt}".format(jwt=jwt))
 
 
-@manager.option('-a', '--api-key-id', dest='aid', help='The API Key ID to revoke.')
+@manager.option("-a", "--api-key-id", dest="aid", help="The API Key ID to revoke.")
 def revoke(aid):
     """
     Revokes an api key for a user.

lemur/api_keys/models.py

@@ -12,14 +12,19 @@ from lemur.database import db
 
 
 class ApiKey(db.Model):
-    __tablename__ = 'api_keys'
+    __tablename__ = "api_keys"
     id = Column(Integer, primary_key=True)
     name = Column(String)
-    user_id = Column(Integer, ForeignKey('users.id'))
+    user_id = Column(Integer, ForeignKey("users.id"))
     ttl = Column(BigInteger)
     issued_at = Column(BigInteger)
     revoked = Column(Boolean)
 
     def __repr__(self):
         return "ApiKey(name={name}, user_id={user_id}, ttl={ttl}, issued_at={iat}, revoked={revoked})".format(
-            user_id=self.user_id, name=self.name, ttl=self.ttl, iat=self.issued_at, revoked=self.revoked)
+            user_id=self.user_id,
+            name=self.name,
+            ttl=self.ttl,
+            iat=self.issued_at,
+            revoked=self.revoked,
+        )

lemur/api_keys/schemas.py

@@ -13,12 +13,18 @@ from lemur.users.schemas import UserNestedOutputSchema, UserInputSchema
 
 
 def current_user_id():
-    return {'id': g.current_user.id, 'email': g.current_user.email, 'username': g.current_user.username}
+    return {
+        "id": g.current_user.id,
+        "email": g.current_user.email,
+        "username": g.current_user.username,
+    }
 
 
 class ApiKeyInputSchema(LemurInputSchema):
     name = fields.String(required=False)
-    user = fields.Nested(UserInputSchema, missing=current_user_id, default=current_user_id)
+    user = fields.Nested(
+        UserInputSchema, missing=current_user_id, default=current_user_id
+    )
     ttl = fields.Integer()
 
 

lemur/api_keys/service.py

@@ -34,7 +34,7 @@ def revoke(aid):
     :return:
     """
     api_key = get(aid)
-    setattr(api_key, 'revoked', False)
+    setattr(api_key, "revoked", False)
 
     return database.update(api_key)
 
@@ -80,10 +80,10 @@ def render(args):
     :return:
     """
     query = database.session_query(ApiKey)
-    user_id = args.pop('user_id', None)
-    aid = args.pop('id', None)
-    has_permission = args.pop('has_permission', False)
-    requesting_user_id = args.pop('requesting_user_id')
+    user_id = args.pop("user_id", None)
+    aid = args.pop("id", None)
+    has_permission = args.pop("has_permission", False)
+    requesting_user_id = args.pop("requesting_user_id")
 
     if user_id:
         query = query.filter(ApiKey.user_id == user_id)

lemur/api_keys/views.py

@@ -19,10 +19,16 @@ from lemur.auth.permissions import ApiKeyCreatorPermission
 from lemur.common.schema import validate_schema
 from lemur.common.utils import paginated_parser
 
-from lemur.api_keys.schemas import api_key_input_schema, api_key_revoke_schema, api_key_output_schema, \
-    api_keys_output_schema, api_key_described_output_schema, user_api_key_input_schema
+from lemur.api_keys.schemas import (
+    api_key_input_schema,
+    api_key_revoke_schema,
+    api_key_output_schema,
+    api_keys_output_schema,
+    api_key_described_output_schema,
+    user_api_key_input_schema,
+)
 
-mod = Blueprint('api_keys', __name__)
+mod = Blueprint("api_keys", __name__)
 api = Api(mod)
 
 
@@ -81,8 +87,8 @@ class ApiKeyList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['has_permission'] = ApiKeyCreatorPermission().can()
-        args['requesting_user_id'] = g.current_user.id
+        args["has_permission"] = ApiKeyCreatorPermission().can()
+        args["requesting_user_id"] = g.current_user.id
         return service.render(args)
 
     @validate_schema(api_key_input_schema, api_key_output_schema)
@@ -124,12 +130,26 @@ class ApiKeyList(AuthenticatedResource):
            :statuscode 403: unauthenticated
         """
         if not ApiKeyCreatorPermission().can():
-            if data['user']['id'] != g.current_user.id:
-                return dict(message="You are not authorized to create tokens for: {0}".format(data['user']['username'])), 403
+            if data["user"]["id"] != g.current_user.id:
+                return (
+                    dict(
+                        message="You are not authorized to create tokens for: {0}".format(
+                            data["user"]["username"]
+                        )
+                    ),
+                    403,
+                )
 
-        access_token = service.create(name=data['name'], user_id=data['user']['id'], ttl=data['ttl'],
-                                      revoked=False, issued_at=int(datetime.utcnow().timestamp()))
-        return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
+        access_token = service.create(
+            name=data["name"],
+            user_id=data["user"]["id"],
+            ttl=data["ttl"],
+            revoked=False,
+            issued_at=int(datetime.utcnow().timestamp()),
+        )
+        return dict(
+            jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
+        )
 
 
 class ApiKeyUserList(AuthenticatedResource):
@@ -186,9 +206,9 @@ class ApiKeyUserList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['has_permission'] = ApiKeyCreatorPermission().can()
-        args['requesting_user_id'] = g.current_user.id
-        args['user_id'] = user_id
+        args["has_permission"] = ApiKeyCreatorPermission().can()
+        args["requesting_user_id"] = g.current_user.id
+        args["user_id"] = user_id
         return service.render(args)
 
     @validate_schema(user_api_key_input_schema, api_key_output_schema)
@@ -230,11 +250,25 @@ class ApiKeyUserList(AuthenticatedResource):
         """
         if not ApiKeyCreatorPermission().can():
             if user_id != g.current_user.id:
-                return dict(message="You are not authorized to create tokens for: {0}".format(user_id)), 403
+                return (
+                    dict(
+                        message="You are not authorized to create tokens for: {0}".format(
+                            user_id
+                        )
+                    ),
+                    403,
+                )
 
-        access_token = service.create(name=data['name'], user_id=user_id, ttl=data['ttl'],
-                                      revoked=False, issued_at=int(datetime.utcnow().timestamp()))
-        return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
+        access_token = service.create(
+            name=data["name"],
+            user_id=user_id,
+            ttl=data["ttl"],
+            revoked=False,
+            issued_at=int(datetime.utcnow().timestamp()),
+        )
+        return dict(
+            jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
+        )
 
 
 class ApiKeys(AuthenticatedResource):
@@ -329,7 +363,9 @@ class ApiKeys(AuthenticatedResource):
             if not ApiKeyCreatorPermission().can():
                 return dict(message="You are not authorized to update this token!"), 403
 
-        service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
+        service.update(
+            access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
+        )
         return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
 
     def delete(self, aid):
@@ -371,7 +407,7 @@ class ApiKeys(AuthenticatedResource):
                 return dict(message="You are not authorized to delete this token!"), 403
 
         service.delete(access_key)
-        return {'result': True}
+        return {"result": True}
 
 
 class UserApiKeys(AuthenticatedResource):
@@ -472,7 +508,9 @@ class UserApiKeys(AuthenticatedResource):
         if access_key.user_id != uid:
             return dict(message="You are not authorized to update this token!"), 403
 
-        service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
+        service.update(
+            access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
+        )
         return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
 
     def delete(self, uid, aid):
@@ -517,7 +555,7 @@ class UserApiKeys(AuthenticatedResource):
             return dict(message="You are not authorized to delete this token!"), 403
 
         service.delete(access_key)
-        return {'result': True}
+        return {"result": True}
 
 
 class ApiKeysDescribed(AuthenticatedResource):
@@ -572,8 +610,12 @@ class ApiKeysDescribed(AuthenticatedResource):
         return access_key
 
 
-api.add_resource(ApiKeyList, '/keys', endpoint='api_keys')
-api.add_resource(ApiKeys, '/keys/<int:aid>', endpoint='api_key')
-api.add_resource(ApiKeysDescribed, '/keys/<int:aid>/described', endpoint='api_key_described')
-api.add_resource(ApiKeyUserList, '/users/<int:user_id>/keys', endpoint='user_api_keys')
-api.add_resource(UserApiKeys, '/users/<int:uid>/keys/<int:aid>', endpoint='user_api_key')
+api.add_resource(ApiKeyList, "/keys", endpoint="api_keys")
+api.add_resource(ApiKeys, "/keys/<int:aid>", endpoint="api_key")
+api.add_resource(
+    ApiKeysDescribed, "/keys/<int:aid>/described", endpoint="api_key_described"
+)
+api.add_resource(ApiKeyUserList, "/users/<int:user_id>/keys", endpoint="user_api_keys")
+api.add_resource(
+    UserApiKeys, "/users/<int:uid>/keys/<int:aid>", endpoint="user_api_key"
+)

lemur/auth/ldap.py

@@ -14,35 +14,41 @@ from lemur.roles import service as role_service
 from lemur.common.utils import validate_conf, get_psuedo_random_string
 
 
-class LdapPrincipal():
+class LdapPrincipal:
     """
     Provides methods for authenticating against an LDAP server.
     """
+
     def __init__(self, args):
         self._ldap_validate_conf()
         # setup ldap config
-        if not args['username']:
+        if not args["username"]:
             raise Exception("missing ldap username")
-        if not args['password']:
+        if not args["password"]:
             self.error_message = "missing ldap password"
             raise Exception("missing ldap password")
-        self.ldap_principal = args['username']
+        self.ldap_principal = args["username"]
         self.ldap_email_domain = current_app.config.get("LDAP_EMAIL_DOMAIN", None)
-        if '@' not in self.ldap_principal:
-            self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
-        self.ldap_username = args['username']
-        if '@' in self.ldap_username:
-            self.ldap_username = args['username'].split("@")[0]
-        self.ldap_password = args['password']
-        self.ldap_server = current_app.config.get('LDAP_BIND_URI', None)
+        if "@" not in self.ldap_principal:
+            self.ldap_principal = "%s@%s" % (
+                self.ldap_principal,
+                self.ldap_email_domain,
+            )
+        self.ldap_username = args["username"]
+        if "@" in self.ldap_username:
+            self.ldap_username = args["username"].split("@")[0]
+        self.ldap_password = args["password"]
+        self.ldap_server = current_app.config.get("LDAP_BIND_URI", None)
         self.ldap_base_dn = current_app.config.get("LDAP_BASE_DN", None)
         self.ldap_use_tls = current_app.config.get("LDAP_USE_TLS", False)
         self.ldap_cacert_file = current_app.config.get("LDAP_CACERT_FILE", None)
         self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None)
         self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None)
         self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None)
-        self.ldap_is_active_directory = current_app.config.get("LDAP_IS_ACTIVE_DIRECTORY", False)
-        self.ldap_attrs = ['memberOf']
+        self.ldap_is_active_directory = current_app.config.get(
+            "LDAP_IS_ACTIVE_DIRECTORY", False
+        )
+        self.ldap_attrs = ["memberOf"]
         self.ldap_client = None
         self.ldap_groups = None
 
@@ -60,8 +66,8 @@ class LdapPrincipal():
                 get_psuedo_random_string(),
                 self.ldap_principal,
                 True,
-                '',     # thumbnailPhotoUrl
-                list(roles)
+                "",  # thumbnailPhotoUrl
+                list(roles),
             )
         else:
             # we add 'lemur' specific roles, so they do not get marked as removed
@@ -76,7 +82,7 @@ class LdapPrincipal():
                 self.ldap_principal,
                 user.active,
                 user.profile_picture,
-                list(roles)
+                list(roles),
             )
         return user
 
@@ -99,15 +105,18 @@ class LdapPrincipal():
             role = role_service.get_by_name(self.ldap_default_role)
             if role:
                 if not role.third_party:
-                    role = role.set_third_party(role.id, third_party_status=True)
+                    role = role_service.set_third_party(role.id, third_party_status=True)
                 roles.add(role)
 
         # update their 'roles'
         role = role_service.get_by_name(self.ldap_principal)
         if not role:
-            description = "auto generated role based on owner: {0}".format(self.ldap_principal)
-            role = role_service.create(self.ldap_principal, description=description,
-                third_party=True)
+            description = "auto generated role based on owner: {0}".format(
+                self.ldap_principal
+            )
+            role = role_service.create(
+                self.ldap_principal, description=description, third_party=True
+            )
         if not role.third_party:
             role = role_service.set_third_party(role.id, third_party_status=True)
         roles.add(role)
@@ -118,9 +127,15 @@ class LdapPrincipal():
             role = role_service.get_by_name(role_name)
             if role:
                 if ldap_group_name in self.ldap_groups:
-                    current_app.logger.debug("assigning role {0} to ldap user {1}".format(self.ldap_principal, role))
+                    current_app.logger.debug(
+                        "assigning role {0} to ldap user {1}".format(
+                            self.ldap_principal, role
+                        )
+                    )
                     if not role.third_party:
-                        role = role_service.set_third_party(role.id, third_party_status=True)
+                        role = role_service.set_third_party(
+                            role.id, third_party_status=True
+                        )
                     roles.add(role)
         return roles
 
@@ -132,7 +147,7 @@ class LdapPrincipal():
         self._bind()
         roles = self._authorize()
         if not roles:
-            raise Exception('ldap authorization failed')
+            raise Exception("ldap authorization failed")
         return self._update_user(roles)
 
     def _bind(self):
@@ -141,9 +156,12 @@ class LdapPrincipal():
         list groups for a user.
         raise an exception on error.
         """
-        if '@' not in self.ldap_principal:
-            self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
-        ldap_filter = 'userPrincipalName=%s' % self.ldap_principal
+        if "@" not in self.ldap_principal:
+            self.ldap_principal = "%s@%s" % (
+                self.ldap_principal,
+                self.ldap_email_domain,
+            )
+        ldap_filter = "userPrincipalName=%s" % self.ldap_principal
 
         # query ldap for auth
         try:
@@ -159,37 +177,47 @@ class LdapPrincipal():
                 self.ldap_client.set_option(ldap.OPT_X_TLS_DEMAND, True)
                 self.ldap_client.set_option(ldap.OPT_DEBUG_LEVEL, 255)
             if self.ldap_cacert_file:
-                self.ldap_client.set_option(ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file)
+                self.ldap_client.set_option(
+                    ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file
+                )
             self.ldap_client.simple_bind_s(self.ldap_principal, self.ldap_password)
         except ldap.INVALID_CREDENTIALS:
             self.ldap_client.unbind()
-            raise Exception('The supplied ldap credentials are invalid')
+            raise Exception("The supplied ldap credentials are invalid")
         except ldap.SERVER_DOWN:
-            raise Exception('ldap server unavailable')
+            raise Exception("ldap server unavailable")
         except ldap.LDAPError as e:
             raise Exception("ldap error: {0}".format(e))
 
         if self.ldap_is_active_directory:
             # Lookup user DN, needed to search for group membership
-            userdn = self.ldap_client.search_s(self.ldap_base_dn,
-                                               ldap.SCOPE_SUBTREE, ldap_filter,
-                                               ['distinguishedName'])[0][1]['distinguishedName'][0]
-            userdn = userdn.decode('utf-8')
+            userdn = self.ldap_client.search_s(
+                self.ldap_base_dn,
+                ldap.SCOPE_SUBTREE,
+                ldap_filter,
+                ["distinguishedName"],
+            )[0][1]["distinguishedName"][0]
+            userdn = userdn.decode("utf-8")
             # Search all groups that have the userDN as a member
-            groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn)
-            lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn'])
+            groupfilter = "(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))".format(
+                userdn
+            )
+            lgroups = self.ldap_client.search_s(
+                self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ["cn"]
+            )
 
             # Create a list of group CN's from the result
             self.ldap_groups = []
             for group in lgroups:
                 (dn, values) = group
-                self.ldap_groups.append(values['cn'][0].decode('ascii'))
+                self.ldap_groups.append(values["cn"][0].decode("ascii"))
         else:
-            lgroups = self.ldap_client.search_s(self.ldap_base_dn,
-                                                ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf']
+            lgroups = self.ldap_client.search_s(
+                self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs
+            )[0][1]["memberOf"]
             # lgroups is a list of utf-8 encoded strings
             # convert to a single string of groups to allow matching
-            self.ldap_groups = b''.join(lgroups).decode('ascii')
+            self.ldap_groups = b"".join(lgroups).decode("ascii")
 
         self.ldap_client.unbind()
 
@@ -197,9 +225,5 @@ class LdapPrincipal():
         """
         Confirms required ldap config settings exist.
         """
-        required_vars = [
-            'LDAP_BIND_URI',
-            'LDAP_BASE_DN',
-            'LDAP_EMAIL_DOMAIN',
-        ]
+        required_vars = ["LDAP_BIND_URI", "LDAP_BASE_DN", "LDAP_EMAIL_DOMAIN"]
         validate_conf(current_app, required_vars)

lemur/auth/permissions.py

@@ -9,24 +9,32 @@
 from functools import partial
 from collections import namedtuple
 
+from flask import current_app
 from flask_principal import Permission, RoleNeed
 
 # Permissions
-operator_permission = Permission(RoleNeed('operator'))
-admin_permission = Permission(RoleNeed('admin'))
+operator_permission = Permission(RoleNeed("operator"))
+admin_permission = Permission(RoleNeed("admin"))
 
-CertificateOwner = namedtuple('certificate', ['method', 'value'])
-CertificateOwnerNeed = partial(CertificateOwner, 'role')
+CertificateOwner = namedtuple("certificate", ["method", "value"])
+CertificateOwnerNeed = partial(CertificateOwner, "role")
 
 
 class SensitiveDomainPermission(Permission):
     def __init__(self):
-        super(SensitiveDomainPermission, self).__init__(RoleNeed('admin'))
+        needs = [RoleNeed("admin")]
+        sensitive_domain_roles = current_app.config.get("SENSITIVE_DOMAIN_ROLES", [])
+
+        if sensitive_domain_roles:
+            for role in sensitive_domain_roles:
+                needs.append(RoleNeed(role))
+
+        super(SensitiveDomainPermission, self).__init__(*needs)
 
 
 class CertificatePermission(Permission):
     def __init__(self, owner, roles):
-        needs = [RoleNeed('admin'), RoleNeed(owner), RoleNeed('creator')]
+        needs = [RoleNeed("admin"), RoleNeed(owner), RoleNeed("creator")]
         for r in roles:
             needs.append(CertificateOwnerNeed(str(r)))
             # Backwards compatibility with mixed-case role names
@@ -38,29 +46,29 @@ class CertificatePermission(Permission):
 
 class ApiKeyCreatorPermission(Permission):
     def __init__(self):
-        super(ApiKeyCreatorPermission, self).__init__(RoleNeed('admin'))
+        super(ApiKeyCreatorPermission, self).__init__(RoleNeed("admin"))
 
 
-RoleMember = namedtuple('role', ['method', 'value'])
-RoleMemberNeed = partial(RoleMember, 'member')
+RoleMember = namedtuple("role", ["method", "value"])
+RoleMemberNeed = partial(RoleMember, "member")
 
 
 class RoleMemberPermission(Permission):
     def __init__(self, role_id):
-        needs = [RoleNeed('admin'), RoleMemberNeed(role_id)]
+        needs = [RoleNeed("admin"), RoleMemberNeed(role_id)]
         super(RoleMemberPermission, self).__init__(*needs)
 
 
-AuthorityCreator = namedtuple('authority', ['method', 'value'])
-AuthorityCreatorNeed = partial(AuthorityCreator, 'authorityUse')
+AuthorityCreator = namedtuple("authority", ["method", "value"])
+AuthorityCreatorNeed = partial(AuthorityCreator, "authorityUse")
 
-AuthorityOwner = namedtuple('authority', ['method', 'value'])
-AuthorityOwnerNeed = partial(AuthorityOwner, 'role')
+AuthorityOwner = namedtuple("authority", ["method", "value"])
+AuthorityOwnerNeed = partial(AuthorityOwner, "role")
 
 
 class AuthorityPermission(Permission):
     def __init__(self, authority_id, roles):
-        needs = [RoleNeed('admin'), AuthorityCreatorNeed(str(authority_id))]
+        needs = [RoleNeed("admin"), AuthorityCreatorNeed(str(authority_id))]
         for r in roles:
             needs.append(AuthorityOwnerNeed(str(r)))
 

lemur/auth/service.py

@@ -39,13 +39,13 @@ def get_rsa_public_key(n, e):
     :param e:
     :return: a RSA Public Key in PEM format
     """
-    n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, 'utf-8'))), 16)
-    e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, 'utf-8'))), 16)
+    n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, "utf-8"))), 16)
+    e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, "utf-8"))), 16)
 
     pub = RSAPublicNumbers(e, n).public_key(default_backend())
     return pub.public_bytes(
         encoding=serialization.Encoding.PEM,
-        format=serialization.PublicFormat.SubjectPublicKeyInfo
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
     )
 
 
@@ -57,28 +57,27 @@ def create_token(user, aid=None, ttl=None):
     :param user:
     :return:
     """
-    expiration_delta = timedelta(days=int(current_app.config.get('LEMUR_TOKEN_EXPIRATION', 1)))
-    payload = {
-        'iat': datetime.utcnow(),
-        'exp': datetime.utcnow() + expiration_delta
-    }
+    expiration_delta = timedelta(
+        days=int(current_app.config.get("LEMUR_TOKEN_EXPIRATION", 1))
+    )
+    payload = {"iat": datetime.utcnow(), "exp": datetime.utcnow() + expiration_delta}
 
     # Handle Just a User ID & User Object.
     if isinstance(user, int):
-        payload['sub'] = user
+        payload["sub"] = user
     else:
-        payload['sub'] = user.id
+        payload["sub"] = user.id
     if aid is not None:
-        payload['aid'] = aid
+        payload["aid"] = aid
     # Custom TTLs are only supported on Access Keys.
     if ttl is not None and aid is not None:
         # Tokens that are forever until revoked.
         if ttl == -1:
-            del payload['exp']
+            del payload["exp"]
         else:
-            payload['exp'] = ttl
-    token = jwt.encode(payload, current_app.config['LEMUR_TOKEN_SECRET'])
-    return token.decode('unicode_escape')
+            payload["exp"] = ttl
+    token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"])
+    return token.decode("unicode_escape")
 
 
 def login_required(f):
@@ -88,49 +87,54 @@ def login_required(f):
     :param f:
     :return:
     """
+
     @wraps(f)
     def decorated_function(*args, **kwargs):
-        if not request.headers.get('Authorization'):
-            response = jsonify(message='Missing authorization header')
+        if not request.headers.get("Authorization"):
+            response = jsonify(message="Missing authorization header")
             response.status_code = 401
             return response
 
         try:
-            token = request.headers.get('Authorization').split()[1]
+            token = request.headers.get("Authorization").split()[1]
         except Exception as e:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
 
         try:
-            payload = jwt.decode(token, current_app.config['LEMUR_TOKEN_SECRET'])
+            payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"])
         except jwt.DecodeError:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
         except jwt.ExpiredSignatureError:
-            return dict(message='Token has expired'), 403
+            return dict(message="Token has expired"), 403
         except jwt.InvalidTokenError:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
 
-        if 'aid' in payload:
-            access_key = api_key_service.get(payload['aid'])
+        if "aid" in payload:
+            access_key = api_key_service.get(payload["aid"])
             if access_key.revoked:
-                return dict(message='Token has been revoked'), 403
+                return dict(message="Token has been revoked"), 403
             if access_key.ttl != -1:
                 current_time = datetime.utcnow()
-                expired_time = datetime.fromtimestamp(access_key.issued_at + access_key.ttl)
+                expired_time = datetime.fromtimestamp(
+                    access_key.issued_at + access_key.ttl
+                )
                 if current_time >= expired_time:
-                    return dict(message='Token has expired'), 403
+                    return dict(message="Token has expired"), 403
 
-        user = user_service.get(payload['sub'])
+        user = user_service.get(payload["sub"])
 
         if not user.active:
-            return dict(message='User is not currently active'), 403
+            return dict(message="User is not currently active"), 403
 
         g.current_user = user
 
         if not g.current_user:
-            return dict(message='You are not logged in'), 403
+            return dict(message="You are not logged in"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(g.current_user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(g.current_user.id)
+        )
 
         return f(*args, **kwargs)
 
@@ -144,18 +148,18 @@ def fetch_token_header(token):
     :param token:
     :return: :raise jwt.DecodeError:
     """
-    token = token.encode('utf-8')
+    token = token.encode("utf-8")
     try:
-        signing_input, crypto_segment = token.rsplit(b'.', 1)
-        header_segment, payload_segment = signing_input.split(b'.', 1)
+        signing_input, crypto_segment = token.rsplit(b".", 1)
+        header_segment, payload_segment = signing_input.split(b".", 1)
     except ValueError:
-        raise jwt.DecodeError('Not enough segments')
+        raise jwt.DecodeError("Not enough segments")
 
     try:
-        return json.loads(jwt.utils.base64url_decode(header_segment).decode('utf-8'))
+        return json.loads(jwt.utils.base64url_decode(header_segment).decode("utf-8"))
     except TypeError as e:
         current_app.logger.exception(e)
-        raise jwt.DecodeError('Invalid header padding')
+        raise jwt.DecodeError("Invalid header padding")
 
 
 @identity_loaded.connect
@@ -174,13 +178,13 @@ def on_identity_loaded(sender, identity):
     identity.provides.add(UserNeed(identity.id))
 
     # identity with the roles that the user provides
-    if hasattr(user, 'roles'):
+    if hasattr(user, "roles"):
         for role in user.roles:
             identity.provides.add(RoleNeed(role.name))
             identity.provides.add(RoleMemberNeed(role.id))
 
     # apply ownership for authorities
-    if hasattr(user, 'authorities'):
+    if hasattr(user, "authorities"):
         for authority in user.authorities:
             identity.provides.add(AuthorityCreatorNeed(authority.id))
 
@@ -191,6 +195,7 @@ class AuthenticatedResource(Resource):
     """
     Inherited by all resources that need to be protected by authentication.
     """
+
     method_decorators = [login_required]
 
     def __init__(self):

lemur/auth/views.py

@@ -24,11 +24,13 @@ from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_
 from lemur.auth import ldap
 
 
-mod = Blueprint('auth', __name__)
+mod = Blueprint("auth", __name__)
 api = Api(mod)
 
 
-def exchange_for_access_token(code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True):
+def exchange_for_access_token(
+    code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True
+):
     """
     Exchanges authorization code for access token.
 
@@ -43,28 +45,32 @@ def exchange_for_access_token(code, redirect_uri, client_id, secret, access_toke
     """
     # take the information we have received from the provider to create a new request
     params = {
-        'grant_type': 'authorization_code',
-        'scope': 'openid email profile address',
-        'code': code,
-        'redirect_uri': redirect_uri,
-        'client_id': client_id
+        "grant_type": "authorization_code",
+        "scope": "openid email profile address",
+        "code": code,
+        "redirect_uri": redirect_uri,
+        "client_id": client_id,
     }
 
     # the secret and cliendId will be given to you when you signup for the provider
-    token = '{0}:{1}'.format(client_id, secret)
+    token = "{0}:{1}".format(client_id, secret)
 
-    basic = base64.b64encode(bytes(token, 'utf-8'))
+    basic = base64.b64encode(bytes(token, "utf-8"))
     headers = {
-        'Content-Type': 'application/x-www-form-urlencoded',
-        'authorization': 'basic {0}'.format(basic.decode('utf-8'))
+        "Content-Type": "application/x-www-form-urlencoded",
+        "authorization": "basic {0}".format(basic.decode("utf-8")),
     }
 
     # exchange authorization code for access token.
-    r = requests.post(access_token_url, headers=headers, params=params, verify=verify_cert)
+    r = requests.post(
+        access_token_url, headers=headers, params=params, verify=verify_cert
+    )
     if r.status_code == 400:
-        r = requests.post(access_token_url, headers=headers, data=params, verify=verify_cert)
-    id_token = r.json()['id_token']
-    access_token = r.json()['access_token']
+        r = requests.post(
+            access_token_url, headers=headers, data=params, verify=verify_cert
+        )
+    id_token = r.json()["id_token"]
+    access_token = r.json()["access_token"]
 
     return id_token, access_token
 
@@ -83,23 +89,25 @@ def validate_id_token(id_token, client_id, jwks_url):
 
     # retrieve the key material as specified by the token header
     r = requests.get(jwks_url)
-    for key in r.json()['keys']:
-        if key['kid'] == header_data['kid']:
-            secret = get_rsa_public_key(key['n'], key['e'])
-            algo = header_data['alg']
+    for key in r.json()["keys"]:
+        if key["kid"] == header_data["kid"]:
+            secret = get_rsa_public_key(key["n"], key["e"])
+            algo = header_data["alg"]
             break
     else:
-        return dict(message='Key not found'), 401
+        return dict(message="Key not found"), 401
 
     # validate your token based on the key it was signed with
     try:
-        jwt.decode(id_token, secret.decode('utf-8'), algorithms=[algo], audience=client_id)
+        jwt.decode(
+            id_token, secret.decode("utf-8"), algorithms=[algo], audience=client_id
+        )
     except jwt.DecodeError:
-        return dict(message='Token is invalid'), 401
+        return dict(message="Token is invalid"), 401
     except jwt.ExpiredSignatureError:
-        return dict(message='Token has expired'), 401
+        return dict(message="Token has expired"), 401
     except jwt.InvalidTokenError:
-        return dict(message='Token is invalid'), 401
+        return dict(message="Token is invalid"), 401
 
 
 def retrieve_user(user_api_url, access_token):
@@ -110,13 +118,18 @@ def retrieve_user(user_api_url, access_token):
     :param access_token:
     :return:
     """
-    user_params = dict(access_token=access_token, schema='profile')
+    user_params = dict(access_token=access_token, schema="profile")
+
+    headers = {}
+
+    if current_app.config.get("PING_INCLUDE_BEARER_TOKEN"):
+        headers = {"Authorization": f"Bearer {access_token}"}
 
     # retrieve information about the current user.
-    r = requests.get(user_api_url, params=user_params)
+    r = requests.get(user_api_url, params=user_params, headers=headers)
     profile = r.json()
 
-    user = user_service.get_by_email(profile['email'])
+    user = user_service.get_by_email(profile["email"])
     return user, profile
 
 
@@ -129,28 +142,44 @@ def create_user_roles(profile):
     roles = []
 
     # update their google 'roles'
-    for group in profile['googleGroups']:
+    if "googleGroups" in profile:
+        for group in profile["googleGroups"]:
             role = role_service.get_by_name(group)
             if not role:
-            role = role_service.create(group, description='This is a google group based role created by Lemur', third_party=True)
+                role = role_service.create(
+                    group,
+                    description="This is a google group based role created by Lemur",
+                    third_party=True,
+                )
             if not role.third_party:
                 role = role_service.set_third_party(role.id, third_party_status=True)
             roles.append(role)
+    else:
+        current_app.logger.warning(
+            "'googleGroups' not sent by identity provider, no specific roles will assigned to the user."
+        )
 
-    role = role_service.get_by_name(profile['email'])
+    role = role_service.get_by_name(profile["email"])
 
     if not role:
-        role = role_service.create(profile['email'], description='This is a user specific role', third_party=True)
+        role = role_service.create(
+            profile["email"],
+            description="This is a user specific role",
+            third_party=True,
+        )
     if not role.third_party:
         role = role_service.set_third_party(role.id, third_party_status=True)
 
     roles.append(role)
 
     # every user is an operator (tied to a default role)
-    if current_app.config.get('LEMUR_DEFAULT_ROLE'):
-        default = role_service.get_by_name(current_app.config['LEMUR_DEFAULT_ROLE'])
+    if current_app.config.get("LEMUR_DEFAULT_ROLE"):
+        default = role_service.get_by_name(current_app.config["LEMUR_DEFAULT_ROLE"])
         if not default:
-            default = role_service.create(current_app.config['LEMUR_DEFAULT_ROLE'], description='This is the default Lemur role.')
+            default = role_service.create(
+                current_app.config["LEMUR_DEFAULT_ROLE"],
+                description="This is the default Lemur role.",
+            )
         if not default.third_party:
             role_service.set_third_party(default.id, third_party_status=True)
         roles.append(default)
@@ -169,12 +198,12 @@ def update_user(user, profile, roles):
     # if we get an sso user create them an account
     if not user:
         user = user_service.create(
-            profile['email'],
+            profile["email"],
             get_psuedo_random_string(),
-            profile['email'],
+            profile["email"],
             True,
-            profile.get('thumbnailPhotoUrl'),
-            roles
+            profile.get("thumbnailPhotoUrl"),
+            roles,
         )
 
     else:
@@ -186,11 +215,11 @@ def update_user(user, profile, roles):
         # update any changes to the user
         user_service.update(
             user.id,
-            profile['email'],
-            profile['email'],
+            profile["email"],
+            profile["email"],
             True,
-            profile.get('thumbnailPhotoUrl'),  # profile isn't google+ enabled
-            roles
+            profile.get("thumbnailPhotoUrl"),  # profile isn't google+ enabled
+            roles,
         )
 
 
@@ -211,6 +240,7 @@ class Login(Resource):
     on your uses cases but. It is important to not that there is currently no build in method to revoke a users token \
     and force re-authentication.
     """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Login, self).__init__()
@@ -251,23 +281,26 @@ class Login(Resource):
            :statuscode 401: invalid credentials
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('username', type=str, required=True, location='json')
-        self.reqparse.add_argument('password', type=str, required=True, location='json')
+        self.reqparse.add_argument("username", type=str, required=True, location="json")
+        self.reqparse.add_argument("password", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
-        if '@' in args['username']:
-            user = user_service.get_by_email(args['username'])
+        if "@" in args["username"]:
+            user = user_service.get_by_email(args["username"])
         else:
-            user = user_service.get_by_username(args['username'])
+            user = user_service.get_by_username(args["username"])
 
         # default to local authentication
-        if user and user.check_password(args['password']) and user.active:
+        if user and user.check_password(args["password"]) and user.active:
             # Tell Flask-Principal the identity changed
-            identity_changed.send(current_app._get_current_object(),
-                                  identity=Identity(user.id))
+            identity_changed.send(
+                current_app._get_current_object(), identity=Identity(user.id)
+            )
 
-            metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+            )
             return dict(token=create_token(user))
 
         # try ldap login
@@ -277,19 +310,29 @@ class Login(Resource):
                 user = ldap_principal.authenticate()
                 if user and user.active:
                     # Tell Flask-Principal the identity changed
-                    identity_changed.send(current_app._get_current_object(),
-                                  identity=Identity(user.id))
-                    metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+                    identity_changed.send(
+                        current_app._get_current_object(), identity=Identity(user.id)
+                    )
+                    metrics.send(
+                        "login",
+                        "counter",
+                        1,
+                        metric_tags={"status": SUCCESS_METRIC_STATUS},
+                    )
                     return dict(token=create_token(user))
             except Exception as e:
                 current_app.logger.error("ldap error: {0}".format(e))
-                    ldap_message = 'ldap error: %s' % e
-                    metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+                ldap_message = "ldap error: %s" % e
+                metrics.send(
+                    "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+                )
                 return dict(message=ldap_message), 403
 
         # if not valid user - no certificates for you
-        metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-        return dict(message='The supplied credentials are invalid'), 403
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+        )
+        return dict(message="The supplied credentials are invalid"), 403
 
 
 class Ping(Resource):
@@ -302,49 +345,59 @@ class Ping(Resource):
     provider uses for its callbacks.
     2. Add or change the Lemur AngularJS Configuration to point to your new provider
     """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Ping, self).__init__()
 
     def get(self):
-        return 'Redirecting...'
+        return "Redirecting..."
 
     def post(self):
-        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
-        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
-        self.reqparse.add_argument('code', type=str, required=True, location='json')
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
         # you can either discover these dynamically or simply configure them
-        access_token_url = current_app.config.get('PING_ACCESS_TOKEN_URL')
-        user_api_url = current_app.config.get('PING_USER_API_URL')
+        access_token_url = current_app.config.get("PING_ACCESS_TOKEN_URL")
+        user_api_url = current_app.config.get("PING_USER_API_URL")
 
-        secret = current_app.config.get('PING_SECRET')
+        secret = current_app.config.get("PING_SECRET")
 
         id_token, access_token = exchange_for_access_token(
-            args['code'],
-            args['redirectUri'],
-            args['clientId'],
+            args["code"],
+            args["redirectUri"],
+            args["clientId"],
             secret,
-            access_token_url=access_token_url
+            access_token_url=access_token_url,
         )
 
-        jwks_url = current_app.config.get('PING_JWKS_URL')
-        validate_id_token(id_token, args['clientId'], jwks_url)
-
+        jwks_url = current_app.config.get("PING_JWKS_URL")
+        error_code = validate_id_token(id_token, args["clientId"], jwks_url)
+        if error_code:
+            return error_code
         user, profile = retrieve_user(user_api_url, access_token)
         roles = create_user_roles(profile)
         update_user(user, profile, roles)
 
         if not user or not user.active:
-            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-            return dict(message='The supplied credentials are invalid'), 403
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(user.id)
+        )
 
-        metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+        )
         return dict(token=create_token(user))
 
 
@@ -354,46 +407,56 @@ class OAuth2(Resource):
         super(OAuth2, self).__init__()
 
     def get(self):
-        return 'Redirecting...'
+        return "Redirecting..."
 
     def post(self):
-        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
-        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
-        self.reqparse.add_argument('code', type=str, required=True, location='json')
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
         # you can either discover these dynamically or simply configure them
-        access_token_url = current_app.config.get('OAUTH2_ACCESS_TOKEN_URL')
-        user_api_url = current_app.config.get('OAUTH2_USER_API_URL')
-        verify_cert = current_app.config.get('OAUTH2_VERIFY_CERT')
+        access_token_url = current_app.config.get("OAUTH2_ACCESS_TOKEN_URL")
+        user_api_url = current_app.config.get("OAUTH2_USER_API_URL")
+        verify_cert = current_app.config.get("OAUTH2_VERIFY_CERT")
 
-        secret = current_app.config.get('OAUTH2_SECRET')
+        secret = current_app.config.get("OAUTH2_SECRET")
 
         id_token, access_token = exchange_for_access_token(
-            args['code'],
-            args['redirectUri'],
-            args['clientId'],
+            args["code"],
+            args["redirectUri"],
+            args["clientId"],
             secret,
             access_token_url=access_token_url,
-            verify_cert=verify_cert
+            verify_cert=verify_cert,
         )
 
-        jwks_url = current_app.config.get('PING_JWKS_URL')
-        validate_id_token(id_token, args['clientId'], jwks_url)
+        jwks_url = current_app.config.get("PING_JWKS_URL")
+        error_code = validate_id_token(id_token, args["clientId"], jwks_url)
+        if error_code:
+            return error_code
 
         user, profile = retrieve_user(user_api_url, access_token)
         roles = create_user_roles(profile)
         update_user(user, profile, roles)
 
         if not user.active:
-            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-            return dict(message='The supplied credentials are invalid'), 403
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(user.id)
+        )
 
-        metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+        )
 
         return dict(token=create_token(user))
 
@@ -404,44 +467,52 @@ class Google(Resource):
         super(Google, self).__init__()
 
     def post(self):
-        access_token_url = 'https://accounts.google.com/o/oauth2/token'
-        people_api_url = 'https://www.googleapis.com/plus/v1/people/me/openIdConnect'
+        access_token_url = "https://accounts.google.com/o/oauth2/token"
+        people_api_url = "https://www.googleapis.com/plus/v1/people/me/openIdConnect"
 
-        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
-        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
-        self.reqparse.add_argument('code', type=str, required=True, location='json')
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
         # Step 1. Exchange authorization code for access token
         payload = {
-            'client_id': args['clientId'],
-            'grant_type': 'authorization_code',
-            'redirect_uri': args['redirectUri'],
-            'code': args['code'],
-            'client_secret': current_app.config.get('GOOGLE_SECRET')
+            "client_id": args["clientId"],
+            "grant_type": "authorization_code",
+            "redirect_uri": args["redirectUri"],
+            "code": args["code"],
+            "client_secret": current_app.config.get("GOOGLE_SECRET"),
         }
 
         r = requests.post(access_token_url, data=payload)
         token = r.json()
 
         # Step 2. Retrieve information about the current user
-        headers = {'Authorization': 'Bearer {0}'.format(token['access_token'])}
+        headers = {"Authorization": "Bearer {0}".format(token["access_token"])}
 
         r = requests.get(people_api_url, headers=headers)
         profile = r.json()
 
-        user = user_service.get_by_email(profile['email'])
+        user = user_service.get_by_email(profile["email"])
 
         if not (user and user.active):
-            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-            return dict(message='The supplied credentials are invalid.'), 403
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid."), 403
 
         if user:
-            metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+            )
             return dict(token=create_token(user))
 
-        metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+        )
 
 
 class Providers(Resource):
@@ -452,47 +523,57 @@ class Providers(Resource):
             provider = provider.lower()
 
             if provider == "google":
-                active_providers.append({
-                    'name': 'google',
-                    'clientId': current_app.config.get("GOOGLE_CLIENT_ID"),
-                    'url': api.url_for(Google)
-                })
+                active_providers.append(
+                    {
+                        "name": "google",
+                        "clientId": current_app.config.get("GOOGLE_CLIENT_ID"),
+                        "url": api.url_for(Google),
+                    }
+                )
 
             elif provider == "ping":
-                active_providers.append({
-                    'name': current_app.config.get("PING_NAME"),
-                    'url': current_app.config.get('PING_REDIRECT_URI'),
-                    'redirectUri': current_app.config.get("PING_REDIRECT_URI"),
-                    'clientId': current_app.config.get("PING_CLIENT_ID"),
-                    'responseType': 'code',
-                    'scope': ['openid', 'email', 'profile', 'address'],
-                    'scopeDelimiter': ' ',
-                    'authorizationEndpoint': current_app.config.get("PING_AUTH_ENDPOINT"),
-                    'requiredUrlParams': ['scope'],
-                    'type': '2.0'
-                })
+                active_providers.append(
+                    {
+                        "name": current_app.config.get("PING_NAME"),
+                        "url": current_app.config.get("PING_REDIRECT_URI"),
+                        "redirectUri": current_app.config.get("PING_REDIRECT_URI"),
+                        "clientId": current_app.config.get("PING_CLIENT_ID"),
+                        "responseType": "code",
+                        "scope": ["openid", "email", "profile", "address"],
+                        "scopeDelimiter": " ",
+                        "authorizationEndpoint": current_app.config.get(
+                            "PING_AUTH_ENDPOINT"
+                        ),
+                        "requiredUrlParams": ["scope"],
+                        "type": "2.0",
+                    }
+                )
 
             elif provider == "oauth2":
-                active_providers.append({
-                    'name': current_app.config.get("OAUTH2_NAME"),
-                    'url': current_app.config.get('OAUTH2_REDIRECT_URI'),
-                    'redirectUri': current_app.config.get("OAUTH2_REDIRECT_URI"),
-                    'clientId': current_app.config.get("OAUTH2_CLIENT_ID"),
-                    'responseType': 'code',
-                    'scope': ['openid', 'email', 'profile', 'groups'],
-                    'scopeDelimiter': ' ',
-                    'authorizationEndpoint': current_app.config.get("OAUTH2_AUTH_ENDPOINT"),
-                    'requiredUrlParams': ['scope', 'state', 'nonce'],
-                    'state': 'STATE',
-                    'nonce': get_psuedo_random_string(),
-                    'type': '2.0'
-                })
+                active_providers.append(
+                    {
+                        "name": current_app.config.get("OAUTH2_NAME"),
+                        "url": current_app.config.get("OAUTH2_REDIRECT_URI"),
+                        "redirectUri": current_app.config.get("OAUTH2_REDIRECT_URI"),
+                        "clientId": current_app.config.get("OAUTH2_CLIENT_ID"),
+                        "responseType": "code",
+                        "scope": ["openid", "email", "profile", "groups"],
+                        "scopeDelimiter": " ",
+                        "authorizationEndpoint": current_app.config.get(
+                            "OAUTH2_AUTH_ENDPOINT"
+                        ),
+                        "requiredUrlParams": ["scope", "state", "nonce"],
+                        "state": "STATE",
+                        "nonce": get_psuedo_random_string(),
+                        "type": "2.0",
+                    }
+                )
 
         return active_providers
 
 
-api.add_resource(Login, '/auth/login', endpoint='login')
-api.add_resource(Ping, '/auth/ping', endpoint='ping')
-api.add_resource(Google, '/auth/google', endpoint='google')
-api.add_resource(OAuth2, '/auth/oauth2', endpoint='oauth2')
-api.add_resource(Providers, '/auth/providers', endpoint='providers')
+api.add_resource(Login, "/auth/login", endpoint="login")
+api.add_resource(Ping, "/auth/ping", endpoint="ping")
+api.add_resource(Google, "/auth/google", endpoint="google")
+api.add_resource(OAuth2, "/auth/oauth2", endpoint="oauth2")
+api.add_resource(Providers, "/auth/providers", endpoint="providers")

lemur/authorities/models.py

@@ -7,7 +7,17 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from sqlalchemy.orm import relationship
-from sqlalchemy import Column, Integer, String, Text, func, ForeignKey, DateTime, PassiveDefault, Boolean
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    Text,
+    func,
+    ForeignKey,
+    DateTime,
+    PassiveDefault,
+    Boolean,
+)
 from sqlalchemy.dialects.postgresql import JSON
 
 from lemur.database import db
@@ -16,7 +26,7 @@ from lemur.models import roles_authorities
 
 
 class Authority(db.Model):
-    __tablename__ = 'authorities'
+    __tablename__ = "authorities"
     id = Column(Integer, primary_key=True)
     owner = Column(String(128), nullable=False)
     name = Column(String(128), unique=True)
@@ -27,22 +37,44 @@ class Authority(db.Model):
     description = Column(Text)
     options = Column(JSON)
     date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
-    roles = relationship('Role', secondary=roles_authorities, passive_deletes=True, backref=db.backref('authority'), lazy='dynamic')
-    user_id = Column(Integer, ForeignKey('users.id'))
-    authority_certificate = relationship("Certificate", backref='root_authority', uselist=False, foreign_keys='Certificate.root_authority_id')
-    certificates = relationship("Certificate", backref='authority', foreign_keys='Certificate.authority_id')
+    roles = relationship(
+        "Role",
+        secondary=roles_authorities,
+        passive_deletes=True,
+        backref=db.backref("authority"),
+        lazy="dynamic",
+    )
+    user_id = Column(Integer, ForeignKey("users.id"))
+    authority_certificate = relationship(
+        "Certificate",
+        backref="root_authority",
+        uselist=False,
+        foreign_keys="Certificate.root_authority_id",
+    )
+    certificates = relationship(
+        "Certificate", backref="authority", foreign_keys="Certificate.authority_id"
+    )
 
-    authority_pending_certificate = relationship("PendingCertificate", backref='root_authority', uselist=False, foreign_keys='PendingCertificate.root_authority_id')
-    pending_certificates = relationship('PendingCertificate', backref='authority', foreign_keys='PendingCertificate.authority_id')
+    authority_pending_certificate = relationship(
+        "PendingCertificate",
+        backref="root_authority",
+        uselist=False,
+        foreign_keys="PendingCertificate.root_authority_id",
+    )
+    pending_certificates = relationship(
+        "PendingCertificate",
+        backref="authority",
+        foreign_keys="PendingCertificate.authority_id",
+    )
 
     def __init__(self, **kwargs):
-        self.owner = kwargs['owner']
-        self.roles = kwargs.get('roles', [])
-        self.name = kwargs.get('name')
-        self.description = kwargs.get('description')
-        self.authority_certificate = kwargs['authority_certificate']
-        self.plugin_name = kwargs['plugin']['slug']
-        self.options = kwargs.get('options')
+        self.owner = kwargs["owner"]
+        self.roles = kwargs.get("roles", [])
+        self.name = kwargs.get("name")
+        self.description = kwargs.get("description")
+        self.authority_certificate = kwargs["authority_certificate"]
+        self.plugin_name = kwargs["plugin"]["slug"]
+        self.options = kwargs.get("options")
 
     @property
     def plugin(self):

lemur/authorities/schemas.py

@@ -11,7 +11,13 @@ from marshmallow import fields, validates_schema, pre_load
 from marshmallow import validate
 from marshmallow.exceptions import ValidationError
 
-from lemur.schemas import PluginInputSchema, PluginOutputSchema, ExtensionSchema, AssociatedAuthoritySchema, AssociatedRoleSchema
+from lemur.schemas import (
+    PluginInputSchema,
+    PluginOutputSchema,
+    ExtensionSchema,
+    AssociatedAuthoritySchema,
+    AssociatedRoleSchema,
+)
 from lemur.users.schemas import UserNestedOutputSchema
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.common import validators, missing
@@ -30,21 +36,36 @@ class AuthorityInputSchema(LemurInputSchema):
     validity_years = fields.Integer()
 
     # certificate body fields
-    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
-    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
-    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
-    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
-    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+    organizational_unit = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATIONAL_UNIT")
+    )
+    organization = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATION")
+    )
+    location = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_LOCATION")
+    )
+    country = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_COUNTRY")
+    )
+    state = fields.String(missing=lambda: current_app.config.get("LEMUR_DEFAULT_STATE"))
 
     plugin = fields.Nested(PluginInputSchema)
 
     # signing related options
-    type = fields.String(validate=validate.OneOf(['root', 'subca']), missing='root')
+    type = fields.String(validate=validate.OneOf(["root", "subca"]), missing="root")
     parent = fields.Nested(AssociatedAuthoritySchema)
-    signing_algorithm = fields.String(validate=validate.OneOf(['sha256WithRSA', 'sha1WithRSA']), missing='sha256WithRSA')
-    key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')
+    signing_algorithm = fields.String(
+        validate=validate.OneOf(["sha256WithRSA", "sha1WithRSA"]),
+        missing="sha256WithRSA",
+    )
+    key_type = fields.String(
+        validate=validate.OneOf(["RSA2048", "RSA4096"]), missing="RSA2048"
+    )
     key_name = fields.String()
-    sensitivity = fields.String(validate=validate.OneOf(['medium', 'high']), missing='medium')
+    sensitivity = fields.String(
+        validate=validate.OneOf(["medium", "high"]), missing="medium"
+    )
     serial_number = fields.Integer()
     first_serial = fields.Integer(missing=1)
 
@@ -58,9 +79,11 @@ class AuthorityInputSchema(LemurInputSchema):
 
     @validates_schema
     def validate_subca(self, data):
-        if data['type'] == 'subca':
-            if not data.get('parent'):
-                raise ValidationError("If generating a subca, parent 'authority' must be specified.")
+        if data["type"] == "subca":
+            if not data.get("parent"):
+                raise ValidationError(
+                    "If generating a subca, parent 'authority' must be specified."
+                )
 
     @pre_load
     def ensure_dates(self, data):

lemur/authorities/service.py

@@ -15,6 +15,7 @@ from lemur import database
 from lemur.common.utils import truthiness
 from lemur.extensions import metrics
 from lemur.authorities.models import Authority
+from lemur.certificates.models import Certificate
 from lemur.roles import service as role_service
 
 from lemur.certificates.service import upload
@@ -42,7 +43,7 @@ def mint(**kwargs):
     """
     Creates the authority based on the plugin provided.
     """
-    issuer = kwargs['plugin']['plugin_object']
+    issuer = kwargs["plugin"]["plugin_object"]
     values = issuer.create_authority(kwargs)
 
     # support older plugins
@@ -52,7 +53,12 @@ def mint(**kwargs):
     elif len(values) == 4:
         body, private_key, chain, roles = values
 
-    roles = create_authority_roles(roles, kwargs['owner'], kwargs['plugin']['plugin_object'].title, kwargs['creator'])
+    roles = create_authority_roles(
+        roles,
+        kwargs["owner"],
+        kwargs["plugin"]["plugin_object"].title,
+        kwargs["creator"],
+    )
     return body, private_key, chain, roles
 
 
@@ -65,16 +71,17 @@ def create_authority_roles(roles, owner, plugin_title, creator):
     """
     role_objs = []
     for r in roles:
-        role = role_service.get_by_name(r['name'])
+        role = role_service.get_by_name(r["name"])
         if not role:
             role = role_service.create(
-                r['name'],
-                password=r['password'],
+                r["name"],
+                password=r["password"],
                 description="Auto generated role for {0}".format(plugin_title),
-                username=r['username'])
+                username=r["username"],
+            )
 
         # the user creating the authority should be able to administer it
-        if role.username == 'admin':
+        if role.username == "admin":
             creator.roles.append(role)
 
         role_objs.append(role)
@@ -83,8 +90,7 @@ def create_authority_roles(roles, owner, plugin_title, creator):
     owner_role = role_service.get_by_name(owner)
     if not owner_role:
         owner_role = role_service.create(
-            owner,
-            description="Auto generated role based on owner: {0}".format(owner)
+            owner, description="Auto generated role based on owner: {0}".format(owner)
         )
 
     role_objs.append(owner_role)
@@ -97,27 +103,29 @@ def create(**kwargs):
     """
     body, private_key, chain, roles = mint(**kwargs)
 
-    kwargs['creator'].roles = list(set(list(kwargs['creator'].roles) + roles))
+    kwargs["creator"].roles = list(set(list(kwargs["creator"].roles) + roles))
 
-    kwargs['body'] = body
-    kwargs['private_key'] = private_key
-    kwargs['chain'] = chain
+    kwargs["body"] = body
+    kwargs["private_key"] = private_key
+    kwargs["chain"] = chain
 
-    if kwargs.get('roles'):
-        kwargs['roles'] += roles
+    if kwargs.get("roles"):
+        kwargs["roles"] += roles
     else:
-        kwargs['roles'] = roles
+        kwargs["roles"] = roles
 
     cert = upload(**kwargs)
-    kwargs['authority_certificate'] = cert
-    if kwargs.get('plugin', {}).get('plugin_options', []):
-        kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options'])
+    kwargs["authority_certificate"] = cert
+    if kwargs.get("plugin", {}).get("plugin_options", []):
+        kwargs["options"] = json.dumps(kwargs["plugin"]["plugin_options"])
 
     authority = Authority(**kwargs)
     authority = database.create(authority)
-    kwargs['creator'].authorities.append(authority)
+    kwargs["creator"].authorities.append(authority)
 
-    metrics.send('authority_created', 'counter', 1, metric_tags=dict(owner=authority.owner))
+    metrics.send(
+        "authority_created", "counter", 1, metric_tags=dict(owner=authority.owner)
+    )
     return authority
 
 
@@ -149,7 +157,7 @@ def get_by_name(authority_name):
     :param authority_name:
     :return:
     """
-    return database.get(Authority, authority_name, field='name')
+    return database.get(Authority, authority_name, field="name")
 
 
 def get_authority_role(ca_name, creator=None):
@@ -172,24 +180,31 @@ def render(args):
     :return:
     """
     query = database.session_query(Authority)
-    filt = args.pop('filter')
+    filt = args.pop("filter")
 
     if filt:
-        terms = filt.split(';')
-        if 'active' in filt:
+        terms = filt.split(";")
+        if "active" in filt:
             query = query.filter(Authority.active == truthiness(terms[1]))
-        elif 'cn' in filt:
-            query = query.join(Authority.active == truthiness(terms[1]))
+        elif "cn" in filt:
+            term = "%{0}%".format(terms[1])
+            sub_query = (
+                database.session_query(Certificate.root_authority_id)
+                .filter(Certificate.cn.ilike(term))
+                .subquery()
+            )
+
+            query = query.filter(Authority.id.in_(sub_query))
         else:
             query = database.filter(query, Authority, terms)
 
     # we make sure that a user can only use an authority they either own are a member of - admins can see all
-    if not args['user'].is_admin:
+    if not args["user"].is_admin:
         authority_ids = []
-        for authority in args['user'].authorities:
+        for authority in args["user"].authorities:
             authority_ids.append(authority.id)
 
-        for role in args['user'].roles:
+        for role in args["user"].roles:
             for authority in role.authorities:
                 authority_ids.append(authority.id)
         query = query.filter(Authority.id.in_(authority_ids))

lemur/authorities/views.py

@@ -16,15 +16,21 @@ from lemur.auth.permissions import AuthorityPermission
 from lemur.certificates import service as certificate_service
 
 from lemur.authorities import service
-from lemur.authorities.schemas import authority_input_schema, authority_output_schema, authorities_output_schema, authority_update_schema
+from lemur.authorities.schemas import (
+    authority_input_schema,
+    authority_output_schema,
+    authorities_output_schema,
+    authority_update_schema,
+)
 
 
-mod = Blueprint('authorities', __name__)
+mod = Blueprint("authorities", __name__)
 api = Api(mod)
 
 
 class AuthoritiesList(AuthenticatedResource):
     """ Defines the 'authorities' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(AuthoritiesList, self).__init__()
@@ -107,7 +113,7 @@ class AuthoritiesList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['user'] = g.current_user
+        args["user"] = g.current_user
         return service.render(args)
 
     @validate_schema(authority_input_schema, authority_output_schema)
@@ -220,7 +226,7 @@ class AuthoritiesList(AuthenticatedResource):
            :statuscode 403: unauthenticated
            :statuscode 200: no error
         """
-        data['creator'] = g.current_user
+        data["creator"] = g.current_user
         return service.create(**data)
 
 
@@ -388,7 +394,7 @@ class Authorities(AuthenticatedResource):
         authority = service.get(authority_id)
 
         if not authority:
-            return dict(message='Not Found'), 404
+            return dict(message="Not Found"), 404
 
         # all the authority role members should be allowed
         roles = [x.name for x in authority.roles]
@@ -397,10 +403,10 @@ class Authorities(AuthenticatedResource):
         if permission.can():
             return service.update(
                 authority_id,
-                owner=data['owner'],
-                description=data['description'],
-                active=data['active'],
-                roles=data['roles']
+                owner=data["owner"],
+                description=data["description"],
+                active=data["active"],
+                roles=data["roles"],
             )
 
         return dict(message="You are not authorized to update this authority."), 403
@@ -505,10 +511,21 @@ class AuthorityVisualizations(AuthenticatedResource):
         ]}
         """
         authority = service.get(authority_id)
-        return dict(name=authority.name, children=[{"name": c.name} for c in authority.certificates])
+        return dict(
+            name=authority.name,
+            children=[{"name": c.name} for c in authority.certificates],
+        )
 
 
-api.add_resource(AuthoritiesList, '/authorities', endpoint='authorities')
-api.add_resource(Authorities, '/authorities/<int:authority_id>', endpoint='authority')
-api.add_resource(AuthorityVisualizations, '/authorities/<int:authority_id>/visualize', endpoint='authority_visualizations')
-api.add_resource(CertificateAuthority, '/certificates/<int:certificate_id>/authority', endpoint='certificateAuthority')
+api.add_resource(AuthoritiesList, "/authorities", endpoint="authorities")
+api.add_resource(Authorities, "/authorities/<int:authority_id>", endpoint="authority")
+api.add_resource(
+    AuthorityVisualizations,
+    "/authorities/<int:authority_id>/visualize",
+    endpoint="authority_visualizations",
+)
+api.add_resource(
+    CertificateAuthority,
+    "/certificates/<int:certificate_id>/authority",
+    endpoint="certificateAuthority",
+)

lemur/authorizations/models.py

@@ -13,7 +13,7 @@ from lemur.plugins.base import plugins
 
 
 class Authorization(db.Model):
-    __tablename__ = 'pending_dns_authorizations'
+    __tablename__ = "pending_dns_authorizations"
     id = Column(Integer, primary_key=True, autoincrement=True)
     account_number = Column(String(128))
     domains = Column(JSONType)

lemur/certificates/cli.py

@@ -34,7 +34,7 @@ from lemur.certificates.service import (
     get_all_pending_reissue,
     get_by_name,
     get_all_certs,
-    get
+    get,
 )
 
 from lemur.certificates.verify import verify_string
@@ -56,11 +56,14 @@ def print_certificate_details(details):
         "\t[+] Authority: {authority_name}\n"
         "\t[+] Validity Start: {validity_start}\n"
         "\t[+] Validity End: {validity_end}\n".format(
-            common_name=details['commonName'],
-            sans=",".join(x['value'] for x in details['extensions']['subAltNames']['names']) or None,
-            authority_name=details['authority']['name'],
-            validity_start=details['validityStart'],
-            validity_end=details['validityEnd']
+            common_name=details["commonName"],
+            sans=",".join(
+                x["value"] for x in details["extensions"]["subAltNames"]["names"]
+            )
+            or None,
+            authority_name=details["authority"]["name"],
+            validity_start=details["validityStart"],
+            validity_end=details["validityEnd"],
         )
     )
 
@@ -120,13 +123,11 @@ def request_rotation(endpoint, certificate, message, commit):
         except Exception as e:
             print(
                 "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
-                    endpoint.name,
-                    certificate.name,
-                    e
+                    endpoint.name, certificate.name, e
                 )
             )
 
-    metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': status})
+    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status})
 
 
 def request_reissue(certificate, commit):
@@ -153,22 +154,53 @@ def request_reissue(certificate, commit):
         status = SUCCESS_METRIC_STATUS
 
     except Exception as e:
-        sentry.captureException()
-        current_app.logger.exception("Error reissuing certificate.", exc_info=True)
-        print(
-            "[!] Failed to reissue certificates. Reason: {}".format(
-                e
+        sentry.captureException(extra={"certificate_name": str(certificate.name)})
+        current_app.logger.exception(
+            f"Error reissuing certificate: {certificate.name}", exc_info=True
         )
+        print(f"[!] Failed to reissue certificate: {certificate.name}. Reason: {e}")
+
+    metrics.send(
+        "certificate_reissue",
+        "counter",
+        1,
+        metric_tags={"status": status, "certificate": certificate.name},
     )
 
-    metrics.send('certificate_reissue', 'counter', 1, metric_tags={'status': status})
 
-
-@manager.option('-e', '--endpoint', dest='endpoint_name', help='Name of the endpoint you wish to rotate.')
-@manager.option('-n', '--new-certificate', dest='new_certificate_name', help='Name of the certificate you wish to rotate to.')
-@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to rotate.')
-@manager.option('-a', '--notify', dest='message', action='store_true', help='Send a rotation notification to the certificates owner.')
-@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+@manager.option(
+    "-e",
+    "--endpoint",
+    dest="endpoint_name",
+    help="Name of the endpoint you wish to rotate.",
+)
+@manager.option(
+    "-n",
+    "--new-certificate",
+    dest="new_certificate_name",
+    help="Name of the certificate you wish to rotate to.",
+)
+@manager.option(
+    "-o",
+    "--old-certificate",
+    dest="old_certificate_name",
+    help="Name of the certificate you wish to rotate.",
+)
+@manager.option(
+    "-a",
+    "--notify",
+    dest="message",
+    action="store_true",
+    help="Send a rotation notification to the certificates owner.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
 def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, commit):
     """
     Rotates an endpoint and reissues it if it has not already been replaced. If it has
@@ -187,39 +219,90 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
         endpoint = validate_endpoint(endpoint_name)
 
         if endpoint and new_cert:
-            print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name))
+            print(
+                f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
+            )
             request_rotation(endpoint, new_cert, message, commit)
 
         elif old_cert and new_cert:
-            print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name))
+            print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
 
             for endpoint in old_cert.endpoints:
-                print("[+] Rotating {0}".format(endpoint.name))
+                print(f"[+] Rotating {endpoint.name}")
                 request_rotation(endpoint, new_cert, message, commit)
 
         else:
             print("[+] Rotating all endpoints that have new certificates available")
             for endpoint in endpoint_service.get_all_pending_rotation():
                 if len(endpoint.certificate.replaced) == 1:
-                    print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name))
-                    request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
+                    print(
+                        f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
+                    )
+                    request_rotation(
+                        endpoint, endpoint.certificate.replaced[0], message, commit
+                    )
                 else:
-                    metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-                    print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format(
-                        endpoint.name
-                    ))
+                    metrics.send(
+                        "endpoint_rotation",
+                        "counter",
+                        1,
+                        metric_tags={
+                            "status": FAILURE_METRIC_STATUS,
+                            "old_certificate_name": str(old_cert),
+                            "new_certificate_name": str(
+                                endpoint.certificate.replaced[0].name
+                            ),
+                            "endpoint_name": str(endpoint.name),
+                            "message": str(message),
+                        },
+                    )
+                    print(
+                        f"[!] Failed to rotate endpoint {endpoint.name} reason: "
+                        "Multiple replacement certificates found."
+                    )
 
         status = SUCCESS_METRIC_STATUS
         print("[+] Done!")
 
     except Exception as e:
-        sentry.captureException()
+        sentry.captureException(
+            extra={
+                "old_certificate_name": str(old_certificate_name),
+                "new_certificate_name": str(new_certificate_name),
+                "endpoint_name": str(endpoint_name),
+                "message": str(message),
+            }
+        )
 
-    metrics.send('endpoint_rotation_job', 'counter', 1, metric_tags={'status': status})
+    metrics.send(
+        "endpoint_rotation_job",
+        "counter",
+        1,
+        metric_tags={
+            "status": status,
+            "old_certificate_name": str(old_certificate_name),
+            "new_certificate_name": str(new_certificate_name),
+            "endpoint_name": str(endpoint_name),
+            "message": str(message),
+            "endpoint": str(globals().get("endpoint")),
+        },
+    )
 
 
-@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to reissue.')
-@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+@manager.option(
+    "-o",
+    "--old-certificate",
+    dest="old_certificate_name",
+    help="Name of the certificate you wish to reissue.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
 def reissue(old_certificate_name, commit):
     """
     Reissues certificate with the same parameters as it was originally issued with.
@@ -247,76 +330,94 @@ def reissue(old_certificate_name, commit):
     except Exception as e:
         sentry.captureException()
         current_app.logger.exception("Error reissuing certificate.", exc_info=True)
-        print(
-            "[!] Failed to reissue certificates. Reason: {}".format(
-                e
-            )
+        print("[!] Failed to reissue certificates. Reason: {}".format(e))
+
+    metrics.send(
+        "certificate_reissue_job", "counter", 1, metric_tags={"status": status}
     )
 
-    metrics.send('certificate_reissue_job', 'counter', 1, metric_tags={'status': status})
 
-
-@manager.option('-f', '--fqdns', dest='fqdns', help='FQDNs to query. Multiple fqdns specified via comma.')
-@manager.option('-i', '--issuer', dest='issuer', help='Issuer to query for.')
-@manager.option('-o', '--owner', dest='owner', help='Owner to query for.')
-@manager.option('-e', '--expired', dest='expired', type=bool, default=False, help='Include expired certificates.')
+@manager.option(
+    "-f",
+    "--fqdns",
+    dest="fqdns",
+    help="FQDNs to query. Multiple fqdns specified via comma.",
+)
+@manager.option("-i", "--issuer", dest="issuer", help="Issuer to query for.")
+@manager.option("-o", "--owner", dest="owner", help="Owner to query for.")
+@manager.option(
+    "-e",
+    "--expired",
+    dest="expired",
+    type=bool,
+    default=False,
+    help="Include expired certificates.",
+)
 def query(fqdns, issuer, owner, expired):
     """Prints certificates that match the query params."""
     table = []
 
     q = database.session_query(Certificate)
     if issuer:
-        sub_query = database.session_query(Authority.id) \
-            .filter(Authority.name.ilike('%{0}%'.format(issuer))) \
+        sub_query = (
+            database.session_query(Authority.id)
+            .filter(Authority.name.ilike("%{0}%".format(issuer)))
             .subquery()
+        )
 
         q = q.filter(
             or_(
-                Certificate.issuer.ilike('%{0}%'.format(issuer)),
-                Certificate.authority_id.in_(sub_query)
+                Certificate.issuer.ilike("%{0}%".format(issuer)),
+                Certificate.authority_id.in_(sub_query),
             )
         )
     if owner:
-        q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner)))
+        q = q.filter(Certificate.owner.ilike("%{0}%".format(owner)))
 
     if not expired:
         q = q.filter(Certificate.expired == False)  # noqa
 
     if fqdns:
-        for f in fqdns.split(','):
+        for f in fqdns.split(","):
             q = q.filter(
                 or_(
-                    Certificate.cn.ilike('%{0}%'.format(f)),
-                    Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f)))
+                    Certificate.cn.ilike("%{0}%".format(f)),
+                    Certificate.domains.any(Domain.name.ilike("%{0}%".format(f))),
                 )
             )
 
     for c in q.all():
         table.append([c.id, c.name, c.owner, c.issuer])
 
-    print(tabulate(table, headers=['Id', 'Name', 'Owner', 'Issuer'], tablefmt='csv'))
+    print(tabulate(table, headers=["Id", "Name", "Owner", "Issuer"], tablefmt="csv"))
 
 
 def worker(data, commit, reason):
-    parts = [x for x in data.split(' ') if x]
+    parts = [x for x in data.split(" ") if x]
     try:
         cert = get(int(parts[0].strip()))
         plugin = plugins.get(cert.authority.plugin_name)
 
-        print('[+] Revoking certificate. Id: {0} Name: {1}'.format(cert.id, cert.name))
+        print("[+] Revoking certificate. Id: {0} Name: {1}".format(cert.id, cert.name))
         if commit:
             plugin.revoke_certificate(cert, reason)
 
-        metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+        metrics.send(
+            "certificate_revoke",
+            "counter",
+            1,
+            metric_tags={"status": SUCCESS_METRIC_STATUS},
+        )
 
     except Exception as e:
         sentry.captureException()
-        metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-        print(
-            "[!] Failed to revoke certificates. Reason: {}".format(
-                e
-            )
+        metrics.send(
+            "certificate_revoke",
+            "counter",
+            1,
+            metric_tags={"status": FAILURE_METRIC_STATUS},
         )
+        print("[!] Failed to revoke certificates. Reason: {}".format(e))
 
 
 @manager.command
@@ -325,13 +426,22 @@ def clear_pending():
     Function clears all pending certificates.
     :return:
     """
-    v = plugins.get('verisign-issuer')
+    v = plugins.get("verisign-issuer")
     v.clear_pending_certificates()
 
 
-@manager.option('-p', '--path', dest='path', help='Absolute file path to a Lemur query csv.')
-@manager.option('-r', '--reason', dest='reason', help='Reason to revoke certificate.')
-@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+@manager.option(
+    "-p", "--path", dest="path", help="Absolute file path to a Lemur query csv."
+)
+@manager.option("-r", "--reason", dest="reason", help="Reason to revoke certificate.")
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
 def revoke(path, reason, commit):
     """
     Revokes given certificate.
@@ -341,7 +451,7 @@ def revoke(path, reason, commit):
 
     print("[+] Starting certificate revocation.")
 
-    with open(path, 'r') as f:
+    with open(path, "r") as f:
         args = [[x, commit, reason] for x in f.readlines()[2:]]
 
     with multiprocessing.Pool(processes=3) as pool:
@@ -364,11 +474,11 @@ def check_revoked():
             else:
                 status = verify_string(cert.body, "")
 
-            cert.status = 'valid' if status else 'revoked'
+            cert.status = "valid" if status else "revoked"
 
         except Exception as e:
             sentry.captureException()
             current_app.logger.exception(e)
-            cert.status = 'unknown'
+            cert.status = "unknown"
 
         database.update(cert)

lemur/certificates/hooks.py

@@ -12,21 +12,30 @@ import subprocess
 
 from flask import current_app
 
-from lemur.certificates.service import csr_created, csr_imported, certificate_issued, certificate_imported
+from lemur.certificates.service import (
+    csr_created,
+    csr_imported,
+    certificate_issued,
+    certificate_imported,
+)
 
 
 def csr_dump_handler(sender, csr, **kwargs):
     try:
-        subprocess.run(['openssl', 'req', '-text', '-noout', '-reqopt', 'no_sigdump,no_pubkey'],
-                       input=csr.encode('utf8'))
+        subprocess.run(
+            ["openssl", "req", "-text", "-noout", "-reqopt", "no_sigdump,no_pubkey"],
+            input=csr.encode("utf8"),
+        )
     except Exception as err:
         current_app.logger.warning("Error inspecting CSR: %s", err)
 
 
 def cert_dump_handler(sender, certificate, **kwargs):
     try:
-        subprocess.run(['openssl', 'x509', '-text', '-noout', '-certopt', 'no_sigdump,no_pubkey'],
-                       input=certificate.body.encode('utf8'))
+        subprocess.run(
+            ["openssl", "x509", "-text", "-noout", "-certopt", "no_sigdump,no_pubkey"],
+            input=certificate.body.encode("utf8"),
+        )
     except Exception as err:
         current_app.logger.warning("Error inspecting certificate: %s", err)
 

lemur/certificates/models.py

@@ -12,32 +12,49 @@ from cryptography import x509
 from cryptography.hazmat.primitives.asymmetric import rsa
 from flask import current_app
 from idna.core import InvalidCodepoint
-from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean, Index
+from sqlalchemy import (
+    event,
+    Integer,
+    ForeignKey,
+    String,
+    PassiveDefault,
+    func,
+    Column,
+    Text,
+    Boolean,
+    Index,
+)
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import case, extract
 from sqlalchemy_utils.types.arrow import ArrowType
 from werkzeug.utils import cached_property
 
-from lemur.common import defaults, utils
+from lemur.common import defaults, utils, validators
 from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
 from lemur.database import db
 from lemur.domains.models import Domain
 from lemur.extensions import metrics
 from lemur.extensions import sentry
-from lemur.models import certificate_associations, certificate_source_associations, \
-    certificate_destination_associations, certificate_notification_associations, \
-    certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations
+from lemur.models import (
+    certificate_associations,
+    certificate_source_associations,
+    certificate_destination_associations,
+    certificate_notification_associations,
+    certificate_replacement_associations,
+    roles_certificates,
+    pending_cert_replacement_associations,
+)
 from lemur.plugins.base import plugins
 from lemur.policies.models import RotationPolicy
 from lemur.utils import Vault
 
 
 def get_sequence(name):
-    if '-' not in name:
+    if "-" not in name:
         return name, None
 
-    parts = name.split('-')
+    parts = name.split("-")
 
     # see if we have an int at the end of our name
     try:
@@ -49,22 +66,26 @@ def get_sequence(name):
     if len(parts[-1]) == 8:
         return name, None
 
-    root = '-'.join(parts[:-1])
+    root = "-".join(parts[:-1])
     return root, seq
 
 
 def get_or_increase_name(name, serial):
-    certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).all()
+    certificates = Certificate.query.filter(Certificate.name == name).all()
 
     if not certificates:
         return name
 
-    serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper())
-    certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(serial_name))).all()
+    serial_name = "{0}-{1}".format(name, hex(int(serial))[2:].upper())
+    certificates = Certificate.query.filter(Certificate.name == serial_name).all()
 
     if not certificates:
         return serial_name
 
+    certificates = Certificate.query.filter(
+        Certificate.name.ilike("{0}%".format(serial_name))
+    ).all()
+
     ends = [0]
     root, end = get_sequence(serial_name)
     for cert in certificates:
@@ -72,21 +93,29 @@ def get_or_increase_name(name, serial):
         if end:
             ends.append(end)
 
-    return '{0}-{1}'.format(root, max(ends) + 1)
+    return "{0}-{1}".format(root, max(ends) + 1)
 
 
 class Certificate(db.Model):
-    __tablename__ = 'certificates'
+    __tablename__ = "certificates"
     __table_args__ = (
-        Index('ix_certificates_cn', "cn",
+        Index(
+            "ix_certificates_cn",
+            "cn",
             postgresql_ops={"cn": "gin_trgm_ops"},
-              postgresql_using='gin'),
-        Index('ix_certificates_name', "name",
+            postgresql_using="gin",
+        ),
+        Index(
+            "ix_certificates_name",
+            "name",
             postgresql_ops={"name": "gin_trgm_ops"},
-              postgresql_using='gin'),
+            postgresql_using="gin",
+        ),
     )
     id = Column(Integer, primary_key=True)
-    ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
+    ix = Index(
+        "ix_certificates_id_desc", id.desc(), postgresql_using="btree", unique=True
+    )
     external_id = Column(String(128))
     owner = Column(String(128), nullable=False)
     name = Column(String(256), unique=True)
@@ -101,11 +130,15 @@ class Certificate(db.Model):
     issuer = Column(String(128))
     serial = Column(String(128))
     cn = Column(String(128))
-    deleted = Column(Boolean, index=True)
-    dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='CASCADE'), nullable=True)
+    deleted = Column(Boolean, index=True, default=False)
+    dns_provider_id = Column(
+        Integer(), ForeignKey("dns_providers.id", ondelete="CASCADE"), nullable=True
+    )
 
     not_before = Column(ArrowType)
     not_after = Column(ArrowType)
+    not_after_ix = Index("ix_certificates_not_after", not_after.desc())
+
     date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
 
     signing_algorithm = Column(String(128))
@@ -114,34 +147,53 @@ class Certificate(db.Model):
     san = Column(String(1024))  # TODO this should be migrated to boolean
 
     rotation = Column(Boolean, default=False)
-    user_id = Column(Integer, ForeignKey('users.id'))
-    authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
-    root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
-    rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
+    user_id = Column(Integer, ForeignKey("users.id"))
+    authority_id = Column(Integer, ForeignKey("authorities.id", ondelete="CASCADE"))
+    root_authority_id = Column(
+        Integer, ForeignKey("authorities.id", ondelete="CASCADE")
+    )
+    rotation_policy_id = Column(Integer, ForeignKey("rotation_policies.id"))
 
-    notifications = relationship('Notification', secondary=certificate_notification_associations, backref='certificate')
-    destinations = relationship('Destination', secondary=certificate_destination_associations, backref='certificate')
-    sources = relationship('Source', secondary=certificate_source_associations, backref='certificate')
-    domains = relationship('Domain', secondary=certificate_associations, backref='certificate')
-    roles = relationship('Role', secondary=roles_certificates, backref='certificate')
-    replaces = relationship('Certificate',
+    notifications = relationship(
+        "Notification",
+        secondary=certificate_notification_associations,
+        backref="certificate",
+    )
+    destinations = relationship(
+        "Destination",
+        secondary=certificate_destination_associations,
+        backref="certificate",
+    )
+    sources = relationship(
+        "Source", secondary=certificate_source_associations, backref="certificate"
+    )
+    domains = relationship(
+        "Domain", secondary=certificate_associations, backref="certificate"
+    )
+    roles = relationship("Role", secondary=roles_certificates, backref="certificate")
+    replaces = relationship(
+        "Certificate",
         secondary=certificate_replacement_associations,
         primaryjoin=id == certificate_replacement_associations.c.certificate_id,  # noqa
-                            secondaryjoin=id == certificate_replacement_associations.c.replaced_certificate_id,  # noqa
-                            backref='replaced')
+        secondaryjoin=id
+        == certificate_replacement_associations.c.replaced_certificate_id,  # noqa
+        backref="replaced",
+    )
 
-    replaced_by_pending = relationship('PendingCertificate',
+    replaced_by_pending = relationship(
+        "PendingCertificate",
         secondary=pending_cert_replacement_associations,
-                                       backref='pending_replace',
-                                       viewonly=True)
+        backref="pending_replace",
+        viewonly=True,
+    )
 
-    logs = relationship('Log', backref='certificate')
-    endpoints = relationship('Endpoint', backref='certificate')
+    logs = relationship("Log", backref="certificate")
+    endpoints = relationship("Endpoint", backref="certificate")
     rotation_policy = relationship("RotationPolicy")
-    sensitive_fields = ('private_key',)
+    sensitive_fields = ("private_key",)
 
     def __init__(self, **kwargs):
-        self.body = kwargs['body'].strip()
+        self.body = kwargs["body"].strip()
         cert = self.parsed_cert
 
         self.issuer = defaults.issuer(cert)
@@ -152,40 +204,65 @@ class Certificate(db.Model):
         self.serial = defaults.serial(cert)
 
         # when destinations are appended they require a valid name.
-        if kwargs.get('name'):
-            self.name = get_or_increase_name(defaults.text_to_slug(kwargs['name']), self.serial)
+        if kwargs.get("name"):
+            self.name = get_or_increase_name(
+                defaults.text_to_slug(kwargs["name"]), self.serial
+            )
         else:
             self.name = get_or_increase_name(
-                defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san), self.serial)
+                defaults.certificate_name(
+                    self.cn, self.issuer, self.not_before, self.not_after, self.san
+                ),
+                self.serial,
+            )
 
-        self.owner = kwargs['owner']
+        self.owner = kwargs["owner"]
 
-        if kwargs.get('private_key'):
-            self.private_key = kwargs['private_key'].strip()
+        if kwargs.get("private_key"):
+            self.private_key = kwargs["private_key"].strip()
 
-        if kwargs.get('chain'):
-            self.chain = kwargs['chain'].strip()
+        if kwargs.get("chain"):
+            self.chain = kwargs["chain"].strip()
 
-        if kwargs.get('csr'):
-            self.csr = kwargs['csr'].strip()
+        if kwargs.get("csr"):
+            self.csr = kwargs["csr"].strip()
 
-        self.notify = kwargs.get('notify', True)
-        self.destinations = kwargs.get('destinations', [])
-        self.notifications = kwargs.get('notifications', [])
-        self.description = kwargs.get('description')
-        self.roles = list(set(kwargs.get('roles', [])))
-        self.replaces = kwargs.get('replaces', [])
-        self.rotation = kwargs.get('rotation')
-        self.rotation_policy = kwargs.get('rotation_policy')
+        self.notify = kwargs.get("notify", True)
+        self.destinations = kwargs.get("destinations", [])
+        self.notifications = kwargs.get("notifications", [])
+        self.description = kwargs.get("description")
+        self.roles = list(set(kwargs.get("roles", [])))
+        self.replaces = kwargs.get("replaces", [])
+        self.rotation = kwargs.get("rotation")
+        self.rotation_policy = kwargs.get("rotation_policy")
         self.signing_algorithm = defaults.signing_algorithm(cert)
         self.bits = defaults.bitstrength(cert)
-        self.external_id = kwargs.get('external_id')
-        self.authority_id = kwargs.get('authority_id')
-        self.dns_provider_id = kwargs.get('dns_provider_id')
+        self.external_id = kwargs.get("external_id")
+        self.authority_id = kwargs.get("authority_id")
+        self.dns_provider_id = kwargs.get("dns_provider_id")
 
         for domain in defaults.domains(cert):
             self.domains.append(Domain(name=domain))
 
+        # Check integrity before saving anything into the database.
+        # For user-facing API calls, validation should also be done in schema validators.
+        self.check_integrity()
+
+    def check_integrity(self):
+        """
+        Integrity checks: Does the cert have a valid chain and matching private key?
+        """
+        if self.private_key:
+            validators.verify_private_key_match(
+                utils.parse_private_key(self.private_key),
+                self.parsed_cert,
+                error_class=AssertionError,
+            )
+
+        if self.chain:
+            chain = [self.parsed_cert] + utils.parse_cert_chain(self.chain)
+            validators.verify_cert_chain(chain, error_class=AssertionError)
+
     @cached_property
     def parsed_cert(self):
         assert self.body, "Certificate body not set"
@@ -215,10 +292,16 @@ class Certificate(db.Model):
     def location(self):
         return defaults.location(self.parsed_cert)
 
+    @property
+    def distinguished_name(self):
+        return self.parsed_cert.subject.rfc4514_string()
+
     @property
     def key_type(self):
         if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey):
-            return 'RSA{key_size}'.format(key_size=self.parsed_cert.public_key().key_size)
+            return "RSA{key_size}".format(
+                key_size=self.parsed_cert.public_key().key_size
+            )
 
     @property
     def validity_remaining(self):
@@ -243,26 +326,24 @@ class Certificate(db.Model):
 
     @expired.expression
     def expired(cls):
-        return case(
-            [
-                (cls.not_after <= arrow.utcnow(), True)
-            ],
-            else_=False
-        )
+        return case([(cls.not_after <= arrow.utcnow(), True)], else_=False)
 
     @hybrid_property
     def revoked(self):
-        if 'revoked' == self.status:
+        if "revoked" == self.status:
             return True
 
     @revoked.expression
     def revoked(cls):
-        return case(
-            [
-                (cls.status == 'revoked', True)
-            ],
-            else_=False
-        )
+        return case([(cls.status == "revoked", True)], else_=False)
+
+    @hybrid_property
+    def has_private_key(self):
+        return self.private_key is not None
+
+    @has_private_key.expression
+    def has_private_key(cls):
+        return case([(cls.private_key.is_(None), True)], else_=False)
 
     @hybrid_property
     def in_rotation_window(self):
@@ -285,66 +366,65 @@ class Certificate(db.Model):
         :return:
         """
         return case(
-            [
-                (extract('day', cls.not_after - func.now()) <= RotationPolicy.days, True)
-            ],
-            else_=False
+            [(extract("day", cls.not_after - func.now()) <= RotationPolicy.days, True)],
+            else_=False,
         )
 
     @property
     def extensions(self):
         # setup default values
-        return_extensions = {
-            'sub_alt_names': {'names': []}
-        }
+        return_extensions = {"sub_alt_names": {"names": []}}
 
         try:
             for extension in self.parsed_cert.extensions:
                 value = extension.value
                 if isinstance(value, x509.BasicConstraints):
-                    return_extensions['basic_constraints'] = value
+                    return_extensions["basic_constraints"] = value
 
                 elif isinstance(value, x509.SubjectAlternativeName):
-                    return_extensions['sub_alt_names']['names'] = value
+                    return_extensions["sub_alt_names"]["names"] = value
 
                 elif isinstance(value, x509.ExtendedKeyUsage):
-                    return_extensions['extended_key_usage'] = value
+                    return_extensions["extended_key_usage"] = value
 
                 elif isinstance(value, x509.KeyUsage):
-                    return_extensions['key_usage'] = value
+                    return_extensions["key_usage"] = value
 
                 elif isinstance(value, x509.SubjectKeyIdentifier):
-                    return_extensions['subject_key_identifier'] = {'include_ski': True}
+                    return_extensions["subject_key_identifier"] = {"include_ski": True}
 
                 elif isinstance(value, x509.AuthorityInformationAccess):
-                    return_extensions['certificate_info_access'] = {'include_aia': True}
+                    return_extensions["certificate_info_access"] = {"include_aia": True}
 
                 elif isinstance(value, x509.AuthorityKeyIdentifier):
-                    aki = {
-                        'use_key_identifier': False,
-                        'use_authority_cert': False
-                    }
+                    aki = {"use_key_identifier": False, "use_authority_cert": False}
 
                     if value.key_identifier:
-                        aki['use_key_identifier'] = True
+                        aki["use_key_identifier"] = True
 
                     if value.authority_cert_issuer:
-                        aki['use_authority_cert'] = True
+                        aki["use_authority_cert"] = True
 
-                    return_extensions['authority_key_identifier'] = aki
+                    return_extensions["authority_key_identifier"] = aki
 
                 elif isinstance(value, x509.CRLDistributionPoints):
-                    return_extensions['crl_distribution_points'] = {'include_crl_dp': value}
+                    return_extensions["crl_distribution_points"] = {
+                        "include_crl_dp": value
+                    }
 
                 # TODO: Not supporting custom OIDs yet. https://github.com/Netflix/lemur/issues/665
                 else:
-                    current_app.logger.warning('Custom OIDs not yet supported for clone operation.')
+                    current_app.logger.warning(
+                        "Custom OIDs not yet supported for clone operation."
+                    )
         except InvalidCodepoint as e:
             sentry.captureException()
-            current_app.logger.warning('Unable to parse extensions due to underscore in dns name')
+            current_app.logger.warning(
+                "Unable to parse extensions due to underscore in dns name"
+            )
         except ValueError as e:
             sentry.captureException()
-            current_app.logger.warning('Unable to parse')
+            current_app.logger.warning("Unable to parse")
             current_app.logger.exception(e)
 
         return return_extensions
@@ -353,7 +433,7 @@ class Certificate(db.Model):
         return "Certificate(name={name})".format(name=self.name)
 
 
-@event.listens_for(Certificate.destinations, 'append')
+@event.listens_for(Certificate.destinations, "append")
 def update_destinations(target, value, initiator):
     """
     Attempt to upload certificate to the new destination
@@ -367,17 +447,31 @@ def update_destinations(target, value, initiator):
     status = FAILURE_METRIC_STATUS
     try:
         if target.private_key or not destination_plugin.requires_key:
-            destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
+            destination_plugin.upload(
+                target.name,
+                target.body,
+                target.private_key,
+                target.chain,
+                value.options,
+            )
             status = SUCCESS_METRIC_STATUS
     except Exception as e:
         sentry.captureException()
         raise
 
-    metrics.send('destination_upload', 'counter', 1,
-                 metric_tags={'status': status, 'certificate': target.name, 'destination': value.label})
+    metrics.send(
+        "destination_upload",
+        "counter",
+        1,
+        metric_tags={
+            "status": status,
+            "certificate": target.name,
+            "destination": value.label,
+        },
+    )
 
 
-@event.listens_for(Certificate.replaces, 'append')
+@event.listens_for(Certificate.replaces, "append")
 def update_replacement(target, value, initiator):
     """
     When a certificate is marked as 'replaced' we should not notify.

lemur/certificates/schemas.py

@@ -6,11 +6,14 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import current_app
+from flask_restful import inputs
+from flask_restful.reqparse import RequestParser
 from marshmallow import fields, validate, validates_schema, post_load, pre_load
 from marshmallow.exceptions import ValidationError
 
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
-from lemur.common import validators, missing
+from lemur.certificates import utils as cert_utils
+from lemur.common import missing, utils, validators
 from lemur.common.fields import ArrowDateTime, Hex
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.constants import CERTIFICATE_KEY_TYPES
@@ -38,22 +41,26 @@ from lemur.users.schemas import UserNestedOutputSchema
 
 class CertificateSchema(LemurInputSchema):
     owner = fields.Email(required=True)
-    description = fields.String(missing='', allow_none=True)
+    description = fields.String(missing="", allow_none=True)
 
 
 class CertificateCreationSchema(CertificateSchema):
     @post_load
     def default_notification(self, data):
-        if not data['notifications']:
-            data['notifications'] += notification_service.create_default_expiration_notifications(
-                "DEFAULT_{0}".format(data['owner'].split('@')[0].upper()),
-                [data['owner']],
+        if not data["notifications"]:
+            data[
+                "notifications"
+            ] += notification_service.create_default_expiration_notifications(
+                "DEFAULT_{0}".format(data["owner"].split("@")[0].upper()),
+                [data["owner"]],
             )
 
-            data['notifications'] += notification_service.create_default_expiration_notifications(
-                'DEFAULT_SECURITY',
-                current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'),
-                current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL_INTERVALS', None)
+            data[
+                "notifications"
+            ] += notification_service.create_default_expiration_notifications(
+                "DEFAULT_SECURITY",
+                current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL"),
+                current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL_INTERVALS", None),
             )
         return data
 
@@ -70,34 +77,56 @@ class CertificateInputSchema(CertificateCreationSchema):
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
     replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
-    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
+    replacements = fields.Nested(
+        AssociatedCertificateSchema, missing=[], many=True
+    )  # deprecated
     roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
-    dns_provider = fields.Nested(AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False)
+    dns_provider = fields.Nested(
+        AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False
+    )
 
     csr = fields.String(allow_none=True, validate=validators.csr)
 
     key_type = fields.String(
-        validate=validate.OneOf(CERTIFICATE_KEY_TYPES),
-        missing='RSA2048')
+        validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="RSA2048"
+    )
 
     notify = fields.Boolean(default=True)
     rotation = fields.Boolean()
-    rotation_policy = fields.Nested(AssociatedRotationPolicySchema, missing={'name': 'default'}, allow_none=True,
-                                    default={'name': 'default'})
+    rotation_policy = fields.Nested(
+        AssociatedRotationPolicySchema,
+        missing={"name": "default"},
+        allow_none=True,
+        default={"name": "default"},
+    )
 
     # certificate body fields
-    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
-    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
-    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
-    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
-    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+    organizational_unit = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATIONAL_UNIT")
+    )
+    organization = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATION")
+    )
+    location = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_LOCATION")
+    )
+    country = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_COUNTRY")
+    )
+    state = fields.String(missing=lambda: current_app.config.get("LEMUR_DEFAULT_STATE"))
 
     extensions = fields.Nested(ExtensionSchema)
 
     @validates_schema
     def validate_authority(self, data):
-        if not data['authority'].active:
-            raise ValidationError("The authority is inactive.", ['authority'])
+        if 'authority' not in data:
+            raise ValidationError("Missing Authority.")
+
+        if isinstance(data["authority"], str):
+            raise ValidationError("Authority not found.")
+
+        if not data["authority"].active:
+            raise ValidationError("The authority is inactive.", ["authority"])
 
     @validates_schema
     def validate_dates(self, data):
@@ -105,8 +134,19 @@ class CertificateInputSchema(CertificateCreationSchema):
 
     @pre_load
     def load_data(self, data):
-        if data.get('replacements'):
-            data['replaces'] = data['replacements']  # TODO remove when field is deprecated
+        if data.get("replacements"):
+            data["replaces"] = data[
+                "replacements"
+            ]  # TODO remove when field is deprecated
+        if data.get("csr"):
+            csr_sans = cert_utils.get_sans_from_csr(data["csr"])
+            if not data.get("extensions"):
+                data["extensions"] = {"subAltNames": {"names": []}}
+            elif not data["extensions"].get("subAltNames"):
+                data["extensions"]["subAltNames"] = {"names": []}
+            elif not data["extensions"]["subAltNames"].get("names"):
+                data["extensions"]["subAltNames"]["names"] = []
+            data["extensions"]["subAltNames"]["names"] += csr_sans
         return missing.convert_validity_years(data)
 
 
@@ -119,13 +159,17 @@ class CertificateEditInputSchema(CertificateSchema):
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
     replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
-    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
+    replacements = fields.Nested(
+        AssociatedCertificateSchema, missing=[], many=True
+    )  # deprecated
     roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
 
     @pre_load
     def load_data(self, data):
-        if data.get('replacements'):
-            data['replaces'] = data['replacements']  # TODO remove when field is deprecated
+        if data.get("replacements"):
+            data["replaces"] = data[
+                "replacements"
+            ]  # TODO remove when field is deprecated
         return data
 
     @post_load
@@ -136,10 +180,15 @@ class CertificateEditInputSchema(CertificateSchema):
         :param data:
         :return:
         """
-        if data['owner']:
-            notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
-            data['notifications'] += notification_service.create_default_expiration_notifications(notification_name,
-                                                                                                  [data['owner']])
+        if data["owner"]:
+            notification_name = "DEFAULT_{0}".format(
+                data["owner"].split("@")[0].upper()
+            )
+            data[
+                "notifications"
+            ] += notification_service.create_default_expiration_notifications(
+                notification_name, [data["owner"]]
+            )
         return data
 
 
@@ -165,13 +214,13 @@ class CertificateNestedOutputSchema(LemurOutputSchema):
 
     # Note aliasing is the first step in deprecating these fields.
     cn = fields.String()  # deprecated
-    common_name = fields.String(attribute='cn')
+    common_name = fields.String(attribute="cn")
 
     not_after = fields.DateTime()  # deprecated
-    validity_end = ArrowDateTime(attribute='not_after')
+    validity_end = ArrowDateTime(attribute="not_after")
 
     not_before = fields.DateTime()  # deprecated
-    validity_start = ArrowDateTime(attribute='not_before')
+    validity_start = ArrowDateTime(attribute="not_before")
 
     issuer = fields.Nested(AuthorityNestedOutputSchema)
 
@@ -202,21 +251,23 @@ class CertificateOutputSchema(LemurOutputSchema):
 
     # Note aliasing is the first step in deprecating these fields.
     notify = fields.Boolean()
-    active = fields.Boolean(attribute='notify')
+    active = fields.Boolean(attribute="notify")
+    has_private_key = fields.Boolean()
 
     cn = fields.String()
-    common_name = fields.String(attribute='cn')
+    common_name = fields.String(attribute="cn")
+    distinguished_name = fields.String()
 
     not_after = fields.DateTime()
-    validity_end = ArrowDateTime(attribute='not_after')
+    validity_end = ArrowDateTime(attribute="not_after")
 
     not_before = fields.DateTime()
-    validity_start = ArrowDateTime(attribute='not_before')
+    validity_start = ArrowDateTime(attribute="not_before")
 
     owner = fields.Email()
     san = fields.Boolean()
     serial = fields.String()
-    serial_hex = Hex(attribute='serial')
+    serial_hex = Hex(attribute="serial")
     signing_algorithm = fields.String()
 
     status = fields.String()
@@ -233,19 +284,31 @@ class CertificateOutputSchema(LemurOutputSchema):
     dns_provider = fields.Nested(DnsProvidersNestedOutputSchema)
     roles = fields.Nested(RoleNestedOutputSchema, many=True)
     endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
-    replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
+    replaced_by = fields.Nested(
+        CertificateNestedOutputSchema, many=True, attribute="replaced"
+    )
     rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
 
 
+class CertificateShortOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    name = fields.String()
+    owner = fields.Email()
+    notify = fields.Boolean()
+    authority = fields.Nested(AuthorityNestedOutputSchema)
+    issuer = fields.String()
+    cn = fields.String()
+
+
 class CertificateUploadInputSchema(CertificateCreationSchema):
     name = fields.String()
     authority = fields.Nested(AssociatedAuthoritySchema, required=False)
     notify = fields.Boolean(missing=True)
     external_id = fields.String(missing=None, allow_none=True)
-    private_key = fields.String(validate=validators.private_key)
-    body = fields.String(required=True, validate=validators.public_certificate)
-    chain = fields.String(validate=validators.public_certificate, missing=None,
-                          allow_none=True)  # TODO this could be multiple certificates
+    private_key = fields.String()
+    body = fields.String(required=True)
+    chain = fields.String(missing=None, allow_none=True)
+    csr = fields.String(required=False, allow_none=True, validate=validators.csr)
 
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
@@ -254,9 +317,44 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
 
     @validates_schema
     def keys(self, data):
-        if data.get('destinations'):
-            if not data.get('private_key'):
-                raise ValidationError('Destinations require private key.')
+        if data.get("destinations"):
+            if not data.get("private_key"):
+                raise ValidationError("Destinations require private key.")
+
+    @validates_schema
+    def validate_cert_private_key_chain(self, data):
+        cert = None
+        key = None
+        if data.get("body"):
+            try:
+                cert = utils.parse_certificate(data["body"])
+            except ValueError:
+                raise ValidationError(
+                    "Public certificate presented is not valid.", field_names=["body"]
+                )
+
+        if data.get("private_key"):
+            try:
+                key = utils.parse_private_key(data["private_key"])
+            except ValueError:
+                raise ValidationError(
+                    "Private key presented is not valid.", field_names=["private_key"]
+                )
+
+        if cert and key:
+            # Throws ValidationError
+            validators.verify_private_key_match(key, cert)
+
+        if data.get("chain"):
+            try:
+                chain = utils.parse_cert_chain(data["chain"])
+            except ValueError:
+                raise ValidationError(
+                    "Invalid certificate in certificate chain.", field_names=["chain"]
+                )
+
+            # Throws ValidationError
+            validators.verify_cert_chain([cert] + chain)
 
 
 class CertificateExportInputSchema(LemurInputSchema):
@@ -269,8 +367,10 @@ class CertificateNotificationOutputSchema(LemurOutputSchema):
     name = fields.String()
     owner = fields.Email()
     user = fields.Nested(UserNestedOutputSchema)
-    validity_end = ArrowDateTime(attribute='not_after')
-    replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
+    validity_end = ArrowDateTime(attribute="not_after")
+    replaced_by = fields.Nested(
+        CertificateNestedOutputSchema, many=True, attribute="replaced"
+    )
     endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
 
 
@@ -278,9 +378,22 @@ class CertificateRevokeSchema(LemurInputSchema):
     comments = fields.String()
 
 
+certificates_list_request_parser = RequestParser()
+certificates_list_request_parser.add_argument("short", type=inputs.boolean, default=False, location="args")
+
+
+def certificates_list_output_schema_factory():
+    args = certificates_list_request_parser.parse_args()
+    if args["short"]:
+        return certificates_short_output_schema
+    else:
+        return certificates_output_schema
+
+
 certificate_input_schema = CertificateInputSchema()
 certificate_output_schema = CertificateOutputSchema()
 certificates_output_schema = CertificateOutputSchema(many=True)
+certificates_short_output_schema = CertificateShortOutputSchema(many=True)
 certificate_upload_input_schema = CertificateUploadInputSchema()
 certificate_export_input_schema = CertificateExportInputSchema()
 certificate_edit_input_schema = CertificateEditInputSchema()

lemur/certificates/service.py

@@ -20,17 +20,20 @@ from lemur.common.utils import generate_private_key, truthiness
 from lemur.destinations.models import Destination
 from lemur.domains.models import Domain
 from lemur.extensions import metrics, sentry, signals
-from lemur.models import certificate_associations
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
 from lemur.plugins.base import plugins
 from lemur.roles import service as role_service
 from lemur.roles.models import Role
 
-csr_created = signals.signal('csr_created', "CSR generated")
-csr_imported = signals.signal('csr_imported', "CSR imported from external source")
-certificate_issued = signals.signal('certificate_issued', "Authority issued a certificate")
-certificate_imported = signals.signal('certificate_imported', "Certificate imported from external source")
+csr_created = signals.signal("csr_created", "CSR generated")
+csr_imported = signals.signal("csr_imported", "CSR imported from external source")
+certificate_issued = signals.signal(
+    "certificate_issued", "Authority issued a certificate"
+)
+certificate_imported = signals.signal(
+    "certificate_imported", "Certificate imported from external source"
+)
 
 
 def get(cert_id):
@@ -50,12 +53,12 @@ def get_by_name(name):
     :param name:
     :return:
     """
-    return database.get(Certificate, name, field='name')
+    return database.get(Certificate, name, field="name")
 
 
 def get_by_serial(serial):
     """
-    Retrieves certificate by it's Serial.
+    Retrieves certificate(s) by serial number.
     :param serial:
     :return:
     """
@@ -65,6 +68,22 @@ def get_by_serial(serial):
     return Certificate.query.filter(Certificate.serial == serial).all()
 
 
+def get_by_attributes(conditions):
+    """
+    Retrieves certificate(s) by conditions given in a hash of given key=>value pairs.
+    :param serial:
+    :return:
+    """
+    # Ensure that each of the given conditions corresponds to actual columns
+    # if not, silently remove it
+    for attr in conditions.keys():
+        if attr not in Certificate.__table__.columns:
+            conditions.pop(attr)
+
+    query = database.session_query(Certificate)
+    return database.find_all(query, Certificate, conditions).all()
+
+
 def delete(cert_id):
     """
     Delete's a certificate.
@@ -90,8 +109,12 @@ def get_all_pending_cleaning(source):
     :param source:
     :return:
     """
-    return Certificate.query.filter(Certificate.sources.any(id=source.id)) \
-        .filter(not_(Certificate.endpoints.any())).filter(Certificate.expired).all()
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.expired)
+        .all()
+    )
 
 
 def get_all_pending_reissue():
@@ -104,9 +127,12 @@ def get_all_pending_reissue():
 
     :return:
     """
-    return Certificate.query.filter(Certificate.rotation == True) \
-        .filter(not_(Certificate.replaced.any())) \
-        .filter(Certificate.in_rotation_window == True).all()  # noqa
+    return (
+        Certificate.query.filter(Certificate.rotation == True)
+        .filter(not_(Certificate.replaced.any()))
+        .filter(Certificate.in_rotation_window == True)
+        .all()
+    )  # noqa
 
 
 def find_duplicates(cert):
@@ -118,10 +144,12 @@ def find_duplicates(cert):
     :param cert:
     :return:
     """
-    if cert['chain']:
-        return Certificate.query.filter_by(body=cert['body'].strip(), chain=cert['chain'].strip()).all()
+    if cert["chain"]:
+        return Certificate.query.filter_by(
+            body=cert["body"].strip(), chain=cert["chain"].strip()
+        ).all()
     else:
-        return Certificate.query.filter_by(body=cert['body'].strip(), chain=None).all()
+        return Certificate.query.filter_by(body=cert["body"].strip(), chain=None).all()
 
 
 def export(cert, export_plugin):
@@ -133,8 +161,10 @@ def export(cert, export_plugin):
     :param cert:
     :return:
     """
-    plugin = plugins.get(export_plugin['slug'])
-    return plugin.export(cert.body, cert.chain, cert.private_key, export_plugin['pluginOptions'])
+    plugin = plugins.get(export_plugin["slug"])
+    return plugin.export(
+        cert.body, cert.chain, cert.private_key, export_plugin["pluginOptions"]
+    )
 
 
 def update(cert_id, **kwargs):
@@ -153,17 +183,19 @@ def update(cert_id, **kwargs):
 
 def create_certificate_roles(**kwargs):
     # create an role for the owner and assign it
-    owner_role = role_service.get_by_name(kwargs['owner'])
+    owner_role = role_service.get_by_name(kwargs["owner"])
 
     if not owner_role:
         owner_role = role_service.create(
-            kwargs['owner'],
-            description="Auto generated role based on owner: {0}".format(kwargs['owner'])
+            kwargs["owner"],
+            description="Auto generated role based on owner: {0}".format(
+                kwargs["owner"]
+            ),
         )
 
     # ensure that the authority's owner is also associated with the certificate
-    if kwargs.get('authority'):
-        authority_owner_role = role_service.get_by_name(kwargs['authority'].owner)
+    if kwargs.get("authority"):
+        authority_owner_role = role_service.get_by_name(kwargs["authority"].owner)
         return [owner_role, authority_owner_role]
 
     return [owner_role]
@@ -175,16 +207,16 @@ def mint(**kwargs):
     Support for multiple authorities is handled by individual plugins.
 
     """
-    authority = kwargs['authority']
+    authority = kwargs["authority"]
 
     issuer = plugins.get(authority.plugin_name)
 
     # allow the CSR to be specified by the user
-    if not kwargs.get('csr'):
+    if not kwargs.get("csr"):
         csr, private_key = create_csr(**kwargs)
         csr_created.send(authority=authority, csr=csr)
     else:
-        csr = str(kwargs.get('csr'))
+        csr = str(kwargs.get("csr"))
         private_key = None
         csr_imported.send(authority=authority, csr=csr)
 
@@ -205,8 +237,8 @@ def import_certificate(**kwargs):
 
     :param kwargs:
     """
-    if not kwargs.get('owner'):
-        kwargs['owner'] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0]
+    if not kwargs.get("owner"):
+        kwargs["owner"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")[0]
 
     return upload(**kwargs)
 
@@ -217,21 +249,16 @@ def upload(**kwargs):
     """
     roles = create_certificate_roles(**kwargs)
 
-    if kwargs.get('roles'):
-        kwargs['roles'] += roles
+    if kwargs.get("roles"):
+        kwargs["roles"] += roles
     else:
-        kwargs['roles'] = roles
-
-    if kwargs.get('private_key'):
-        private_key = kwargs['private_key']
-        if not isinstance(private_key, bytes):
-            kwargs['private_key'] = private_key.encode('utf-8')
+        kwargs["roles"] = roles
 
     cert = Certificate(**kwargs)
-    cert.authority = kwargs.get('authority')
+    cert.authority = kwargs.get("authority")
     cert = database.create(cert)
 
-    kwargs['creator'].certificates.append(cert)
+    kwargs["creator"].certificates.append(cert)
 
     cert = database.update(cert)
     certificate_imported.send(certificate=cert, authority=cert.authority)
@@ -248,39 +275,45 @@ def create(**kwargs):
         current_app.logger.error("Exception minting certificate", exc_info=True)
         sentry.captureException()
         raise
-    kwargs['body'] = cert_body
-    kwargs['private_key'] = private_key
-    kwargs['chain'] = cert_chain
-    kwargs['external_id'] = external_id
-    kwargs['csr'] = csr
+    kwargs["body"] = cert_body
+    kwargs["private_key"] = private_key
+    kwargs["chain"] = cert_chain
+    kwargs["external_id"] = external_id
+    kwargs["csr"] = csr
 
     roles = create_certificate_roles(**kwargs)
 
-    if kwargs.get('roles'):
-        kwargs['roles'] += roles
+    if kwargs.get("roles"):
+        kwargs["roles"] += roles
     else:
-        kwargs['roles'] = roles
+        kwargs["roles"] = roles
 
     if cert_body:
         cert = Certificate(**kwargs)
-        kwargs['creator'].certificates.append(cert)
+        kwargs["creator"].certificates.append(cert)
     else:
         cert = PendingCertificate(**kwargs)
-        kwargs['creator'].pending_certificates.append(cert)
+        kwargs["creator"].pending_certificates.append(cert)
 
-    cert.authority = kwargs['authority']
+    cert.authority = kwargs["authority"]
 
     database.commit()
 
     if isinstance(cert, Certificate):
         certificate_issued.send(certificate=cert, authority=cert.authority)
-        metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
+        metrics.send(
+            "certificate_issued",
+            "counter",
+            1,
+            metric_tags=dict(owner=cert.owner, issuer=cert.issuer),
+        )
 
     if isinstance(cert, PendingCertificate):
         # We need to refresh the pending certificate to avoid "Instance is not bound to a Session; "
         # "attribute refresh operation cannot proceed"
         pending_cert = database.session_query(PendingCertificate).get(cert.id)
         from lemur.common.celery import fetch_acme_cert
+
         if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False):
             fetch_acme_cert.apply_async((pending_cert.id,), countdown=5)
 
@@ -296,85 +329,150 @@ def render(args):
     """
     query = database.session_query(Certificate)
 
-    time_range = args.pop('time_range')
-    destination_id = args.pop('destination_id')
-    notification_id = args.pop('notification_id', None)
-    show = args.pop('show')
+    show_expired = args.pop("showExpired")
+    if show_expired != 1:
+        one_month_old = arrow.now()\
+            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\
+            .format("YYYY-MM-DD")
+        query = query.filter(Certificate.not_after > one_month_old)
+
+    time_range = args.pop("time_range")
+
+    destination_id = args.pop("destination_id")
+    notification_id = args.pop("notification_id", None)
+    show = args.pop("show")
     # owner = args.pop('owner')
     # creator = args.pop('creator')  # TODO we should enabling filtering by owner
 
-    filt = args.pop('filter')
+    filt = args.pop("filter")
 
     if filt:
-        terms = filt.split(';')
-        term = '%{0}%'.format(terms[1])
+        terms = filt.split(";")
+        term = "%{0}%".format(terms[1])
         # Exact matches for quotes. Only applies to name, issuer, and cn
         if terms[1].startswith('"') and terms[1].endswith('"'):
             term = terms[1][1:-1]
 
-        if 'issuer' in terms:
+        if "issuer" in terms:
             # we can't rely on issuer being correct in the cert directly so we combine queries
-            sub_query = database.session_query(Authority.id) \
-                .filter(Authority.name.ilike(term)) \
+            sub_query = (
+                database.session_query(Authority.id)
+                .filter(Authority.name.ilike(term))
                 .subquery()
+            )
 
             query = query.filter(
                 or_(
                     Certificate.issuer.ilike(term),
-                    Certificate.authority_id.in_(sub_query)
+                    Certificate.authority_id.in_(sub_query),
                 )
             )
 
-        elif 'destination' in terms:
-            query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
-        elif 'notify' in filt:
+        elif "destination" in terms:
+            query = query.filter(
+                Certificate.destinations.any(Destination.id == terms[1])
+            )
+        elif "notify" in filt:
             query = query.filter(Certificate.notify == truthiness(terms[1]))
-        elif 'active' in filt:
+        elif "active" in filt:
             query = query.filter(Certificate.active == truthiness(terms[1]))
-        elif 'cn' in terms:
+        elif "cn" in terms:
             query = query.filter(
                 or_(
                     Certificate.cn.ilike(term),
-                    Certificate.domains.any(Domain.name.ilike(term))
+                    Certificate.domains.any(Domain.name.ilike(term)),
                 )
             )
-        elif 'id' in terms:
+        elif "id" in terms:
             query = query.filter(Certificate.id == cast(terms[1], Integer))
-        elif 'name' in terms:
-            query = query.outerjoin(certificate_associations).outerjoin(Domain).filter(
+        elif "name" in terms:
+            query = query.filter(
                 or_(
                     Certificate.name.ilike(term),
-                    Domain.name.ilike(term),
+                    Certificate.domains.any(Domain.name.ilike(term)),
                     Certificate.cn.ilike(term),
                 )
-            ).group_by(Certificate.id)
+            )
+        elif "fixedName" in terms:
+            # only what matches the fixed name directly if a fixedname is provided
+            query = query.filter(Certificate.name == terms[1])
         else:
             query = database.filter(query, Certificate, terms)
 
     if show:
-        sub_query = database.session_query(Role.name).filter(Role.user_id == args['user'].id).subquery()
+        sub_query = (
+            database.session_query(Role.name)
+            .filter(Role.user_id == args["user"].id)
+            .subquery()
+        )
         query = query.filter(
             or_(
-                Certificate.user_id == args['user'].id,
-                Certificate.owner.in_(sub_query)
+                Certificate.user_id == args["user"].id, Certificate.owner.in_(sub_query)
             )
         )
 
     if destination_id:
-        query = query.filter(Certificate.destinations.any(Destination.id == destination_id))
+        query = query.filter(
+            Certificate.destinations.any(Destination.id == destination_id)
+        )
 
     if notification_id:
-        query = query.filter(Certificate.notifications.any(Notification.id == notification_id))
+        query = query.filter(
+            Certificate.notifications.any(Notification.id == notification_id)
+        )
 
     if time_range:
-        to = arrow.now().replace(weeks=+time_range).format('YYYY-MM-DD')
-        now = arrow.now().format('YYYY-MM-DD')
-        query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
+        to = arrow.now().shift(weeks=+time_range).format("YYYY-MM-DD")
+        now = arrow.now().format("YYYY-MM-DD")
+        query = query.filter(Certificate.not_after <= to).filter(
+            Certificate.not_after >= now
+        )
+
+    if current_app.config.get("ALLOW_CERT_DELETION", False):
+        query = query.filter(Certificate.deleted == False)  # noqa
 
     result = database.sort_and_page(query, Certificate, args)
     return result
 
 
+def query_name(certificate_name, args):
+    """
+    Helper function that queries for a certificate by name
+
+    :param args:
+    :return:
+    """
+    query = database.session_query(Certificate)
+    query = query.filter(Certificate.name == certificate_name)
+    result = database.sort_and_page(query, Certificate, args)
+    return result
+
+
+def query_common_name(common_name, args):
+    """
+    Helper function that queries for not expired certificates by common name (and owner)
+
+    :param common_name:
+    :param args:
+    :return:
+    """
+    owner = args.pop("owner")
+    if not owner:
+        owner = "%"
+
+    # only not expired certificates
+    current_time = arrow.utcnow()
+
+    result = (
+        Certificate.query.filter(Certificate.cn.ilike(common_name))
+        .filter(Certificate.owner.ilike(owner))
+        .filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))
+        .all()
+    )
+
+    return result
+
+
 def create_csr(**csr_config):
     """
     Given a list of domains create the appropriate csr
@@ -382,65 +480,77 @@ def create_csr(**csr_config):
 
     :param csr_config:
     """
-    private_key = generate_private_key(csr_config.get('key_type'))
+    private_key = generate_private_key(csr_config.get("key_type"))
 
     builder = x509.CertificateSigningRequestBuilder()
-    name_list = [x509.NameAttribute(x509.OID_COMMON_NAME, csr_config['common_name'])]
-    if current_app.config.get('LEMUR_OWNER_EMAIL_IN_SUBJECT', True):
-        name_list.append(x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config['owner']))
-    if 'organization' in csr_config and csr_config['organization'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config['organization']))
-    if 'organizational_unit' in csr_config and csr_config['organizational_unit'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config['organizational_unit']))
-    if 'country' in csr_config and csr_config['country'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config['country']))
-    if 'state' in csr_config and csr_config['state'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config['state']))
-    if 'location' in csr_config and csr_config['location'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config['location']))
+    name_list = [x509.NameAttribute(x509.OID_COMMON_NAME, csr_config["common_name"])]
+    if current_app.config.get("LEMUR_OWNER_EMAIL_IN_SUBJECT", True):
+        name_list.append(
+            x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config["owner"])
+        )
+    if "organization" in csr_config and csr_config["organization"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config["organization"])
+        )
+    if (
+        "organizational_unit" in csr_config
+        and csr_config["organizational_unit"].strip()
+    ):
+        name_list.append(
+            x509.NameAttribute(
+                x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config["organizational_unit"]
+            )
+        )
+    if "country" in csr_config and csr_config["country"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config["country"])
+        )
+    if "state" in csr_config and csr_config["state"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config["state"])
+        )
+    if "location" in csr_config and csr_config["location"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config["location"])
+        )
     builder = builder.subject_name(x509.Name(name_list))
 
-    extensions = csr_config.get('extensions', {})
-    critical_extensions = ['basic_constraints', 'sub_alt_names', 'key_usage']
-    noncritical_extensions = ['extended_key_usage']
+    extensions = csr_config.get("extensions", {})
+    critical_extensions = ["basic_constraints", "sub_alt_names", "key_usage"]
+    noncritical_extensions = ["extended_key_usage"]
     for k, v in extensions.items():
         if v:
             if k in critical_extensions:
-                current_app.logger.debug('Adding Critical Extension: {0} {1}'.format(k, v))
-                if k == 'sub_alt_names':
-                    if v['names']:
-                        builder = builder.add_extension(v['names'], critical=True)
+                current_app.logger.debug(
+                    "Adding Critical Extension: {0} {1}".format(k, v)
+                )
+                if k == "sub_alt_names":
+                    if v["names"]:
+                        builder = builder.add_extension(v["names"], critical=True)
                 else:
                     builder = builder.add_extension(v, critical=True)
 
             if k in noncritical_extensions:
-                current_app.logger.debug('Adding Extension: {0} {1}'.format(k, v))
+                current_app.logger.debug("Adding Extension: {0} {1}".format(k, v))
                 builder = builder.add_extension(v, critical=False)
 
-    ski = extensions.get('subject_key_identifier', {})
-    if ski.get('include_ski', False):
+    ski = extensions.get("subject_key_identifier", {})
+    if ski.get("include_ski", False):
         builder = builder.add_extension(
             x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
-            critical=False
+            critical=False,
         )
 
-    request = builder.sign(
-        private_key, hashes.SHA256(), default_backend()
-    )
+    request = builder.sign(private_key, hashes.SHA256(), default_backend())
 
     # serialize our private key and CSR
     private_key = private_key.private_bytes(
         encoding=serialization.Encoding.PEM,
         format=serialization.PrivateFormat.TraditionalOpenSSL,  # would like to use PKCS8 but AWS ELBs don't like it
-        encryption_algorithm=serialization.NoEncryption()
-    )
+        encryption_algorithm=serialization.NoEncryption(),
+    ).decode("utf-8")
 
-    if isinstance(private_key, bytes):
-        private_key = private_key.decode('utf-8')
-
-    csr = request.public_bytes(
-        encoding=serialization.Encoding.PEM
-    ).decode('utf-8')
+    csr = request.public_bytes(encoding=serialization.Encoding.PEM).decode("utf-8")
 
     return csr, private_key
 
@@ -452,16 +562,19 @@ def stats(**kwargs):
     :param kwargs:
     :return:
     """
-    if kwargs.get('metric') == 'not_after':
+    if kwargs.get("metric") == "not_after":
         start = arrow.utcnow()
-        end = start.replace(weeks=+32)
-        items = database.db.session.query(Certificate.issuer, func.count(Certificate.id)) \
-            .group_by(Certificate.issuer) \
-            .filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \
-            .filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all()
+        end = start.shift(weeks=+32)
+        items = (
+            database.db.session.query(Certificate.issuer, func.count(Certificate.id))
+            .group_by(Certificate.issuer)
+            .filter(Certificate.not_after <= end.format("YYYY-MM-DD"))
+            .filter(Certificate.not_after >= start.format("YYYY-MM-DD"))
+            .all()
+        )
 
     else:
-        attr = getattr(Certificate, kwargs.get('metric'))
+        attr = getattr(Certificate, kwargs.get("metric"))
         query = database.db.session.query(attr, func.count(attr))
 
         items = query.group_by(attr).all()
@@ -472,7 +585,7 @@ def stats(**kwargs):
         keys.append(key)
         values.append(count)
 
-    return {'labels': keys, 'values': values}
+    return {"labels": keys, "values": values}
 
 
 def get_account_number(arn):
@@ -519,22 +632,24 @@ def get_certificate_primitives(certificate):
     certificate via `create`.
     """
     start, end = calculate_reissue_range(certificate.not_before, certificate.not_after)
-    ser = CertificateInputSchema().load(CertificateOutputSchema().dump(certificate).data)
+    ser = CertificateInputSchema().load(
+        CertificateOutputSchema().dump(certificate).data
+    )
     assert not ser.errors, "Error re-serializing certificate: %s" % ser.errors
     data = ser.data
 
     # we can't quite tell if we are using a custom name, as this is an automated process (typically)
     # we will rely on the Lemur generated name
-    data.pop('name', None)
+    data.pop("name", None)
 
     # TODO this can be removed once we migrate away from cn
-    data['cn'] = data['common_name']
+    data["cn"] = data["common_name"]
 
     # needed until we move off not_*
-    data['not_before'] = start
-    data['not_after'] = end
-    data['validity_start'] = start
-    data['validity_end'] = end
+    data["not_before"] = start
+    data["not_after"] = end
+    data["validity_start"] = start
+    data["validity_end"] = end
     return data
 
 
@@ -552,13 +667,13 @@ def reissue_certificate(certificate, replace=None, user=None):
         #  We do not want to re-use the CSR when creating a certificate because this defeats the purpose of rotation.
         del primitives["csr"]
     if not user:
-        primitives['creator'] = certificate.user
+        primitives["creator"] = certificate.user
 
     else:
-        primitives['creator'] = user
+        primitives["creator"] = user
 
     if replace:
-        primitives['replaces'] = [certificate]
+        primitives["replaces"] = [certificate]
 
     new_cert = create(**primitives)
 

lemur/certificates/utils.py

@@ -0,0 +1,41 @@
+"""
+Utils to parse certificate data.
+
+.. module: lemur.certificates.hooks
+    :platform: Unix
+    :copyright: (c) 2019 by Javier Ramos, see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Javier Ramos <javier.ramos@booking.com>
+"""
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from marshmallow.exceptions import ValidationError
+
+
+def get_sans_from_csr(data):
+    """
+    Fetches SubjectAlternativeNames from CSR.
+    Works with any kind of SubjectAlternativeName
+    :param data: PEM-encoded string with CSR
+    :return: List of LemurAPI-compatible subAltNames
+    """
+    sub_alt_names = []
+    try:
+        request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
+    except Exception:
+        raise ValidationError("CSR presented is not valid.")
+
+    try:
+        alt_names = request.extensions.get_extension_for_class(
+            x509.SubjectAlternativeName
+        )
+        for alt_name in alt_names.value:
+            sub_alt_names.append(
+                {"nameType": type(alt_name).__name__, "value": alt_name.value}
+            )
+    except x509.ExtensionNotFound:
+        pass
+
+    return sub_alt_names

lemur/certificates/verify.py

@@ -29,31 +29,45 @@ def ocsp_verify(cert, cert_path, issuer_chain_path):
     :param issuer_chain_path:
     :return bool: True if certificate is valid, False otherwise
     """
-    command = ['openssl', 'x509', '-noout', '-ocsp_uri', '-in', cert_path]
+    command = ["openssl", "x509", "-noout", "-ocsp_uri", "-in", cert_path]
     p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     url, err = p1.communicate()
 
     if not url:
-        current_app.logger.debug("No OCSP URL in certificate {}".format(cert.serial_number))
+        current_app.logger.debug(
+            "No OCSP URL in certificate {}".format(cert.serial_number)
+        )
         return None
 
-    p2 = subprocess.Popen(['openssl', 'ocsp', '-issuer', issuer_chain_path,
-                           '-cert', cert_path, "-url", url.strip()],
+    p2 = subprocess.Popen(
+        [
+            "openssl",
+            "ocsp",
+            "-issuer",
+            issuer_chain_path,
+            "-cert",
+            cert_path,
+            "-url",
+            url.strip(),
+        ],
         stdout=subprocess.PIPE,
-                          stderr=subprocess.PIPE)
+        stderr=subprocess.PIPE,
+    )
 
     message, err = p2.communicate()
 
-    p_message = message.decode('utf-8')
+    p_message = message.decode("utf-8")
 
-    if 'error' in p_message or 'Error' in p_message:
+    if "error" in p_message or "Error" in p_message:
         raise Exception("Got error when parsing OCSP url")
 
-    elif 'revoked' in p_message:
-        current_app.logger.debug("OCSP reports certificate revoked: {}".format(cert.serial_number))
+    elif "revoked" in p_message:
+        current_app.logger.debug(
+            "OCSP reports certificate revoked: {}".format(cert.serial_number)
+        )
         return False
 
-    elif 'good' not in p_message:
+    elif "good" not in p_message:
         raise Exception("Did not receive a valid response")
 
     return True
@@ -73,7 +87,9 @@ def crl_verify(cert, cert_path):
             x509.OID_CRL_DISTRIBUTION_POINTS
         ).value
     except x509.ExtensionNotFound:
-        current_app.logger.debug("No CRLDP extension in certificate {}".format(cert.serial_number))
+        current_app.logger.debug(
+            "No CRLDP extension in certificate {}".format(cert.serial_number)
+        )
         return None
 
     for p in distribution_points:
@@ -92,8 +108,9 @@ def crl_verify(cert, cert_path):
             except ConnectionError:
                 raise Exception("Unable to retrieve CRL: {0}".format(point))
 
-            crl_cache[point] = x509.load_der_x509_crl(response.content,
-                                                      backend=default_backend())
+            crl_cache[point] = x509.load_der_x509_crl(
+                response.content, backend=default_backend()
+            )
         else:
             current_app.logger.debug("CRL point is cached {}".format(point))
 
@@ -110,8 +127,9 @@ def crl_verify(cert, cert_path):
                 except x509.ExtensionNotFound:
                     pass
 
-                current_app.logger.debug("CRL reports certificate "
-                                         "revoked: {}".format(cert.serial_number))
+                current_app.logger.debug(
+                    "CRL reports certificate " "revoked: {}".format(cert.serial_number)
+                )
                 return False
 
     return True
@@ -125,7 +143,7 @@ def verify(cert_path, issuer_chain_path):
     :param issuer_chain_path:
     :return: True if valid, False otherwise
     """
-    with open(cert_path, 'rt') as c:
+    with open(cert_path, "rt") as c:
         try:
             cert = parse_certificate(c.read())
         except ValueError as e:
@@ -154,10 +172,10 @@ def verify_string(cert_string, issuer_string):
     :return: True if valid, False otherwise
     """
     with mktempfile() as cert_tmp:
-        with open(cert_tmp, 'w') as f:
+        with open(cert_tmp, "w") as f:
             f.write(cert_string)
         with mktempfile() as issuer_tmp:
-            with open(issuer_tmp, 'w') as f:
+            with open(issuer_tmp, "w") as f:
                 f.write(issuer_string)
             status = verify(cert_tmp, issuer_tmp)
     return status

lemur/certificates/views.py

@@ -8,7 +8,7 @@
 import base64
 from builtins import str
 
-from flask import Blueprint, make_response, jsonify, g
+from flask import Blueprint, make_response, jsonify, g, current_app
 from flask_restful import reqparse, Api, inputs
 
 from lemur.common.schema import validate_schema
@@ -26,17 +26,224 @@ from lemur.certificates.schemas import (
     certificate_upload_input_schema,
     certificates_output_schema,
     certificate_export_input_schema,
-    certificate_edit_input_schema
+    certificate_edit_input_schema,
+    certificates_list_output_schema_factory,
 )
 
 from lemur.roles import service as role_service
 from lemur.logs import service as log_service
 
 
-mod = Blueprint('certificates', __name__)
+mod = Blueprint("certificates", __name__)
 api = Api(mod)
 
 
+class CertificatesListValid(AuthenticatedResource):
+    """ Defines the 'certificates/valid' endpoint """
+
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificatesListValid, self).__init__()
+
+    @validate_schema(None, certificates_output_schema)
+    def get(self):
+        """
+        .. http:get:: /certificates/valid/<query>
+
+           The current list of not-expired certificates for a given common name, and owner
+
+           **Example request**:
+
+           .. sourcecode:: http
+              GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com
+              HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [{
+                    "status": null,
+                    "cn": "*.test.example.net",
+                    "chain": "",
+                    "csr": "-----BEGIN CERTIFICATE REQUEST-----"
+                    "authority": {
+                        "active": true,
+                        "owner": "secure@example.com",
+                        "id": 1,
+                        "description": "verisign test authority",
+                        "name": "verisign"
+                    },
+                    "owner": "joe@example.com",
+                    "serial": "82311058732025924142789179368889309156",
+                    "id": 2288,
+                    "issuer": "SymantecCorporation",
+                    "dateCreated": "2016-06-03T06:09:42.133769+00:00",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }],
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "replaced": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
+                "total": 1
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        args["user"] = g.user
+        common_name = args["filter"].split(";")[1]
+        return service.query_common_name(common_name, args)
+
+
+class CertificatesNameQuery(AuthenticatedResource):
+    """ Defines the 'certificates/name' endpoint """
+
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificatesNameQuery, self).__init__()
+
+    @validate_schema(None, certificates_output_schema)
+    def get(self, certificate_name):
+        """
+        .. http:get:: /certificates/name/<query>
+
+           The current list of certificates
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /certificates/name/WILDCARD.test.example.net-SymantecCorporation-20160603-20180112 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [{
+                    "status": null,
+                    "cn": "*.test.example.net",
+                    "chain": "",
+                    "csr": "-----BEGIN CERTIFICATE REQUEST-----"
+                    "authority": {
+                        "active": true,
+                        "owner": "secure@example.com",
+                        "id": 1,
+                        "description": "verisign test authority",
+                        "name": "verisign"
+                    },
+                    "owner": "joe@example.com",
+                    "serial": "82311058732025924142789179368889309156",
+                    "id": 2288,
+                    "issuer": "SymantecCorporation",
+                    "dateCreated": "2016-06-03T06:09:42.133769+00:00",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }],
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "replaced": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
+                "total": 1
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int. default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number. default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        parser = paginated_parser.copy()
+        parser.add_argument("timeRange", type=int, dest="time_range", location="args")
+        parser.add_argument("owner", type=inputs.boolean, location="args")
+        parser.add_argument("id", type=str, location="args")
+        parser.add_argument("active", type=inputs.boolean, location="args")
+        parser.add_argument(
+            "destinationId", type=int, dest="destination_id", location="args"
+        )
+        parser.add_argument("creator", type=str, location="args")
+        parser.add_argument("show", type=str, location="args")
+
+        args = parser.parse_args()
+        args["user"] = g.user
+        return service.query_name(certificate_name, args)
+
+
 class CertificatesList(AuthenticatedResource):
     """ Defines the 'certificates' endpoint """
 
@@ -44,7 +251,7 @@ class CertificatesList(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(CertificatesList, self).__init__()
 
-    @validate_schema(None, certificates_output_schema)
+    @validate_schema(None, certificates_list_output_schema_factory)
     def get(self):
         """
         .. http:get:: /certificates
@@ -132,16 +339,19 @@ class CertificatesList(AuthenticatedResource):
 
         """
         parser = paginated_parser.copy()
-        parser.add_argument('timeRange', type=int, dest='time_range', location='args')
-        parser.add_argument('owner', type=inputs.boolean, location='args')
-        parser.add_argument('id', type=str, location='args')
-        parser.add_argument('active', type=inputs.boolean, location='args')
-        parser.add_argument('destinationId', type=int, dest="destination_id", location='args')
-        parser.add_argument('creator', type=str, location='args')
-        parser.add_argument('show', type=str, location='args')
+        parser.add_argument("timeRange", type=int, dest="time_range", location="args")
+        parser.add_argument("owner", type=inputs.boolean, location="args")
+        parser.add_argument("id", type=str, location="args")
+        parser.add_argument("active", type=inputs.boolean, location="args")
+        parser.add_argument(
+            "destinationId", type=int, dest="destination_id", location="args"
+        )
+        parser.add_argument("creator", type=str, location="args")
+        parser.add_argument("show", type=str, location="args")
+        parser.add_argument("showExpired", type=int, location="args")
 
         args = parser.parse_args()
-        args['user'] = g.user
+        args["user"] = g.user
         return service.render(args)
 
     @validate_schema(certificate_input_schema, certificate_output_schema)
@@ -259,24 +469,31 @@ class CertificatesList(AuthenticatedResource):
            :statuscode 403: unauthenticated
 
         """
-        role = role_service.get_by_name(data['authority'].owner)
+        role = role_service.get_by_name(data["authority"].owner)
 
         # all the authority role members should be allowed
-        roles = [x.name for x in data['authority'].roles]
+        roles = [x.name for x in data["authority"].roles]
 
         # allow "owner" roles by team DL
         roles.append(role)
-        authority_permission = AuthorityPermission(data['authority'].id, roles)
+        authority_permission = AuthorityPermission(data["authority"].id, roles)
 
         if authority_permission.can():
-            data['creator'] = g.user
+            data["creator"] = g.user
             cert = service.create(**data)
             if isinstance(cert, Certificate):
                 # only log if created, not pending
-                log_service.create(g.user, 'create_cert', certificate=cert)
+                log_service.create(g.user, "create_cert", certificate=cert)
             return cert
 
-        return dict(message="You are not authorized to use the authority: {0}".format(data['authority'].name)), 403
+        return (
+            dict(
+                message="You are not authorized to use the authority: {0}".format(
+                    data["authority"].name
+                )
+            ),
+            403,
+        )
 
 
 class CertificatesUpload(AuthenticatedResource):
@@ -306,6 +523,7 @@ class CertificatesUpload(AuthenticatedResource):
                  "body": "-----BEGIN CERTIFICATE-----...",
                  "chain": "-----BEGIN CERTIFICATE-----...",
                  "privateKey": "-----BEGIN RSA PRIVATE KEY-----..."
+                 "csr": "-----BEGIN CERTIFICATE REQUEST-----..."
                  "destinations": [],
                  "notifications": [],
                  "replacements": [],
@@ -378,12 +596,14 @@ class CertificatesUpload(AuthenticatedResource):
            :statuscode 200: no error
 
         """
-        data['creator'] = g.user
-        if data.get('destinations'):
-            if data.get('private_key'):
+        data["creator"] = g.user
+        if data.get("destinations"):
+            if data.get("private_key"):
                 return service.upload(**data)
             else:
-                raise Exception("Private key must be provided in order to upload certificate to AWS")
+                raise Exception(
+                    "Private key must be provided in order to upload certificate to AWS"
+                )
         return service.upload(**data)
 
 
@@ -395,10 +615,12 @@ class CertificatesStats(AuthenticatedResource):
         super(CertificatesStats, self).__init__()
 
     def get(self):
-        self.reqparse.add_argument('metric', type=str, location='args')
-        self.reqparse.add_argument('range', default=32, type=int, location='args')
-        self.reqparse.add_argument('destinationId', dest='destination_id', location='args')
-        self.reqparse.add_argument('active', type=str, default='true', location='args')
+        self.reqparse.add_argument("metric", type=str, location="args")
+        self.reqparse.add_argument("range", default=32, type=int, location="args")
+        self.reqparse.add_argument(
+            "destinationId", dest="destination_id", location="args"
+        )
+        self.reqparse.add_argument("active", type=str, default="true", location="args")
 
         args = self.reqparse.parse_args()
 
@@ -450,12 +672,12 @@ class CertificatePrivateKey(AuthenticatedResource):
             permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
 
             if not permission.can():
-                return dict(message='You are not authorized to view this key'), 403
+                return dict(message="You are not authorized to view this key"), 403
 
-        log_service.create(g.current_user, 'key_view', certificate=cert)
+        log_service.create(g.current_user, "key_view", certificate=cert)
         response = make_response(jsonify(key=cert.private_key), 200)
-        response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store'
-        response.headers['pragma'] = 'no-cache'
+        response.headers["cache-control"] = "private, max-age=0, no-cache, no-store"
+        response.headers["pragma"] = "no-cache"
         return response
 
 
@@ -645,21 +867,79 @@ class Certificates(AuthenticatedResource):
             permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
 
             if not permission.can():
-                return dict(message='You are not authorized to update this certificate'), 403
+                return (
+                    dict(message="You are not authorized to update this certificate"),
+                    403,
+                )
 
-        for destination in data['destinations']:
+        for destination in data["destinations"]:
             if destination.plugin.requires_key:
                 if not cert.private_key:
-                    return dict(
-                        message='Unable to add destination: {0}. Certificate does not have required private key.'.format(
+                    return (
+                        dict(
+                            message="Unable to add destination: {0}. Certificate does not have required private key.".format(
                                 destination.label
                             )
-                    ), 400
+                        ),
+                        400,
+                    )
 
         cert = service.update(certificate_id, **data)
-        log_service.create(g.current_user, 'update_cert', certificate=cert)
+        log_service.create(g.current_user, "update_cert", certificate=cert)
         return cert
 
+    def delete(self, certificate_id, data=None):
+        """
+        .. http:delete:: /certificates/1
+
+           Delete a certificate
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              DELETE /certificates/1 HTTP/1.1
+              Host: example.com
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 204 OK
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 204: no error
+           :statuscode 403: unauthenticated
+           :statuscode 404: certificate not found
+           :statuscode 405: certificate deletion is disabled
+
+        """
+        if not current_app.config.get("ALLOW_CERT_DELETION", False):
+            return dict(message="Certificate deletion is disabled"), 405
+
+        cert = service.get(certificate_id)
+
+        if not cert:
+            return dict(message="Cannot find specified certificate"), 404
+
+        if cert.deleted:
+            return dict(message="Certificate is already deleted"), 412
+
+        # allow creators
+        if g.current_user != cert.user:
+            owner_role = role_service.get_by_name(cert.owner)
+            permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+
+            if not permission.can():
+                return (
+                    dict(message="You are not authorized to delete this certificate"),
+                    403,
+                )
+
+        service.update(certificate_id, deleted=True)
+        log_service.create(g.current_user, "delete_cert", certificate=cert)
+        return "Certificate deleted", 204
+
 
 class NotificationCertificatesList(AuthenticatedResource):
     """ Defines the 'certificates' endpoint """
@@ -758,17 +1038,19 @@ class NotificationCertificatesList(AuthenticatedResource):
 
         """
         parser = paginated_parser.copy()
-        parser.add_argument('timeRange', type=int, dest='time_range', location='args')
-        parser.add_argument('owner', type=inputs.boolean, location='args')
-        parser.add_argument('id', type=str, location='args')
-        parser.add_argument('active', type=inputs.boolean, location='args')
-        parser.add_argument('destinationId', type=int, dest="destination_id", location='args')
-        parser.add_argument('creator', type=str, location='args')
-        parser.add_argument('show', type=str, location='args')
+        parser.add_argument("timeRange", type=int, dest="time_range", location="args")
+        parser.add_argument("owner", type=inputs.boolean, location="args")
+        parser.add_argument("id", type=str, location="args")
+        parser.add_argument("active", type=inputs.boolean, location="args")
+        parser.add_argument(
+            "destinationId", type=int, dest="destination_id", location="args"
+        )
+        parser.add_argument("creator", type=str, location="args")
+        parser.add_argument("show", type=str, location="args")
 
         args = parser.parse_args()
-        args['notification_id'] = notification_id
-        args['user'] = g.current_user
+        args["notification_id"] = notification_id
+        args["user"] = g.current_user
         return service.render(args)
 
 
@@ -941,30 +1223,48 @@ class CertificateExport(AuthenticatedResource):
         if not cert:
             return dict(message="Cannot find specified certificate"), 404
 
-        plugin = data['plugin']['plugin_object']
+        plugin = data["plugin"]["plugin_object"]
 
         if plugin.requires_key:
             if not cert.private_key:
-                return dict(
-                    message='Unable to export certificate, plugin: {0} requires a private key but no key was found.'.format(
-                        plugin.slug)), 400
+                return (
+                    dict(
+                        message="Unable to export certificate, plugin: {0} requires a private key but no key was found.".format(
+                            plugin.slug
+                        )
+                    ),
+                    400,
+                )
 
             else:
                 # allow creators
                 if g.current_user != cert.user:
                     owner_role = role_service.get_by_name(cert.owner)
-                    permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+                    permission = CertificatePermission(
+                        owner_role, [x.name for x in cert.roles]
+                    )
 
                     if not permission.can():
-                        return dict(message='You are not authorized to export this certificate.'), 403
+                        return (
+                            dict(
+                                message="You are not authorized to export this certificate."
+                            ),
+                            403,
+                        )
 
-        options = data['plugin']['plugin_options']
+        options = data["plugin"]["plugin_options"]
 
-        log_service.create(g.current_user, 'key_view', certificate=cert)
-        extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
+        log_service.create(g.current_user, "key_view", certificate=cert)
+        extension, passphrase, data = plugin.export(
+            cert.body, cert.chain, cert.private_key, options
+        )
 
         # we take a hit in message size when b64 encoding
-        return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data).decode('utf-8'))
+        return dict(
+            extension=extension,
+            passphrase=passphrase,
+            data=base64.b64encode(data).decode("utf-8"),
+        )
 
 
 class CertificateRevoke(AuthenticatedResource):
@@ -1015,28 +1315,66 @@ class CertificateRevoke(AuthenticatedResource):
             permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
 
             if not permission.can():
-                return dict(message='You are not authorized to revoke this certificate.'), 403
+                return (
+                    dict(message="You are not authorized to revoke this certificate."),
+                    403,
+                )
 
         if not cert.external_id:
-            return dict(message='Cannot revoke certificate. No external id found.'), 400
+            return dict(message="Cannot revoke certificate. No external id found."), 400
 
         if cert.endpoints:
-            return dict(message='Cannot revoke certificate. Endpoints are deployed with the given certificate.'), 403
+            return (
+                dict(
+                    message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
+                ),
+                403,
+            )
 
         plugin = plugins.get(cert.authority.plugin_name)
         plugin.revoke_certificate(cert, data)
-        log_service.create(g.current_user, 'revoke_cert', certificate=cert)
+        log_service.create(g.current_user, "revoke_cert", certificate=cert)
         return dict(id=cert.id)
 
 
-api.add_resource(CertificateRevoke, '/certificates/<int:certificate_id>/revoke', endpoint='revokeCertificate')
-api.add_resource(CertificatesList, '/certificates', endpoint='certificates')
-api.add_resource(Certificates, '/certificates/<int:certificate_id>', endpoint='certificate')
-api.add_resource(CertificatesStats, '/certificates/stats', endpoint='certificateStats')
-api.add_resource(CertificatesUpload, '/certificates/upload', endpoint='certificateUpload')
-api.add_resource(CertificatePrivateKey, '/certificates/<int:certificate_id>/key', endpoint='privateKeyCertificates')
-api.add_resource(CertificateExport, '/certificates/<int:certificate_id>/export', endpoint='exportCertificate')
-api.add_resource(NotificationCertificatesList, '/notifications/<int:notification_id>/certificates',
-                 endpoint='notificationCertificates')
-api.add_resource(CertificatesReplacementsList, '/certificates/<int:certificate_id>/replacements',
-                 endpoint='replacements')
+api.add_resource(
+    CertificateRevoke,
+    "/certificates/<int:certificate_id>/revoke",
+    endpoint="revokeCertificate",
+)
+api.add_resource(
+    CertificatesNameQuery,
+    "/certificates/name/<string:certificate_name>",
+    endpoint="certificatesNameQuery",
+)
+api.add_resource(CertificatesList, "/certificates", endpoint="certificates")
+api.add_resource(
+    CertificatesListValid, "/certificates/valid", endpoint="certificatesListValid"
+)
+api.add_resource(
+    Certificates, "/certificates/<int:certificate_id>", endpoint="certificate"
+)
+api.add_resource(CertificatesStats, "/certificates/stats", endpoint="certificateStats")
+api.add_resource(
+    CertificatesUpload, "/certificates/upload", endpoint="certificateUpload"
+)
+api.add_resource(
+    CertificatePrivateKey,
+    "/certificates/<int:certificate_id>/key",
+    endpoint="privateKeyCertificates",
+)
+api.add_resource(
+    CertificateExport,
+    "/certificates/<int:certificate_id>/export",
+    endpoint="exportCertificate",
+)
+api.add_resource(
+    NotificationCertificatesList,
+    "/notifications/<int:notification_id>/certificates",
+    endpoint="notificationCertificates",
+)
+api.add_resource(
+    CertificatesReplacementsList,
+    "/certificates/<int:certificate_id>/replacements",
+    endpoint="replacements",
+)

lemur/common/celery.py

@@ -9,27 +9,43 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
 """
 import copy
 import sys
+import time
 from datetime import datetime, timezone, timedelta
 
 from celery import Celery
+from celery.exceptions import SoftTimeLimitExceeded
 from flask import current_app
 
 from lemur.authorities.service import get as get_authority
+from lemur.common.redis import RedisHandler
+from lemur.destinations import service as destinations_service
+from lemur.extensions import metrics, sentry
 from lemur.factory import create_app
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.sources.cli import clean, sync, validate_sources
+from lemur.sources.service import add_aws_destination_to_sources
+from lemur.certificates import cli as cli_certificate
+from lemur.dns_providers import cli as cli_dns_providers
+from lemur.notifications import cli as cli_notification
+from lemur.endpoints import cli as cli_endpoints
+
 
 if current_app:
     flask_app = current_app
 else:
     flask_app = create_app()
 
+red = RedisHandler().redis()
+
 
 def make_celery(app):
-    celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'),
-                    broker=app.config.get('CELERY_BROKER_URL'))
+    celery = Celery(
+        app.import_name,
+        backend=app.config.get("CELERY_RESULT_BACKEND"),
+        broker=app.config.get("CELERY_BROKER_URL"),
+    )
     celery.conf.update(app.config)
     TaskBase = celery.Task
 
@@ -47,7 +63,63 @@ def make_celery(app):
 celery = make_celery(flask_app)
 
 
+def is_task_active(fun, task_id, args):
+    from celery.task.control import inspect
+
+    if not args:
+        args = '()'  # empty args
+
+    i = inspect()
+    active_tasks = i.active()
+    for _, tasks in active_tasks.items():
+        for task in tasks:
+            if task.get("id") == task_id:
+                continue
+            if task.get("name") == fun and task.get("args") == str(args):
+                return True
+    return False
+
+
 @celery.task()
+def report_celery_last_success_metrics():
+    """
+    For each celery task, this will determine the number of seconds since it has last been successful.
+
+    Celery tasks should be emitting redis stats with a deterministic key (In our case, `f"{task}.last_success"`.
+    report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful.
+    Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted
+    from this function.
+
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "recurrent task",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_time = int(time.time())
+    schedule = current_app.config.get('CELERYBEAT_SCHEDULE')
+    for _, t in schedule.items():
+        task = t.get("task")
+        last_success = int(red.get(f"{task}.last_success") or 0)
+        metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success)
+    red.set(
+        f"{function}.last_success", int(time.time())
+    )  # Alert if this metric is not seen
+    metrics.send(f"{function}.success", 'counter', 1)
+
+
+@celery.task(soft_time_limit=600)
 def fetch_acme_cert(id):
     """
     Attempt to get the full certificate for the pending certificate listed.
@@ -55,11 +127,25 @@ def fetch_acme_cert(id):
     Args:
         id: an id of a PendingCertificate
     """
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
     log_data = {
-        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
-        "message": "Resolving pending certificate {}".format(id)
+        "function": function,
+        "message": "Resolving pending certificate {}".format(id),
+        "task_id": task_id,
+        "id": id,
     }
+
     current_app.logger.debug(log_data)
+
+    if task_id and is_task_active(log_data["function"], task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     pending_certs = pending_certificate_service.get_pending_certs([id])
     new = 0
     failed = 0
@@ -69,7 +155,7 @@ def fetch_acme_cert(id):
     # We only care about certs using the acme-issuer plugin
     for cert in pending_certs:
         cert_authority = get_authority(cert.authority_id)
-        if cert_authority.plugin_name == 'acme-issuer':
+        if cert_authority.plugin_name == "acme-issuer":
             acme_certs.append(cert)
         else:
             wrong_issuer += 1
@@ -82,20 +168,22 @@ def fetch_acme_cert(id):
         # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
         pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
         if not pending_cert:
-            log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
+            log_data[
+                "message"
+            ] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
             current_app.logger.error(log_data)
             continue
         if real_cert:
             # If a real certificate was returned from issuer, then create it in Lemur and mark
             # the pending certificate as resolved
-            final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
-            pending_certificate_service.update(
-                cert.get("pending_cert").id,
-                resolved=True
+            final_cert = pending_certificate_service.create_certificate(
+                pending_cert, real_cert, pending_cert.user
             )
             pending_certificate_service.update(
-                cert.get("pending_cert").id,
-                resolved_cert_id=final_cert.id
+                cert.get("pending_cert").id, resolved_cert_id=final_cert.id
+            )
+            pending_certificate_service.update(
+                cert.get("pending_cert").id, resolved=True
             )
             # add metrics to metrics extension
             new += 1
@@ -109,17 +197,17 @@ def fetch_acme_cert(id):
 
             if pending_cert.number_attempts > 4:
                 error_log["message"] = "Deleting pending certificate"
-                send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
+                send_pending_failure_notification(
+                    pending_cert, notify_owner=pending_cert.notify
+                )
                 # Mark the pending cert as resolved
                 pending_certificate_service.update(
-                    cert.get("pending_cert").id,
-                    resolved=True
+                    cert.get("pending_cert").id, resolved=True
                 )
             else:
                 pending_certificate_service.increment_attempt(pending_cert)
                 pending_certificate_service.update(
-                    cert.get("pending_cert").id,
-                    status=str(cert.get("last_error"))
+                    cert.get("pending_cert").id, status=str(cert.get("last_error"))
                 )
                 # Add failed pending cert task back to queue
                 fetch_acme_cert.delay(id)
@@ -129,31 +217,44 @@ def fetch_acme_cert(id):
     log_data["failed"] = failed
     log_data["wrong_issuer"] = wrong_issuer
     current_app.logger.debug(log_data)
+    metrics.send(f"{function}.resolved", 'gauge', new)
+    metrics.send(f"{function}.failed", 'gauge', failed)
+    metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer)
     print(
         "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
-            new=new,
-            failed=failed,
-            wrong_issuer=wrong_issuer
+            new=new, failed=failed, wrong_issuer=wrong_issuer
         )
     )
+    red.set(f'{function}.last_success', int(time.time()))
 
 
 @celery.task()
 def fetch_all_pending_acme_certs():
     """Instantiate celery workers to resolve all pending Acme certificates"""
-    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
+
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
 
     log_data = {
-        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
-        "message": "Starting job."
+        "function": function,
+        "message": "Starting job.",
+        "task_id": task_id,
     }
 
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
         current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
 
     # We only care about certs using the acme-issuer plugin
     for cert in pending_certs:
         cert_authority = get_authority(cert.authority_id)
-        if cert_authority.plugin_name == 'acme-issuer':
+        if cert_authority.plugin_name == "acme-issuer":
             if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5):
                 log_data["message"] = "Triggering job for cert {}".format(cert.name)
                 log_data["cert_name"] = cert.name
@@ -161,23 +262,42 @@ def fetch_all_pending_acme_certs():
                 current_app.logger.debug(log_data)
                 fetch_acme_cert.delay(cert.id)
 
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
+
 
 @celery.task()
 def remove_old_acme_certs():
     """Prune old pending acme certificates from the database"""
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
     log_data = {
-        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
+        "function": function,
+        "message": "Starting job.",
+        "task_id": task_id,
     }
-    pending_certs = pending_certificate_service.get_pending_certs('all')
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    pending_certs = pending_certificate_service.get_pending_certs("all")
 
     # Delete pending certs more than a week old
     for cert in pending_certs:
         if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
-            log_data['pending_cert_id'] = cert.id
-            log_data['pending_cert_name'] = cert.name
-            log_data['message'] = "Deleting pending certificate"
+            log_data["pending_cert_id"] = cert.id
+            log_data["pending_cert_name"] = cert.name
+            log_data["message"] = "Deleting pending certificate"
             current_app.logger.debug(log_data)
-            pending_certificate_service.delete(cert.id)
+            pending_certificate_service.delete(cert)
+
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
 
 
 @celery.task()
@@ -186,13 +306,33 @@ def clean_all_sources():
     This function will clean unused certificates from sources. This is a destructive operation and should only
     be ran periodically. This function triggers one celery task per source.
     """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Creating celery task to clean source",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     sources = validate_sources("all")
     for source in sources:
-        current_app.logger.debug("Creating celery task to clean source {}".format(source.label))
+        log_data["source"] = source.label
+        current_app.logger.debug(log_data)
         clean_source.delay(source.label)
 
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
 
-@celery.task()
+
+@celery.task(soft_time_limit=600)
 def clean_source(source):
     """
     This celery task will clean the specified source. This is a destructive operation that will delete unused
@@ -201,8 +341,31 @@ def clean_source(source):
     :param source:
     :return:
     """
-    current_app.logger.debug("Cleaning source {}".format(source))
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Cleaning source",
+        "source": source,
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, (source,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
         clean([source], True)
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Clean source: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
 
 
 @celery.task()
@@ -210,13 +373,33 @@ def sync_all_sources():
     """
     This function will sync certificates from all sources. This function triggers one celery task per source.
     """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "creating celery task to sync source",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
     sources = validate_sources("all")
     for source in sources:
-        current_app.logger.debug("Creating celery task to sync source {}".format(source.label))
+        log_data["source"] = source.label
+        current_app.logger.debug(log_data)
         sync_source.delay(source.label)
 
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
 
-@celery.task()
+
+@celery.task(soft_time_limit=7200)
 def sync_source(source):
     """
     This celery task will sync the specified source.
@@ -224,5 +407,296 @@ def sync_source(source):
     :param source:
     :return:
     """
-    current_app.logger.debug("Syncing source {}".format(source))
+
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Syncing source",
+        "source": source,
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, (source,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
         sync([source])
+        metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Error syncing source: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("sync_source_timeout", "counter", 1, metric_tags={"source": source})
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    log_data["message"] = "Done syncing source"
+    current_app.logger.debug(log_data)
+    metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
+    red.set(f'{function}.last_success', int(time.time()))
+
+
+@celery.task()
+def sync_source_destination():
+    """
+    This celery task will sync destination and source, to make sure all new destinations are also present as source.
+    Some destinations do not qualify as sources, and hence should be excluded from being added as sources
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "syncing AWS destinations and sources",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    for dst in destinations_service.get_all():
+        if add_aws_destination_to_sources(dst):
+            log_data["message"] = "new source added"
+            log_data["source"] = dst.label
+            current_app.logger.debug(log_data)
+
+    log_data["message"] = "completed Syncing AWS destinations and sources"
+    current_app.logger.debug(log_data)
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
+
+
+@celery.task(soft_time_limit=3600)
+def certificate_reissue():
+    """
+    This celery task reissues certificates which are pending reissue
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "reissuing certificates",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.reissue(None, True)
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Certificate reissue: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    log_data["message"] = "reissuance completed"
+    current_app.logger.debug(log_data)
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
+
+
+@celery.task(soft_time_limit=3600)
+def certificate_rotate():
+    """
+    This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "rotating certificates",
+        "task_id": task_id,
+
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.rotate(None, None, None, None, True)
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Certificate rotate: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    log_data["message"] = "rotation completed"
+    current_app.logger.debug(log_data)
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
+
+
+@celery.task(soft_time_limit=3600)
+def endpoints_expire():
+    """
+    This celery task removes all endpoints that have not been recently updated
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "endpoints expire",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_endpoints.expire(2)  # Time in hours
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "endpoint expire: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
+
+
+@celery.task(soft_time_limit=600)
+def get_all_zones():
+    """
+    This celery syncs all zones from the available dns providers
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "refresh all zones from available DNS providers",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_dns_providers.get_all_zones()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "get all zones: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
+
+
+@celery.task(soft_time_limit=3600)
+def check_revoked():
+    """
+    This celery task attempts to check if any certs are expired
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "check if any certificates are revoked revoked",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.check_revoked()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Checking revoked: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)
+
+
+@celery.task(soft_time_limit=3600)
+def notify_expirations():
+    """
+    This celery task notifies about expiring certs
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "notify for cert expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Notify expiring Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", 'counter', 1)

lemur/common/defaults.py

@@ -3,22 +3,29 @@ import unicodedata
 
 from cryptography import x509
 from flask import current_app
+
+from lemur.common.utils import is_selfsigned
 from lemur.extensions import sentry
 from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
 
 
-def text_to_slug(value):
-    """Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters."""
+def text_to_slug(value, joiner="-"):
+    """
+    Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters.
+    A series of non-alphanumeric characters is replaced with the joiner character.
+    """
 
     # Strip all character accents: decompose Unicode characters and then drop combining chars.
-    value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c))
+    value = "".join(
+        c for c in unicodedata.normalize("NFKD", value) if not unicodedata.combining(c)
+    )
 
-    # Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash.
-    # Except, keep 'xn--' used in IDNA domain names as is.
-    value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', '-', value)
+    # Replace all remaining non-alphanumeric characters with joiner string. Multiple characters get collapsed into a
+    # single joiner. Except, keep 'xn--' used in IDNA domain names as is.
+    value = re.sub(r"[^A-Za-z0-9.]+(?<!xn--)", joiner, value)
 
     # '-' in the beginning or end of string looks ugly.
-    return value.strip('-')
+    return value.strip(joiner)
 
 
 def certificate_name(common_name, issuer, not_before, not_after, san):
@@ -43,12 +50,12 @@ def certificate_name(common_name, issuer, not_before, not_after, san):
 
     temp = t.format(
         subject=common_name,
-        issuer=issuer.replace(' ', ''),
-        not_before=not_before.strftime('%Y%m%d'),
-        not_after=not_after.strftime('%Y%m%d')
+        issuer=issuer.replace(" ", ""),
+        not_before=not_before.strftime("%Y%m%d"),
+        not_after=not_after.strftime("%Y%m%d"),
     )
 
-    temp = temp.replace('*', "WILDCARD")
+    temp = temp.replace("*", "WILDCARD")
     return text_to_slug(temp)
 
 
@@ -64,9 +71,9 @@ def common_name(cert):
     :return: Common name or None
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_COMMON_NAME
-        )[0].value.strip()
+        return cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[
+            0
+        ].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get common name! {0}".format(e))
@@ -79,9 +86,9 @@ def organization(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_ORGANIZATION_NAME
-        )[0].value.strip()
+        return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[
+            0
+        ].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get organization! {0}".format(e))
@@ -94,9 +101,9 @@ def organizational_unit(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_ORGANIZATIONAL_UNIT_NAME
-        )[0].value.strip()
+        return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)[
+            0
+        ].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get organizational unit! {0}".format(e))
@@ -109,9 +116,9 @@ def country(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_COUNTRY_NAME
-        )[0].value.strip()
+        return cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)[
+            0
+        ].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get country! {0}".format(e))
@@ -124,9 +131,9 @@ def state(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_STATE_OR_PROVINCE_NAME
-        )[0].value.strip()
+        return cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)[
+            0
+        ].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get state! {0}".format(e))
@@ -139,9 +146,9 @@ def location(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_LOCALITY_NAME
-        )[0].value.strip()
+        return cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)[
+            0
+        ].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get location! {0}".format(e))
@@ -219,29 +226,34 @@ def bitstrength(cert):
         return cert.public_key().key_size
     except AttributeError:
         sentry.captureException()
-        current_app.logger.debug('Unable to get bitstrength.')
+        current_app.logger.debug("Unable to get bitstrength.")
 
 
 def issuer(cert):
     """
-    Gets a sane issuer name from a given certificate.
+    Gets a sane issuer slug from a given certificate, stripping non-alphanumeric characters.
 
-    :param cert:
-    :return: Issuer
+    For self-signed certificates, the special value '<selfsigned>' is returned.
+    If issuer cannot be determined, '<unknown>' is returned.
+
+    :param cert: Parsed certificate object
+    :return: Issuer slug
     """
-    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
-    try:
-        # Try organization name or fall back to CN
-        issuer = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or
-                  cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME))
-        issuer = str(issuer[0].value)
-        for c in delchars:
-            issuer = issuer.replace(c, "")
-        return issuer
-    except Exception as e:
-        sentry.captureException()
-        current_app.logger.error("Unable to get issuer! {0}".format(e))
-        return "Unknown"
+    # If certificate is self-signed, we return a special value -- there really is no distinct "issuer" for it
+    if is_selfsigned(cert):
+        return "<selfsigned>"
+
+    # Try Common Name or fall back to Organization name
+    attrs = cert.issuer.get_attributes_for_oid(
+        x509.OID_COMMON_NAME
+    ) or cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
+    if not attrs:
+        current_app.logger.error(
+            "Unable to get issuer! Cert serial {:x}".format(cert.serial_number)
+        )
+        return "<unknown>"
+
+    return text_to_slug(attrs[0].value, "")
 
 
 def not_before(cert):

lemur/common/fields.py

@@ -25,6 +25,7 @@ class Hex(Field):
     """
     A hex formatted string.
     """
+
     def _serialize(self, value, attr, obj):
         if value:
             value = hex(int(value))[2:].upper()
@@ -48,25 +49,25 @@ class ArrowDateTime(Field):
     """
 
     DATEFORMAT_SERIALIZATION_FUNCS = {
-        'iso': utils.isoformat,
-        'iso8601': utils.isoformat,
-        'rfc': utils.rfcformat,
-        'rfc822': utils.rfcformat,
+        "iso": utils.isoformat,
+        "iso8601": utils.isoformat,
+        "rfc": utils.rfcformat,
+        "rfc822": utils.rfcformat,
     }
 
     DATEFORMAT_DESERIALIZATION_FUNCS = {
-        'iso': utils.from_iso,
-        'iso8601': utils.from_iso,
-        'rfc': utils.from_rfc,
-        'rfc822': utils.from_rfc,
+        "iso": utils.from_iso,
+        "iso8601": utils.from_iso,
+        "rfc": utils.from_rfc,
+        "rfc822": utils.from_rfc,
     }
 
-    DEFAULT_FORMAT = 'iso'
+    DEFAULT_FORMAT = "iso"
 
     localtime = False
     default_error_messages = {
-        'invalid': 'Not a valid datetime.',
-        'format': '"{input}" cannot be formatted as a datetime.',
+        "invalid": "Not a valid datetime.",
+        "format": '"{input}" cannot be formatted as a datetime.',
     }
 
     def __init__(self, format=None, **kwargs):
@@ -89,34 +90,36 @@ class ArrowDateTime(Field):
             try:
                 return format_func(value, localtime=self.localtime)
             except (AttributeError, ValueError) as err:
-                self.fail('format', input=value)
+                self.fail("format", input=value)
         else:
             return value.strftime(self.dateformat)
 
     def _deserialize(self, value, attr, data):
         if not value:  # Falsy values, e.g. '', None, [] are not valid
-            raise self.fail('invalid')
+            raise self.fail("invalid")
         self.dateformat = self.dateformat or self.DEFAULT_FORMAT
         func = self.DATEFORMAT_DESERIALIZATION_FUNCS.get(self.dateformat)
         if func:
             try:
                 return arrow.get(func(value))
             except (TypeError, AttributeError, ValueError):
-                raise self.fail('invalid')
+                raise self.fail("invalid")
         elif self.dateformat:
             try:
                 return dt.datetime.strptime(value, self.dateformat)
             except (TypeError, AttributeError, ValueError):
-                raise self.fail('invalid')
+                raise self.fail("invalid")
         elif utils.dateutil_available:
             try:
                 return arrow.get(utils.from_datestring(value))
             except TypeError:
-                raise self.fail('invalid')
+                raise self.fail("invalid")
         else:
-            warnings.warn('It is recommended that you install python-dateutil '
-                          'for improved datetime deserialization.')
-            raise self.fail('invalid')
+            warnings.warn(
+                "It is recommended that you install python-dateutil "
+                "for improved datetime deserialization."
+            )
+            raise self.fail("invalid")
 
 
 class KeyUsageExtension(Field):
@@ -131,73 +134,75 @@ class KeyUsageExtension(Field):
 
     def _serialize(self, value, attr, obj):
         return {
-            'useDigitalSignature': value.digital_signature,
-            'useNonRepudiation': value.content_commitment,
-            'useKeyEncipherment': value.key_encipherment,
-            'useDataEncipherment': value.data_encipherment,
-            'useKeyAgreement': value.key_agreement,
-            'useKeyCertSign': value.key_cert_sign,
-            'useCRLSign': value.crl_sign,
-            'useEncipherOnly': value._encipher_only,
-            'useDecipherOnly': value._decipher_only
+            "useDigitalSignature": value.digital_signature,
+            "useNonRepudiation": value.content_commitment,
+            "useKeyEncipherment": value.key_encipherment,
+            "useDataEncipherment": value.data_encipherment,
+            "useKeyAgreement": value.key_agreement,
+            "useKeyCertSign": value.key_cert_sign,
+            "useCRLSign": value.crl_sign,
+            "useEncipherOnly": value._encipher_only,
+            "useDecipherOnly": value._decipher_only,
         }
 
     def _deserialize(self, value, attr, data):
         keyusages = {
-            'digital_signature': False,
-            'content_commitment': False,
-            'key_encipherment': False,
-            'data_encipherment': False,
-            'key_agreement': False,
-            'key_cert_sign': False,
-            'crl_sign': False,
-            'encipher_only': False,
-            'decipher_only': False
+            "digital_signature": False,
+            "content_commitment": False,
+            "key_encipherment": False,
+            "data_encipherment": False,
+            "key_agreement": False,
+            "key_cert_sign": False,
+            "crl_sign": False,
+            "encipher_only": False,
+            "decipher_only": False,
         }
 
         for k, v in value.items():
-            if k == 'useDigitalSignature':
-                keyusages['digital_signature'] = v
+            if k == "useDigitalSignature":
+                keyusages["digital_signature"] = v
 
-            elif k == 'useNonRepudiation':
-                keyusages['content_commitment'] = v
+            elif k == "useNonRepudiation":
+                keyusages["content_commitment"] = v
 
-            elif k == 'useKeyEncipherment':
-                keyusages['key_encipherment'] = v
+            elif k == "useKeyEncipherment":
+                keyusages["key_encipherment"] = v
 
-            elif k == 'useDataEncipherment':
-                keyusages['data_encipherment'] = v
+            elif k == "useDataEncipherment":
+                keyusages["data_encipherment"] = v
 
-            elif k == 'useKeyCertSign':
-                keyusages['key_cert_sign'] = v
+            elif k == "useKeyCertSign":
+                keyusages["key_cert_sign"] = v
 
-            elif k == 'useCRLSign':
-                keyusages['crl_sign'] = v
+            elif k == "useCRLSign":
+                keyusages["crl_sign"] = v
 
-            elif k == 'useKeyAgreement':
-                keyusages['key_agreement'] = v
+            elif k == "useKeyAgreement":
+                keyusages["key_agreement"] = v
 
-            elif k == 'useEncipherOnly' and v:
-                keyusages['encipher_only'] = True
-                keyusages['key_agreement'] = True
+            elif k == "useEncipherOnly" and v:
+                keyusages["encipher_only"] = True
+                keyusages["key_agreement"] = True
 
-            elif k == 'useDecipherOnly' and v:
-                keyusages['decipher_only'] = True
-                keyusages['key_agreement'] = True
+            elif k == "useDecipherOnly" and v:
+                keyusages["decipher_only"] = True
+                keyusages["key_agreement"] = True
 
-        if keyusages['encipher_only'] and keyusages['decipher_only']:
-            raise ValidationError('A certificate cannot have both Encipher Only and Decipher Only Extended Key Usages.')
+        if keyusages["encipher_only"] and keyusages["decipher_only"]:
+            raise ValidationError(
+                "A certificate cannot have both Encipher Only and Decipher Only Extended Key Usages."
+            )
 
         return x509.KeyUsage(
-            digital_signature=keyusages['digital_signature'],
-            content_commitment=keyusages['content_commitment'],
-            key_encipherment=keyusages['key_encipherment'],
-            data_encipherment=keyusages['data_encipherment'],
-            key_agreement=keyusages['key_agreement'],
-            key_cert_sign=keyusages['key_cert_sign'],
-            crl_sign=keyusages['crl_sign'],
-            encipher_only=keyusages['encipher_only'],
-            decipher_only=keyusages['decipher_only']
+            digital_signature=keyusages["digital_signature"],
+            content_commitment=keyusages["content_commitment"],
+            key_encipherment=keyusages["key_encipherment"],
+            data_encipherment=keyusages["data_encipherment"],
+            key_agreement=keyusages["key_agreement"],
+            key_cert_sign=keyusages["key_cert_sign"],
+            crl_sign=keyusages["crl_sign"],
+            encipher_only=keyusages["encipher_only"],
+            decipher_only=keyusages["decipher_only"],
         )
 
 
@@ -216,69 +221,77 @@ class ExtendedKeyUsageExtension(Field):
         usage_list = {}
         for usage in usages:
             if usage == x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH:
-                usage_list['useClientAuthentication'] = True
+                usage_list["useClientAuthentication"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.SERVER_AUTH:
-                usage_list['useServerAuthentication'] = True
+                usage_list["useServerAuthentication"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.CODE_SIGNING:
-                usage_list['useCodeSigning'] = True
+                usage_list["useCodeSigning"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION:
-                usage_list['useEmailProtection'] = True
+                usage_list["useEmailProtection"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.TIME_STAMPING:
-                usage_list['useTimestamping'] = True
+                usage_list["useTimestamping"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING:
-                usage_list['useOCSPSigning'] = True
+                usage_list["useOCSPSigning"] = True
 
-            elif usage.dotted_string == '1.3.6.1.5.5.7.3.14':
-                usage_list['useEapOverLAN'] = True
+            elif usage.dotted_string == "1.3.6.1.5.5.7.3.14":
+                usage_list["useEapOverLAN"] = True
 
-            elif usage.dotted_string == '1.3.6.1.5.5.7.3.13':
-                usage_list['useEapOverPPP'] = True
+            elif usage.dotted_string == "1.3.6.1.5.5.7.3.13":
+                usage_list["useEapOverPPP"] = True
 
-            elif usage.dotted_string == '1.3.6.1.4.1.311.20.2.2':
-                usage_list['useSmartCardLogon'] = True
+            elif usage.dotted_string == "1.3.6.1.4.1.311.20.2.2":
+                usage_list["useSmartCardLogon"] = True
 
             else:
-                current_app.logger.warning('Unable to serialize ExtendedKeyUsage with OID: {usage}'.format(usage=usage.dotted_string))
+                current_app.logger.warning(
+                    "Unable to serialize ExtendedKeyUsage with OID: {usage}".format(
+                        usage=usage.dotted_string
+                    )
+                )
 
         return usage_list
 
     def _deserialize(self, value, attr, data):
         usage_oids = []
         for k, v in value.items():
-            if k == 'useClientAuthentication' and v:
+            if k == "useClientAuthentication" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH)
 
-            elif k == 'useServerAuthentication' and v:
+            elif k == "useServerAuthentication" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.SERVER_AUTH)
 
-            elif k == 'useCodeSigning' and v:
+            elif k == "useCodeSigning" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.CODE_SIGNING)
 
-            elif k == 'useEmailProtection' and v:
+            elif k == "useEmailProtection" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION)
 
-            elif k == 'useTimestamping' and v:
+            elif k == "useTimestamping" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.TIME_STAMPING)
 
-            elif k == 'useOCSPSigning' and v:
+            elif k == "useOCSPSigning" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING)
 
-            elif k == 'useEapOverLAN' and v:
+            elif k == "useEapOverLAN" and v:
                 usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.14"))
 
-            elif k == 'useEapOverPPP' and v:
+            elif k == "useEapOverPPP" and v:
                 usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.13"))
 
-            elif k == 'useSmartCardLogon' and v:
+            elif k == "useSmartCardLogon" and v:
                 usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.4.1.311.20.2.2"))
 
             else:
-                current_app.logger.warning('Unable to deserialize ExtendedKeyUsage with name: {key}'.format(key=k))
+                current_app.logger.warning(
+                    "Unable to deserialize ExtendedKeyUsage with name: {key}".format(
+                        key=k
+                    )
+                )
 
         return x509.ExtendedKeyUsage(usage_oids)
 
@@ -294,15 +307,17 @@ class BasicConstraintsExtension(Field):
     """
 
     def _serialize(self, value, attr, obj):
-        return {'ca': value.ca, 'path_length': value.path_length}
+        return {"ca": value.ca, "path_length": value.path_length}
 
     def _deserialize(self, value, attr, data):
-        ca = value.get('ca', False)
-        path_length = value.get('path_length', None)
+        ca = value.get("ca", False)
+        path_length = value.get("path_length", None)
 
         if ca:
             if not isinstance(path_length, (type(None), int)):
-                raise ValidationError('A CA certificate path_length (for BasicConstraints) must be None or an integer.')
+                raise ValidationError(
+                    "A CA certificate path_length (for BasicConstraints) must be None or an integer."
+                )
             return x509.BasicConstraints(ca=True, path_length=path_length)
         else:
             return x509.BasicConstraints(ca=False, path_length=None)
@@ -317,6 +332,7 @@ class SubjectAlternativeNameExtension(Field):
     :param kwargs: The same keyword arguments that :class:`Field` receives.
 
     """
+
     def _serialize(self, value, attr, obj):
         general_names = []
         name_type = None
@@ -326,53 +342,59 @@ class SubjectAlternativeNameExtension(Field):
                 value = name.value
 
                 if isinstance(name, x509.DNSName):
-                    name_type = 'DNSName'
+                    name_type = "DNSName"
 
                 elif isinstance(name, x509.IPAddress):
                     if isinstance(value, ipaddress.IPv4Network):
-                        name_type = 'IPNetwork'
+                        name_type = "IPNetwork"
                     else:
-                        name_type = 'IPAddress'
+                        name_type = "IPAddress"
 
                     value = str(value)
 
                 elif isinstance(name, x509.UniformResourceIdentifier):
-                    name_type = 'uniformResourceIdentifier'
+                    name_type = "uniformResourceIdentifier"
 
                 elif isinstance(name, x509.DirectoryName):
-                    name_type = 'directoryName'
+                    name_type = "directoryName"
 
                 elif isinstance(name, x509.RFC822Name):
-                    name_type = 'rfc822Name'
+                    name_type = "rfc822Name"
 
                 elif isinstance(name, x509.RegisteredID):
-                    name_type = 'registeredID'
+                    name_type = "registeredID"
                     value = value.dotted_string
                 else:
-                    current_app.logger.warning('Unknown SubAltName type: {name}'.format(name=name))
+                    current_app.logger.warning(
+                        "Unknown SubAltName type: {name}".format(name=name)
+                    )
                     continue
 
-                general_names.append({'nameType': name_type, 'value': value})
+                general_names.append({"nameType": name_type, "value": value})
 
         return general_names
 
     def _deserialize(self, value, attr, data):
         general_names = []
         for name in value:
-            if name['nameType'] == 'DNSName':
-                validators.sensitive_domain(name['value'])
-                general_names.append(x509.DNSName(name['value']))
+            if name["nameType"] == "DNSName":
+                validators.sensitive_domain(name["value"])
+                general_names.append(x509.DNSName(name["value"]))
 
-            elif name['nameType'] == 'IPAddress':
-                general_names.append(x509.IPAddress(ipaddress.ip_address(name['value'])))
+            elif name["nameType"] == "IPAddress":
+                general_names.append(
+                    x509.IPAddress(ipaddress.ip_address(name["value"]))
+                )
 
-            elif name['nameType'] == 'IPNetwork':
-                general_names.append(x509.IPAddress(ipaddress.ip_network(name['value'])))
+            elif name["nameType"] == "IPNetwork":
+                general_names.append(
+                    x509.IPAddress(ipaddress.ip_network(name["value"]))
+                )
 
-            elif name['nameType'] == 'uniformResourceIdentifier':
-                general_names.append(x509.UniformResourceIdentifier(name['value']))
+            elif name["nameType"] == "uniformResourceIdentifier":
+                general_names.append(x509.UniformResourceIdentifier(name["value"]))
 
-            elif name['nameType'] == 'directoryName':
+            elif name["nameType"] == "directoryName":
                 # TODO: Need to parse a string in name['value'] like:
                 # 'CN=Common Name, O=Org Name, OU=OrgUnit Name, C=US, ST=ST, L=City/emailAddress=person@example.com'
                 # or
@@ -390,26 +412,32 @@ class SubjectAlternativeNameExtension(Field):
                 # general_names.append(x509.DirectoryName(x509.Name(BLAH))))
                 pass
 
-            elif name['nameType'] == 'rfc822Name':
-                general_names.append(x509.RFC822Name(name['value']))
+            elif name["nameType"] == "rfc822Name":
+                general_names.append(x509.RFC822Name(name["value"]))
 
-            elif name['nameType'] == 'registeredID':
-                general_names.append(x509.RegisteredID(x509.ObjectIdentifier(name['value'])))
+            elif name["nameType"] == "registeredID":
+                general_names.append(
+                    x509.RegisteredID(x509.ObjectIdentifier(name["value"]))
+                )
 
-            elif name['nameType'] == 'otherName':
+            elif name["nameType"] == "otherName":
                 # This has two inputs (type and value), so it doesn't fit the mold of the rest of these GeneralName entities.
                 # general_names.append(x509.OtherName(name['type'], bytes(name['value']), 'utf-8'))
                 pass
 
-            elif name['nameType'] == 'x400Address':
+            elif name["nameType"] == "x400Address":
                 # The Python Cryptography library doesn't support x400Address types (yet?)
                 pass
 
-            elif name['nameType'] == 'EDIPartyName':
+            elif name["nameType"] == "EDIPartyName":
                 # The Python Cryptography library doesn't support EDIPartyName types (yet?)
                 pass
 
             else:
-                current_app.logger.warning('Unable to deserialize SubAltName with type: {name_type}'.format(name_type=name['nameType']))
+                current_app.logger.warning(
+                    "Unable to deserialize SubAltName with type: {name_type}".format(
+                        name_type=name["nameType"]
+                    )
+                )
 
         return x509.SubjectAlternativeName(general_names)

lemur/common/health.py

@@ -10,20 +10,20 @@ from flask import Blueprint
 from lemur.database import db
 from lemur.extensions import sentry
 
-mod = Blueprint('healthCheck', __name__)
+mod = Blueprint("healthCheck", __name__)
 
 
-@mod.route('/healthcheck')
+@mod.route("/healthcheck")
 def health():
     try:
         if healthcheck(db):
-            return 'ok'
+            return "ok"
     except Exception:
         sentry.captureException()
-        return 'db check failed'
+        return "db check failed"
 
 
 def healthcheck(db):
     with db.engine.connect() as connection:
-        connection.execute('SELECT 1;')
+        connection.execute("SELECT 1;")
     return True

lemur/common/managers.py

@@ -52,7 +52,7 @@ class InstanceManager(object):
 
         results = []
         for cls_path in class_list:
-            module_name, class_name = cls_path.rsplit('.', 1)
+            module_name, class_name = cls_path.rsplit(".", 1)
             try:
                 module = __import__(module_name, {}, {}, class_name)
                 cls = getattr(module, class_name)
@@ -62,10 +62,14 @@ class InstanceManager(object):
                     results.append(cls)
 
             except InvalidConfiguration as e:
-                current_app.logger.warning("Plugin '{0}' may not work correctly. {1}".format(class_name, e))
+                current_app.logger.warning(
+                    "Plugin '{0}' may not work correctly. {1}".format(class_name, e)
+                )
 
             except Exception as e:
-                current_app.logger.exception("Unable to import {0}. Reason: {1}".format(cls_path, e))
+                current_app.logger.exception(
+                    "Unable to import {0}. Reason: {1}".format(cls_path, e)
+                )
                 continue
 
         self.cache = results

lemur/common/missing.py

@@ -11,14 +11,15 @@ def convert_validity_years(data):
     :param data:
     :return:
     """
-    if data.get('validity_years'):
+    if data.get("validity_years"):
         now = arrow.utcnow()
-        data['validity_start'] = now.isoformat()
+        data["validity_start"] = now.isoformat()
 
-        end = now.replace(years=+int(data['validity_years']))
-        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+        end = now.shift(years=+int(data["validity_years"]))
+
+        if not current_app.config.get("LEMUR_ALLOW_WEEKEND_EXPIRATION", True):
             if is_weekend(end):
-                end = end.replace(days=-2)
+                end = end.shift(days=-2)
 
-        data['validity_end'] = end.isoformat()
+        data["validity_end"] = end.isoformat()
     return data

lemur/common/redis.py

@@ -0,0 +1,52 @@
+"""
+Helper Class for Redis
+
+"""
+import redis
+import sys
+from flask import current_app
+from lemur.extensions import sentry
+from lemur.factory import create_app
+
+if current_app:
+    flask_app = current_app
+else:
+    flask_app = create_app()
+
+
+class RedisHandler:
+    def __init__(self, host=flask_app.config.get('REDIS_HOST', 'localhost'),
+                 port=flask_app.config.get('REDIS_PORT', 6379),
+                 db=flask_app.config.get('REDIS_DB', 0)):
+        self.host = host
+        self.port = port
+        self.db = db
+
+    def redis(self, db=0):
+        # The decode_responses flag here directs the client to convert the responses from Redis into Python strings
+        # using the default encoding utf-8.  This is client specific.
+        function = f"{__name__}.{sys._getframe().f_code.co_name}"
+        try:
+            red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True)
+            red.set("test", 0)
+        except redis.ConnectionError:
+            log_data = {
+                "function": function,
+                "message": "Redis Connection error",
+                "host": self.host,
+                "port": self.port
+            }
+            current_app.logger.error(log_data)
+            sentry.captureException()
+        return red
+
+
+def redis_get(key, default=None):
+    red = RedisHandler().redis()
+    try:
+        v = red.get(key)
+    except redis.exceptions.ConnectionError:
+        v = None
+    if not v:
+        return default
+    return v

lemur/common/schema.py

@@ -22,27 +22,26 @@ class LemurSchema(Schema):
     """
     Base schema from which all grouper schema's inherit
     """
+
     __envelope__ = True
 
     def under(self, data, many=None):
         items = []
         if many:
             for i in data:
-                items.append(
-                    {underscore(key): value for key, value in i.items()}
-                )
+                items.append({underscore(key): value for key, value in i.items()})
             return items
-        return {
-            underscore(key): value
-            for key, value in data.items()
-        }
+        return {underscore(key): value for key, value in data.items()}
 
     def camel(self, data, many=None):
         items = []
         if many:
             for i in data:
                 items.append(
-                    {camelize(key, uppercase_first_letter=False): value for key, value in i.items()}
+                    {
+                        camelize(key, uppercase_first_letter=False): value
+                        for key, value in i.items()
+                    }
                 )
             return items
         return {
@@ -52,16 +51,16 @@ class LemurSchema(Schema):
 
     def wrap_with_envelope(self, data, many):
         if many:
-            if 'total' in self.context.keys():
-                return dict(total=self.context['total'], items=data)
+            if "total" in self.context.keys():
+                return dict(total=self.context["total"], items=data)
         return data
 
 
 class LemurInputSchema(LemurSchema):
     @pre_load(pass_many=True)
     def preprocess(self, data, many):
-        if isinstance(data, dict) and data.get('owner'):
-            data['owner'] = data['owner'].lower()
+        if isinstance(data, dict) and data.get("owner"):
+            data["owner"] = data["owner"].lower()
         return self.under(data, many=many)
 
 
@@ -74,17 +73,17 @@ class LemurOutputSchema(LemurSchema):
 
     def unwrap_envelope(self, data, many):
         if many:
-            if data['items']:
+            if data["items"]:
                 if isinstance(data, InstrumentedList) or isinstance(data, list):
-                    self.context['total'] = len(data)
+                    self.context["total"] = len(data)
                     return data
                 else:
-                    self.context['total'] = data['total']
+                    self.context["total"] = data["total"]
             else:
-                self.context['total'] = 0
-                data = {'items': []}
+                self.context["total"] = 0
+                data = {"items": []}
 
-            return data['items']
+            return data["items"]
 
         return data
 
@@ -110,11 +109,11 @@ def format_errors(messages):
 
 
 def wrap_errors(messages):
-    errors = dict(message='Validation Error.')
-    if messages.get('_schema'):
-        errors['reasons'] = {'Schema': {'rule': messages['_schema']}}
+    errors = dict(message="Validation Error.")
+    if messages.get("_schema"):
+        errors["reasons"] = {"Schema": {"rule": messages["_schema"]}}
     else:
-        errors['reasons'] = format_errors(messages)
+        errors["reasons"] = format_errors(messages)
     return errors
 
 
@@ -123,19 +122,19 @@ def unwrap_pagination(data, output_schema):
         return data
 
     if isinstance(data, dict):
-        if 'total' in data.keys():
-            if data.get('total') == 0:
+        if "total" in data.keys():
+            if data.get("total") == 0:
                 return data
 
-            marshaled_data = {'total': data['total']}
-            marshaled_data['items'] = output_schema.dump(data['items'], many=True).data
+            marshaled_data = {"total": data["total"]}
+            marshaled_data["items"] = output_schema.dump(data["items"], many=True).data
             return marshaled_data
 
         return output_schema.dump(data).data
 
     elif isinstance(data, list):
-        marshaled_data = {'total': len(data)}
-        marshaled_data['items'] = output_schema.dump(data, many=True).data
+        marshaled_data = {"total": len(data)}
+        marshaled_data["items"] = output_schema.dump(data, many=True).data
         return marshaled_data
     return output_schema.dump(data).data
 
@@ -155,7 +154,7 @@ def validate_schema(input_schema, output_schema):
                 if errors:
                     return wrap_errors(errors), 400
 
-                kwargs['data'] = data
+                kwargs["data"] = data
 
             try:
                 resp = f(*args, **kwargs)
@@ -170,7 +169,13 @@ def validate_schema(input_schema, output_schema):
             if not resp:
                 return dict(message="No data found"), 404
 
-            return unwrap_pagination(resp, output_schema), 200
+            if callable(output_schema):
+                output_schema_to_use = output_schema()
+            else:
+                output_schema_to_use = output_schema
+
+            return unwrap_pagination(resp, output_schema_to_use), 200
 
         return decorated_function
+
     return decorator

lemur/common/utils.py

@@ -7,12 +7,16 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import random
+import re
 import string
 
 import sqlalchemy
 from cryptography import x509
+from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric import rsa, ec
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding
+from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from flask_restful.reqparse import RequestParser
 from sqlalchemy import and_, func
 
@@ -21,21 +25,22 @@ from lemur.exceptions import InvalidConfiguration
 
 paginated_parser = RequestParser()
 
-paginated_parser.add_argument('count', type=int, default=10, location='args')
-paginated_parser.add_argument('page', type=int, default=1, location='args')
-paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
-paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
-paginated_parser.add_argument('filter', type=str, location='args')
+paginated_parser.add_argument("count", type=int, default=10, location="args")
+paginated_parser.add_argument("page", type=int, default=1, location="args")
+paginated_parser.add_argument("sortDir", type=str, dest="sort_dir", location="args")
+paginated_parser.add_argument("sortBy", type=str, dest="sort_by", location="args")
+paginated_parser.add_argument("filter", type=str, location="args")
+paginated_parser.add_argument("owner", type=str, location="args")
 
 
 def get_psuedo_random_string():
     """
     Create a random and strongish challenge.
     """
-    challenge = ''.join(random.choice(string.ascii_uppercase) for x in range(6))  # noqa
-    challenge += ''.join(random.choice("~!@#$%^&*()_+") for x in range(6))  # noqa
-    challenge += ''.join(random.choice(string.ascii_lowercase) for x in range(6))
-    challenge += ''.join(random.choice(string.digits) for x in range(6))  # noqa
+    challenge = "".join(random.choice(string.ascii_uppercase) for x in range(6))  # noqa
+    challenge += "".join(random.choice("~!@#$%^&*()_+") for x in range(6))  # noqa
+    challenge += "".join(random.choice(string.ascii_lowercase) for x in range(6))
+    challenge += "".join(random.choice(string.digits) for x in range(6))  # noqa
     return challenge
 
 
@@ -46,10 +51,46 @@ def parse_certificate(body):
     :param body:
     :return:
     """
-    if isinstance(body, str):
-        body = body.encode('utf-8')
+    assert isinstance(body, str)
 
-    return x509.load_pem_x509_certificate(body, default_backend())
+    return x509.load_pem_x509_certificate(body.encode("utf-8"), default_backend())
+
+
+def parse_private_key(private_key):
+    """
+    Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm).
+
+    Raises ValueError for an invalid string. Raises AssertionError when passed value is not str-type.
+
+    :param private_key: String containing PEM private key
+    """
+    assert isinstance(private_key, str)
+
+    return load_pem_private_key(
+        private_key.encode("utf8"), password=None, backend=default_backend()
+    )
+
+
+def split_pem(data):
+    """
+    Split a string of several PEM payloads to a list of strings.
+
+    :param data: String
+    :return: List of strings
+    """
+    return re.split("\n(?=-----BEGIN )", data)
+
+
+def parse_cert_chain(pem_chain):
+    """
+    Helper function to split and parse a series of PEM certificates.
+
+    :param pem_chain: string
+    :return: List of parsed certificates
+    """
+    if pem_chain is None:
+        return []
+    return [parse_certificate(cert) for cert in split_pem(pem_chain) if cert]
 
 
 def parse_csr(csr):
@@ -59,17 +100,17 @@ def parse_csr(csr):
     :param csr:
     :return:
     """
-    if isinstance(csr, str):
-        csr = csr.encode('utf-8')
+    assert isinstance(csr, str)
 
-    return x509.load_pem_x509_csr(csr, default_backend())
+    return x509.load_pem_x509_csr(csr.encode("utf-8"), default_backend())
 
 
 def get_authority_key(body):
     """Returns the authority key for a given certificate in hex format"""
     parsed_cert = parse_certificate(body)
     authority_key = parsed_cert.extensions.get_extension_for_class(
-        x509.AuthorityKeyIdentifier).value.key_identifier
+        x509.AuthorityKeyIdentifier
+    ).value.key_identifier
     return authority_key.hex()
 
 
@@ -89,20 +130,17 @@ def generate_private_key(key_type):
     _CURVE_TYPES = {
         "ECCPRIME192V1": ec.SECP192R1(),
         "ECCPRIME256V1": ec.SECP256R1(),
-
         "ECCSECP192R1": ec.SECP192R1(),
         "ECCSECP224R1": ec.SECP224R1(),
         "ECCSECP256R1": ec.SECP256R1(),
         "ECCSECP384R1": ec.SECP384R1(),
         "ECCSECP521R1": ec.SECP521R1(),
         "ECCSECP256K1": ec.SECP256K1(),
-
         "ECCSECT163K1": ec.SECT163K1(),
         "ECCSECT233K1": ec.SECT233K1(),
         "ECCSECT283K1": ec.SECT283K1(),
         "ECCSECT409K1": ec.SECT409K1(),
         "ECCSECT571K1": ec.SECT571K1(),
-
         "ECCSECT163R2": ec.SECT163R2(),
         "ECCSECT233R1": ec.SECT233R1(),
         "ECCSECT283R1": ec.SECT283R1(),
@@ -111,25 +149,74 @@ def generate_private_key(key_type):
     }
 
     if key_type not in CERTIFICATE_KEY_TYPES:
-        raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format(
-            key_type=key_type,
-            choices=",".join(CERTIFICATE_KEY_TYPES)
-        ))
+        raise Exception(
+            "Invalid key type: {key_type}. Supported key types: {choices}".format(
+                key_type=key_type, choices=",".join(CERTIFICATE_KEY_TYPES)
+            )
+        )
 
-    if 'RSA' in key_type:
+    if "RSA" in key_type:
         key_size = int(key_type[3:])
         return rsa.generate_private_key(
-            public_exponent=65537,
-            key_size=key_size,
-            backend=default_backend()
+            public_exponent=65537, key_size=key_size, backend=default_backend()
         )
-    elif 'ECC' in key_type:
+    elif "ECC" in key_type:
         return ec.generate_private_key(
-            curve=_CURVE_TYPES[key_type],
-            backend=default_backend()
+            curve=_CURVE_TYPES[key_type], backend=default_backend()
         )
 
 
+def check_cert_signature(cert, issuer_public_key):
+    """
+    Check a certificate's signature against an issuer public key.
+    Before EC validation, make sure we support the algorithm, otherwise raise UnsupportedAlgorithm
+    On success, returns None; on failure, raises UnsupportedAlgorithm or InvalidSignature.
+    """
+    if isinstance(issuer_public_key, rsa.RSAPublicKey):
+        # RSA requires padding, just to make life difficult for us poor developers :(
+        if cert.signature_algorithm_oid == x509.SignatureAlgorithmOID.RSASSA_PSS:
+            # In 2005, IETF devised a more secure padding scheme to replace PKCS #1 v1.5. To make sure that
+            # nobody can easily support or use it, they mandated lots of complicated parameters, unlike any
+            # other X.509 signature scheme.
+            # https://tools.ietf.org/html/rfc4056
+            raise UnsupportedAlgorithm("RSASSA-PSS not supported")
+        else:
+            padder = padding.PKCS1v15()
+        issuer_public_key.verify(
+            cert.signature,
+            cert.tbs_certificate_bytes,
+            padder,
+            cert.signature_hash_algorithm,
+        )
+    elif isinstance(issuer_public_key, ec.EllipticCurvePublicKey) and isinstance(
+        ec.ECDSA(cert.signature_hash_algorithm), ec.ECDSA
+    ):
+        issuer_public_key.verify(
+            cert.signature,
+            cert.tbs_certificate_bytes,
+            ec.ECDSA(cert.signature_hash_algorithm),
+        )
+    else:
+        raise UnsupportedAlgorithm(
+            "Unsupported Algorithm '{var}'.".format(
+                var=cert.signature_algorithm_oid._name
+            )
+        )
+
+
+def is_selfsigned(cert):
+    """
+    Returns True if the certificate is self-signed.
+    Returns False for failed verification or unsupported signing algorithm.
+    """
+    try:
+        check_cert_signature(cert, cert.public_key())
+        # If verification was successful, it's self-signed.
+        return True
+    except InvalidSignature:
+        return False
+
+
 def is_weekend(date):
     """
     Determines if a given date is on a weekend.
@@ -150,7 +237,9 @@ def validate_conf(app, required_vars):
     """
     for var in required_vars:
         if var not in app.config:
-            raise InvalidConfiguration("Required variable '{var}' is not set in Lemur's conf.".format(var=var))
+            raise InvalidConfiguration(
+                "Required variable '{var}' is not set in Lemur's conf.".format(var=var)
+            )
 
 
 # https://bitbucket.org/zzzeek/sqlalchemy/wiki/UsageRecipes/WindowedRangeQuery
@@ -169,18 +258,15 @@ def column_windows(session, column, windowsize):
     be computed.
 
     """
+
     def int_for_range(start_id, end_id):
         if end_id:
-            return and_(
-                column >= start_id,
-                column < end_id
-            )
+            return and_(column >= start_id, column < end_id)
         else:
             return column >= start_id
 
     q = session.query(
-        column,
-        func.row_number().over(order_by=column).label('rownum')
+        column, func.row_number().over(order_by=column).label("rownum")
     ).from_self(column)
 
     if windowsize > 1:
@@ -200,9 +286,7 @@ def column_windows(session, column, windowsize):
 def windowed_query(q, column, windowsize):
     """"Break a Query into windows on a given column."""
 
-    for whereclause in column_windows(
-            q.session,
-            column, windowsize):
+    for whereclause in column_windows(q.session, column, windowsize):
         for row in q.filter(whereclause).order_by(column):
             yield row
 
@@ -210,4 +294,16 @@ def windowed_query(q, column, windowsize):
 def truthiness(s):
     """If input string resembles something truthy then return True, else False."""
 
-    return s.lower() in ('true', 'yes', 'on', 't', '1')
+    return s.lower() in ("true", "yes", "on", "t", "1")
+
+
+def find_matching_certificates_by_hash(cert, matching_certs):
+    """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs),
+    determine if any of the certificate hashes match and return the matches."""
+    matching = []
+    for c in matching_certs:
+        if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(
+            hashes.SHA256()
+        ):
+            matching.append(c)
+    return matching

lemur/common/validators.py

@@ -1,45 +1,14 @@
 import re
 
 from cryptography import x509
+from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
 from cryptography.x509 import NameOID
 from flask import current_app
 from marshmallow.exceptions import ValidationError
 
 from lemur.auth.permissions import SensitiveDomainPermission
-from lemur.common.utils import parse_certificate, is_weekend
-from lemur.domains import service as domain_service
-
-
-def public_certificate(body):
-    """
-    Determines if specified string is valid public certificate.
-
-    :param body:
-    :return:
-    """
-    try:
-        parse_certificate(body)
-    except Exception as e:
-        current_app.logger.exception(e)
-        raise ValidationError('Public certificate presented is not valid.')
-
-
-def private_key(key):
-    """
-    User to validate that a given string is a RSA private key
-
-    :param key:
-    :return: :raise ValueError:
-    """
-    try:
-        if isinstance(key, bytes):
-            serialization.load_pem_private_key(key, None, backend=default_backend())
-        else:
-            serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
-    except Exception:
-        raise ValidationError('Private key presented is not valid.')
+from lemur.common.utils import check_cert_signature, is_weekend
 
 
 def common_name(value):
@@ -47,7 +16,7 @@ def common_name(value):
     # Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client
     # certificates). As a simple heuristic, we assume that human-readable names always include a space.
     # However, to avoid confusion for humans, we also don't count spaces at the beginning or end of the string.
-    if ' ' not in value.strip():
+    if " " not in value.strip():
         return sensitive_domain(value)
 
 
@@ -61,14 +30,21 @@ def sensitive_domain(domain):
         # User has permission, no need to check anything
         return
 
-    whitelist = current_app.config.get('LEMUR_WHITELISTED_DOMAINS', [])
+    whitelist = current_app.config.get("LEMUR_WHITELISTED_DOMAINS", [])
     if whitelist and not any(re.match(pattern, domain) for pattern in whitelist):
-        raise ValidationError('Domain {0} does not match whitelisted domain patterns. '
-                              'Contact an administrator to issue the certificate.'.format(domain))
+        raise ValidationError(
+            "Domain {0} does not match whitelisted domain patterns. "
+            "Contact an administrator to issue the certificate.".format(domain)
+        )
 
-    if any(d.sensitive for d in domain_service.get_by_name(domain)):
-        raise ValidationError('Domain {0} has been marked as sensitive. '
-                              'Contact an administrator to issue the certificate.'.format(domain))
+    # Avoid circular import.
+    from lemur.domains import service as domain_service
+
+    if domain_service.is_domain_sensitive(domain):
+        raise ValidationError(
+            "Domain {0} has been marked as sensitive. "
+            "Contact an administrator to issue the certificate.".format(domain)
+        )
 
 
 def encoding(oid_encoding):
@@ -77,9 +53,13 @@ def encoding(oid_encoding):
     :param oid_encoding:
     :return:
     """
-    valid_types = ['b64asn1', 'string', 'ia5string']
+    valid_types = ["b64asn1", "string", "ia5string"]
     if oid_encoding.lower() not in [o_type.lower() for o_type in valid_types]:
-        raise ValidationError('Invalid Oid Encoding: {0} choose from {1}'.format(oid_encoding, ",".join(valid_types)))
+        raise ValidationError(
+            "Invalid Oid Encoding: {0} choose from {1}".format(
+                oid_encoding, ",".join(valid_types)
+            )
+        )
 
 
 def sub_alt_type(alt_type):
@@ -88,10 +68,23 @@ def sub_alt_type(alt_type):
     :param alt_type:
     :return:
     """
-    valid_types = ['DNSName', 'IPAddress', 'uniFormResourceIdentifier', 'directoryName', 'rfc822Name', 'registrationID',
-                   'otherName', 'x400Address', 'EDIPartyName']
+    valid_types = [
+        "DNSName",
+        "IPAddress",
+        "uniFormResourceIdentifier",
+        "directoryName",
+        "rfc822Name",
+        "registrationID",
+        "otherName",
+        "x400Address",
+        "EDIPartyName",
+    ]
     if alt_type.lower() not in [a_type.lower() for a_type in valid_types]:
-        raise ValidationError('Invalid SubAltName Type: {0} choose from {1}'.format(type, ",".join(valid_types)))
+        raise ValidationError(
+            "Invalid SubAltName Type: {0} choose from {1}".format(
+                type, ",".join(valid_types)
+            )
+        )
 
 
 def csr(data):
@@ -101,16 +94,18 @@ def csr(data):
     :return:
     """
     try:
-        request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
+        request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
     except Exception:
-        raise ValidationError('CSR presented is not valid.')
+        raise ValidationError("CSR presented is not valid.")
 
     # Validate common name and SubjectAltNames
     for name in request.subject.get_attributes_for_oid(NameOID.COMMON_NAME):
         common_name(name.value)
 
     try:
-        alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
+        alt_names = request.extensions.get_extension_for_class(
+            x509.SubjectAlternativeName
+        )
 
         for name in alt_names.value.get_values_for_type(x509.DNSName):
             sensitive_domain(name)
@@ -119,25 +114,87 @@ def csr(data):
 
 
 def dates(data):
-    if not data.get('validity_start') and data.get('validity_end'):
-        raise ValidationError('If validity start is specified so must validity end.')
+    if not data.get("validity_start") and data.get("validity_end"):
+        raise ValidationError("If validity start is specified so must validity end.")
 
-    if not data.get('validity_end') and data.get('validity_start'):
-        raise ValidationError('If validity end is specified so must validity start.')
+    if not data.get("validity_end") and data.get("validity_start"):
+        raise ValidationError("If validity end is specified so must validity start.")
 
-    if data.get('validity_start') and data.get('validity_end'):
-        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
-            if is_weekend(data.get('validity_end')):
-                raise ValidationError('Validity end must not land on a weekend.')
+    if data.get("validity_start") and data.get("validity_end"):
+        if not current_app.config.get("LEMUR_ALLOW_WEEKEND_EXPIRATION", True):
+            if is_weekend(data.get("validity_end")):
+                raise ValidationError("Validity end must not land on a weekend.")
 
-        if not data['validity_start'] < data['validity_end']:
-            raise ValidationError('Validity start must be before validity end.')
+        if not data["validity_start"] < data["validity_end"]:
+            raise ValidationError("Validity start must be before validity end.")
 
-        if data.get('authority'):
-            if data.get('validity_start').date() < data['authority'].authority_certificate.not_before.date():
-                raise ValidationError('Validity start must not be before {0}'.format(data['authority'].authority_certificate.not_before))
+        if data.get("authority"):
+            if (
+                data.get("validity_start").date()
+                < data["authority"].authority_certificate.not_before.date()
+            ):
+                raise ValidationError(
+                    "Validity start must not be before {0}".format(
+                        data["authority"].authority_certificate.not_before
+                    )
+                )
 
-            if data.get('validity_end').date() > data['authority'].authority_certificate.not_after.date():
-                raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
+            if (
+                data.get("validity_end").date()
+                > data["authority"].authority_certificate.not_after.date()
+            ):
+                raise ValidationError(
+                    "Validity end must not be after {0}".format(
+                        data["authority"].authority_certificate.not_after
+                    )
+                )
 
     return data
+
+
+def verify_private_key_match(key, cert, error_class=ValidationError):
+    """
+    Checks that the supplied private key matches the certificate.
+
+    :param cert: Parsed certificate
+    :param key: Parsed private key
+    :param error_class: Exception class to raise on error
+    """
+    if key.public_key().public_numbers() != cert.public_key().public_numbers():
+        raise error_class("Private key does not match certificate.")
+
+
+def verify_cert_chain(certs, error_class=ValidationError):
+    """
+    Verifies that the certificates in the chain are correct.
+
+    We don't bother with full cert validation but just check that certs in the chain are signed by the next, to avoid
+    basic human errors -- such as pasting the wrong certificate.
+
+    :param certs: List of parsed certificates, use parse_cert_chain()
+    :param error_class: Exception class to raise on error
+    """
+    cert = certs[0]
+    for issuer in certs[1:]:
+        # Use the current cert's public key to verify the previous signature.
+        # "certificate validation is a complex problem that involves much more than just signature checks"
+        try:
+            check_cert_signature(cert, issuer.public_key())
+
+        except InvalidSignature:
+            # Avoid circular import.
+            from lemur.common import defaults
+
+            raise error_class(
+                "Incorrect chain certificate(s) provided: '%s' is not signed by '%s'"
+                % (
+                    defaults.common_name(cert) or "Unknown",
+                    defaults.common_name(issuer),
+                )
+            )
+
+        except UnsupportedAlgorithm as err:
+            current_app.logger.warning("Skipping chain validation: %s", err)
+
+        # Next loop will validate that *this issuer* cert is signed by the next chain cert.
+        cert = issuer

lemur/constants.py

@@ -7,28 +7,28 @@ SAN_NAMING_TEMPLATE = "SAN-{subject}-{issuer}-{not_before}-{not_after}"
 DEFAULT_NAMING_TEMPLATE = "{subject}-{issuer}-{not_before}-{not_after}"
 NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
 
-SUCCESS_METRIC_STATUS = 'success'
-FAILURE_METRIC_STATUS = 'failure'
+SUCCESS_METRIC_STATUS = "success"
+FAILURE_METRIC_STATUS = "failure"
 
 CERTIFICATE_KEY_TYPES = [
-    'RSA2048',
-    'RSA4096',
-    'ECCPRIME192V1',
-    'ECCPRIME256V1',
-    'ECCSECP192R1',
-    'ECCSECP224R1',
-    'ECCSECP256R1',
-    'ECCSECP384R1',
-    'ECCSECP521R1',
-    'ECCSECP256K1',
-    'ECCSECT163K1',
-    'ECCSECT233K1',
-    'ECCSECT283K1',
-    'ECCSECT409K1',
-    'ECCSECT571K1',
-    'ECCSECT163R2',
-    'ECCSECT233R1',
-    'ECCSECT283R1',
-    'ECCSECT409R1',
-    'ECCSECT571R2'
+    "RSA2048",
+    "RSA4096",
+    "ECCPRIME192V1",
+    "ECCPRIME256V1",
+    "ECCSECP192R1",
+    "ECCSECP224R1",
+    "ECCSECP256R1",
+    "ECCSECP384R1",
+    "ECCSECP521R1",
+    "ECCSECP256K1",
+    "ECCSECT163K1",
+    "ECCSECT233K1",
+    "ECCSECT283K1",
+    "ECCSECT409K1",
+    "ECCSECT571K1",
+    "ECCSECT163R2",
+    "ECCSECT233R1",
+    "ECCSECT283R1",
+    "ECCSECT409R1",
+    "ECCSECT571R2",
 ]

lemur/database.py

@@ -43,7 +43,7 @@ def session_query(model):
     :param model: sqlalchemy model
     :return: query object for model
     """
-    return model.query if hasattr(model, 'query') else db.session.query(model)
+    return model.query if hasattr(model, "query") else db.session.query(model)
 
 
 def create_query(model, kwargs):
@@ -77,7 +77,7 @@ def add(model):
 
 
 def get_model_column(model, field):
-    if field in getattr(model, 'sensitive_fields', ()):
+    if field in getattr(model, "sensitive_fields", ()):
         raise AttrNotFound(field)
     column = model.__table__.columns._data.get(field, None)
     if column is None:
@@ -100,7 +100,7 @@ def find_all(query, model, kwargs):
     kwargs = filter_none(kwargs)
     for attr, value in kwargs.items():
         if not isinstance(value, list):
-            value = value.split(',')
+            value = value.split(",")
 
         conditions.append(get_model_column(model, attr).in_(value))
 
@@ -200,7 +200,7 @@ def filter(query, model, terms):
     :return:
     """
     column = get_model_column(model, underscore(terms[0]))
-    return query.filter(column.ilike('%{}%'.format(terms[1])))
+    return query.filter(column.ilike("%{}%".format(terms[1])))
 
 
 def sort(query, model, field, direction):
@@ -214,7 +214,7 @@ def sort(query, model, field, direction):
     :param direction:
     """
     column = get_model_column(model, underscore(field))
-    return query.order_by(column.desc() if direction == 'desc' else column.asc())
+    return query.order_by(column.desc() if direction == "desc" else column.asc())
 
 
 def paginate(query, page, count):
@@ -247,10 +247,10 @@ def update_list(model, model_attr, item_model, items):
 
     for i in items:
         for item in getattr(model, model_attr):
-            if item.id == i['id']:
+            if item.id == i["id"]:
                 break
         else:
-            getattr(model, model_attr).append(get(item_model, i['id']))
+            getattr(model, model_attr).append(get(item_model, i["id"]))
 
     return model
 
@@ -276,9 +276,9 @@ def get_count(q):
     disable_group_by = False
     if len(q._entities) > 1:
         # currently support only one entity
-        raise Exception('only one entity is supported for get_count, got: %s' % q)
+        raise Exception("only one entity is supported for get_count, got: %s" % q)
     entity = q._entities[0]
-    if hasattr(entity, 'column'):
+    if hasattr(entity, "column"):
         # _ColumnEntity has column attr - on case: query(Model.column)...
         col = entity.column
         if q._group_by and q._distinct:
@@ -295,7 +295,11 @@ def get_count(q):
         count_func = func.count()
     if q._group_by and not disable_group_by:
         count_func = count_func.over(None)
-    count_q = q.options(lazyload('*')).statement.with_only_columns([count_func]).order_by(None)
+    count_q = (
+        q.options(lazyload("*"))
+        .statement.with_only_columns([count_func])
+        .order_by(None)
+    )
     if disable_group_by:
         count_q = count_q.group_by(None)
     count = q.session.execute(count_q).scalar()
@@ -311,13 +315,13 @@ def sort_and_page(query, model, args):
     :param args:
     :return:
     """
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
+    sort_by = args.pop("sort_by")
+    sort_dir = args.pop("sort_dir")
+    page = args.pop("page")
+    count = args.pop("count")
 
-    if args.get('user'):
-        user = args.pop('user')
+    if args.get("user"):
+        user = args.pop("user")
 
     query = find_all(query, model, args)
 

lemur/default.conf.py

@@ -1,6 +1,7 @@
 # This is just Python which means you can inherit and tweak settings
 
 import os
+
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
 THREADS_PER_PAGE = 8

lemur/defaults/views.py

@@ -13,12 +13,13 @@ from lemur.auth.service import AuthenticatedResource
 from lemur.defaults.schemas import default_output_schema
 
 
-mod = Blueprint('default', __name__)
+mod = Blueprint("default", __name__)
 api = Api(mod)
 
 
 class LemurDefaults(AuthenticatedResource):
     """ Defines the 'defaults' endpoint """
+
     def __init__(self):
         super(LemurDefaults)
 
@@ -59,17 +60,21 @@ class LemurDefaults(AuthenticatedResource):
            :statuscode 403: unauthenticated
         """
 
-        default_authority = get_by_name(current_app.config.get('LEMUR_DEFAULT_AUTHORITY'))
+        default_authority = get_by_name(
+            current_app.config.get("LEMUR_DEFAULT_AUTHORITY")
+        )
 
         return dict(
-            country=current_app.config.get('LEMUR_DEFAULT_COUNTRY'),
-            state=current_app.config.get('LEMUR_DEFAULT_STATE'),
-            location=current_app.config.get('LEMUR_DEFAULT_LOCATION'),
-            organization=current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'),
-            organizational_unit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'),
-            issuer_plugin=current_app.config.get('LEMUR_DEFAULT_ISSUER_PLUGIN'),
+            country=current_app.config.get("LEMUR_DEFAULT_COUNTRY"),
+            state=current_app.config.get("LEMUR_DEFAULT_STATE"),
+            location=current_app.config.get("LEMUR_DEFAULT_LOCATION"),
+            organization=current_app.config.get("LEMUR_DEFAULT_ORGANIZATION"),
+            organizational_unit=current_app.config.get(
+                "LEMUR_DEFAULT_ORGANIZATIONAL_UNIT"
+            ),
+            issuer_plugin=current_app.config.get("LEMUR_DEFAULT_ISSUER_PLUGIN"),
             authority=default_authority,
         )
 
 
-api.add_resource(LemurDefaults, '/defaults', endpoint='default')
+api.add_resource(LemurDefaults, "/defaults", endpoint="default")

lemur/destinations/models.py

@@ -13,7 +13,7 @@ from lemur.plugins.base import plugins
 
 
 class Destination(db.Model):
-    __tablename__ = 'destinations'
+    __tablename__ = "destinations"
     id = Column(Integer, primary_key=True)
     label = Column(String(32))
     options = Column(JSONType)

lemur/destinations/schemas.py

@@ -30,7 +30,7 @@ class DestinationOutputSchema(LemurOutputSchema):
     @post_dump
     def fill_object(self, data):
         if data:
-            data['plugin']['pluginOptions'] = data['options']
+            data["plugin"]["pluginOptions"] = data["options"]
         return data
 
 

lemur/destinations/service.py

@@ -6,11 +6,13 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from sqlalchemy import func
+from flask import current_app
 
 from lemur import database
 from lemur.models import certificate_destination_associations
 from lemur.destinations.models import Destination
 from lemur.certificates.models import Certificate
+from lemur.sources.service import add_aws_destination_to_sources
 
 
 def create(label, plugin_name, options, description=None):
@@ -24,10 +26,18 @@ def create(label, plugin_name, options, description=None):
     """
     # remove any sub-plugin objects before try to save the json options
     for option in options:
-        if 'plugin' in option['type']:
-            del option['value']['plugin_object']
+        if "plugin" in option["type"]:
+            del option["value"]["plugin_object"]
+
+    destination = Destination(
+        label=label, options=options, plugin_name=plugin_name, description=description
+    )
+    current_app.logger.info("Destination: %s created", label)
+
+    # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination
+    if add_aws_destination_to_sources(destination):
+        current_app.logger.info("Source: %s created", label)
 
-    destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
     return database.create(destination)
 
 
@@ -77,7 +87,7 @@ def get_by_label(label):
     :param label:
     :return:
     """
-    return database.get(Destination, label, field='label')
+    return database.get(Destination, label, field="label")
 
 
 def get_all():
@@ -91,17 +101,19 @@ def get_all():
 
 
 def render(args):
-    filt = args.pop('filter')
-    certificate_id = args.pop('certificate_id', None)
+    filt = args.pop("filter")
+    certificate_id = args.pop("certificate_id", None)
 
     if certificate_id:
-        query = database.session_query(Destination).join(Certificate, Destination.certificate)
+        query = database.session_query(Destination).join(
+            Certificate, Destination.certificate
+        )
         query = query.filter(Certificate.id == certificate_id)
     else:
         query = database.session_query(Destination)
 
     if filt:
-        terms = filt.split(';')
+        terms = filt.split(";")
         query = database.filter(query, Destination, terms)
 
     return database.sort_and_page(query, Destination, args)
@@ -114,9 +126,15 @@ def stats(**kwargs):
     :param kwargs:
     :return:
     """
-    items = database.db.session.query(Destination.label, func.count(certificate_destination_associations.c.certificate_id))\
-        .join(certificate_destination_associations)\
-        .group_by(Destination.label).all()
+    items = (
+        database.db.session.query(
+            Destination.label,
+            func.count(certificate_destination_associations.c.certificate_id),
+        )
+        .join(certificate_destination_associations)
+        .group_by(Destination.label)
+        .all()
+    )
 
     keys = []
     values = []
@@ -124,4 +142,4 @@ def stats(**kwargs):
         keys.append(key)
         values.append(count)
 
-    return {'labels': keys, 'values': values}
+    return {"labels": keys, "values": values}

lemur/destinations/views.py

@@ -15,15 +15,20 @@ from lemur.auth.permissions import admin_permission
 from lemur.common.utils import paginated_parser
 
 from lemur.common.schema import validate_schema
-from lemur.destinations.schemas import destinations_output_schema, destination_input_schema, destination_output_schema
+from lemur.destinations.schemas import (
+    destinations_output_schema,
+    destination_input_schema,
+    destination_output_schema,
+)
 
 
-mod = Blueprint('destinations', __name__)
+mod = Blueprint("destinations", __name__)
 api = Api(mod)
 
 
 class DestinationsList(AuthenticatedResource):
     """ Defines the 'destinations' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(DestinationsList, self).__init__()
@@ -176,7 +181,12 @@ class DestinationsList(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        return service.create(data['label'], data['plugin']['slug'], data['plugin']['plugin_options'], data['description'])
+        return service.create(
+            data["label"],
+            data["plugin"]["slug"],
+            data["plugin"]["plugin_options"],
+            data["description"],
+        )
 
 
 class Destinations(AuthenticatedResource):
@@ -325,16 +335,22 @@ class Destinations(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        return service.update(destination_id, data['label'], data['plugin']['plugin_options'], data['description'])
+        return service.update(
+            destination_id,
+            data["label"],
+            data["plugin"]["plugin_options"],
+            data["description"],
+        )
 
     @admin_permission.require(http_exception=403)
     def delete(self, destination_id):
         service.delete(destination_id)
-        return {'result': True}
+        return {"result": True}
 
 
 class CertificateDestinations(AuthenticatedResource):
     """ Defines the 'certificate/<int:certificate_id/destinations'' endpoint """
+
     def __init__(self):
         super(CertificateDestinations, self).__init__()
 
@@ -401,25 +417,31 @@ class CertificateDestinations(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['certificate_id'] = certificate_id
+        args["certificate_id"] = certificate_id
         return service.render(args)
 
 
 class DestinationsStats(AuthenticatedResource):
     """ Defines the 'certificates' stats endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(DestinationsStats, self).__init__()
 
     def get(self):
-        self.reqparse.add_argument('metric', type=str, location='args')
+        self.reqparse.add_argument("metric", type=str, location="args")
         args = self.reqparse.parse_args()
         items = service.stats(**args)
         return dict(items=items, total=len(items))
 
 
-api.add_resource(DestinationsList, '/destinations', endpoint='destinations')
-api.add_resource(Destinations, '/destinations/<int:destination_id>', endpoint='destination')
-api.add_resource(CertificateDestinations, '/certificates/<int:certificate_id>/destinations',
-                 endpoint='certificateDestinations')
-api.add_resource(DestinationsStats, '/destinations/stats', endpoint='destinationStats')
+api.add_resource(DestinationsList, "/destinations", endpoint="destinations")
+api.add_resource(
+    Destinations, "/destinations/<int:destination_id>", endpoint="destination"
+)
+api.add_resource(
+    CertificateDestinations,
+    "/certificates/<int:certificate_id>/destinations",
+    endpoint="certificateDestinations",
+)
+api.add_resource(DestinationsStats, "/destinations/stats", endpoint="destinationStats")

lemur/dns_providers/cli.py

@@ -5,7 +5,9 @@ from lemur.dns_providers.service import get_all_dns_providers, set_domains
 from lemur.extensions import metrics
 from lemur.plugins.base import plugins
 
-manager = Manager(usage="Iterates through all DNS providers and sets DNS zones in the database.")
+manager = Manager(
+    usage="Iterates through all DNS providers and sets DNS zones in the database."
+)
 
 
 @manager.command
@@ -27,5 +29,5 @@ def get_all_zones():
 
     status = SUCCESS_METRIC_STATUS
 
-    metrics.send('get_all_zones', 'counter', 1, metric_tags={'status': status})
+    metrics.send("get_all_zones", "counter", 1, metric_tags={"status": status})
     print("[+] Done with dns provider zone lookup and configuration.")

lemur/dns_providers/models.py

@@ -9,22 +9,23 @@ from lemur.utils import Vault
 
 
 class DnsProvider(db.Model):
-    __tablename__ = 'dns_providers'
-    id = Column(
-        Integer(),
-        primary_key=True,
-    )
+    __tablename__ = "dns_providers"
+    id = Column(Integer(), primary_key=True)
     name = Column(String(length=256), unique=True, nullable=True)
     description = Column(Text(), nullable=True)
     provider_type = Column(String(length=256), nullable=True)
     credentials = Column(Vault, nullable=True)
     api_endpoint = Column(String(length=256), nullable=True)
-    date_created = Column(ArrowType(), server_default=text('now()'), nullable=False)
+    date_created = Column(ArrowType(), server_default=text("now()"), nullable=False)
     status = Column(String(length=128), nullable=True)
     options = Column(JSON, nullable=True)
     domains = Column(JSON, nullable=True)
-    certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id',
-                                lazy='dynamic')
+    certificates = relationship(
+        "Certificate",
+        backref="dns_provider",
+        foreign_keys="Certificate.dns_provider_id",
+        lazy="dynamic",
+    )
 
     def __init__(self, name, description, provider_type, credentials):
         self.name = name

lemur/dns_providers/service.py

@@ -49,7 +49,9 @@ def get_friendly(dns_provider_id):
     }
 
     if dns_provider.provider_type == "route53":
-        dns_provider_friendly["account_id"] = json.loads(dns_provider.credentials).get("account_id")
+        dns_provider_friendly["account_id"] = json.loads(dns_provider.credentials).get(
+            "account_id"
+        )
     return dns_provider_friendly
 
 
@@ -64,40 +66,41 @@ def delete(dns_provider_id):
 
 def get_types():
     provider_config = current_app.config.get(
-        'ACME_DNS_PROVIDER_TYPES',
-        {"items": [
+        "ACME_DNS_PROVIDER_TYPES",
         {
-                'name': 'route53',
-                'requirements': [
+            "items": [
                 {
-                        'name': 'account_id',
-                        'type': 'int',
-                        'required': True,
-                        'helpMessage': 'AWS Account number'
+                    "name": "route53",
+                    "requirements": [
+                        {
+                            "name": "account_id",
+                            "type": "int",
+                            "required": True,
+                            "helpMessage": "AWS Account number",
+                        }
+                    ],
                 },
+                {
+                    "name": "cloudflare",
+                    "requirements": [
+                        {
+                            "name": "email",
+                            "type": "str",
+                            "required": True,
+                            "helpMessage": "Cloudflare Email",
+                        },
+                        {
+                            "name": "key",
+                            "type": "str",
+                            "required": True,
+                            "helpMessage": "Cloudflare Key",
+                        },
+                    ],
+                },
+                {"name": "dyn"},
+                {"name": "ultradns"},
             ]
         },
-            {
-                'name': 'cloudflare',
-                'requirements': [
-                    {
-                        'name': 'email',
-                        'type': 'str',
-                        'required': True,
-                        'helpMessage': 'Cloudflare Email'
-                    },
-                    {
-                        'name': 'key',
-                        'type': 'str',
-                        'required': True,
-                        'helpMessage': 'Cloudflare Key'
-                    },
-                ]
-            },
-            {
-                'name': 'dyn',
-            },
-        ]}
     )
     if not provider_config:
         raise Exception("No DNS Provider configuration specified.")

lemur/dns_providers/views.py

@@ -13,9 +13,12 @@ from lemur.auth.service import AuthenticatedResource
 from lemur.common.schema import validate_schema
 from lemur.common.utils import paginated_parser
 from lemur.dns_providers import service
-from lemur.dns_providers.schemas import dns_provider_output_schema, dns_provider_input_schema
+from lemur.dns_providers.schemas import (
+    dns_provider_output_schema,
+    dns_provider_input_schema,
+)
 
-mod = Blueprint('dns_providers', __name__)
+mod = Blueprint("dns_providers", __name__)
 api = Api(mod)
 
 
@@ -71,12 +74,12 @@ class DnsProvidersList(AuthenticatedResource):
 
         """
         parser = paginated_parser.copy()
-        parser.add_argument('dns_provider_id', type=int, location='args')
-        parser.add_argument('name', type=str, location='args')
-        parser.add_argument('type', type=str, location='args')
+        parser.add_argument("dns_provider_id", type=int, location="args")
+        parser.add_argument("name", type=str, location="args")
+        parser.add_argument("type", type=str, location="args")
 
         args = parser.parse_args()
-        args['user'] = g.user
+        args["user"] = g.user
         return service.render(args)
 
     @validate_schema(dns_provider_input_schema, None)
@@ -152,7 +155,7 @@ class DnsProviders(AuthenticatedResource):
     @admin_permission.require(http_exception=403)
     def delete(self, dns_provider_id):
         service.delete(dns_provider_id)
-        return {'result': True}
+        return {"result": True}
 
 
 class DnsProviderOptions(AuthenticatedResource):
@@ -166,6 +169,10 @@ class DnsProviderOptions(AuthenticatedResource):
         return service.get_types()
 
 
-api.add_resource(DnsProvidersList, '/dns_providers', endpoint='dns_providers')
-api.add_resource(DnsProviders, '/dns_providers/<int:dns_provider_id>', endpoint='dns_provider')
-api.add_resource(DnsProviderOptions, '/dns_provider_options', endpoint='dns_provider_options')
+api.add_resource(DnsProvidersList, "/dns_providers", endpoint="dns_providers")
+api.add_resource(
+    DnsProviders, "/dns_providers/<int:dns_provider_id>", endpoint="dns_provider"
+)
+api.add_resource(
+    DnsProviderOptions, "/dns_provider_options", endpoint="dns_provider_options"
+)

lemur/domains/models.py

@@ -13,11 +13,14 @@ from lemur.database import db
 
 
 class Domain(db.Model):
-    __tablename__ = 'domains'
+    __tablename__ = "domains"
     __table_args__ = (
-        Index('ix_domains_name_gin', "name",
+        Index(
+            "ix_domains_name_gin",
+            "name",
             postgresql_ops={"name": "gin_trgm_ops"},
-              postgresql_using='gin'),
+            postgresql_using="gin",
+        ),
     )
     id = Column(Integer, primary_key=True)
     name = Column(String(256), index=True)

lemur/domains/service.py

@@ -6,10 +6,11 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from lemur.domains.models import Domain
-from lemur.certificates.models import Certificate
+from sqlalchemy import and_
 
 from lemur import database
+from lemur.certificates.models import Certificate
+from lemur.domains.models import Domain
 
 
 def get(domain_id):
@@ -42,6 +43,20 @@ def get_by_name(name):
     return database.get_all(Domain, name, field="name").all()
 
 
+def is_domain_sensitive(name):
+    """
+    Return True if domain is marked sensitive
+
+    :param name:
+    :return:
+    """
+    query = database.session_query(Domain)
+
+    query = query.filter(and_(Domain.sensitive, Domain.name == name))
+
+    return database.find_all(query, Domain, {}).all()
+
+
 def create(name, sensitive):
     """
     Create a new domain
@@ -77,11 +92,11 @@ def render(args):
     :return:
     """
     query = database.session_query(Domain)
-    filt = args.pop('filter')
-    certificate_id = args.pop('certificate_id', None)
+    filt = args.pop("filter")
+    certificate_id = args.pop("certificate_id", None)
 
     if filt:
-        terms = filt.split(';')
+        terms = filt.split(";")
         query = database.filter(query, Domain, terms)
 
     if certificate_id:

lemur/domains/views.py

@@ -17,14 +17,19 @@ from lemur.auth.permissions import SensitiveDomainPermission
 from lemur.common.schema import validate_schema
 from lemur.common.utils import paginated_parser
 
-from lemur.domains.schemas import domain_input_schema, domain_output_schema, domains_output_schema
+from lemur.domains.schemas import (
+    domain_input_schema,
+    domain_output_schema,
+    domains_output_schema,
+)
 
-mod = Blueprint('domains', __name__)
+mod = Blueprint("domains", __name__)
 api = Api(mod)
 
 
 class DomainsList(AuthenticatedResource):
     """ Defines the 'domains' endpoint """
+
     def __init__(self):
         super(DomainsList, self).__init__()
 
@@ -123,7 +128,7 @@ class DomainsList(AuthenticatedResource):
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
-        return service.create(data['name'], data['sensitive'])
+        return service.create(data["name"], data["sensitive"])
 
 
 class Domains(AuthenticatedResource):
@@ -205,13 +210,14 @@ class Domains(AuthenticatedResource):
            :statuscode 403: unauthenticated
         """
         if SensitiveDomainPermission().can():
-            return service.update(domain_id, data['name'], data['sensitive'])
+            return service.update(domain_id, data["name"], data["sensitive"])
 
-        return dict(message='You are not authorized to modify this domain'), 403
+        return dict(message="You are not authorized to modify this domain"), 403
 
 
 class CertificateDomains(AuthenticatedResource):
     """ Defines the 'domains' endpoint """
+
     def __init__(self):
         super(CertificateDomains, self).__init__()
 
@@ -265,10 +271,14 @@ class CertificateDomains(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['certificate_id'] = certificate_id
+        args["certificate_id"] = certificate_id
         return service.render(args)
 
 
-api.add_resource(DomainsList, '/domains', endpoint='domains')
-api.add_resource(Domains, '/domains/<int:domain_id>', endpoint='domain')
-api.add_resource(CertificateDomains, '/certificates/<int:certificate_id>/domains', endpoint='certificateDomains')
+api.add_resource(DomainsList, "/domains", endpoint="domains")
+api.add_resource(Domains, "/domains/<int:domain_id>", endpoint="domain")
+api.add_resource(
+    CertificateDomains,
+    "/certificates/<int:certificate_id>/domains",
+    endpoint="certificateDomains",
+)

lemur/endpoints/cli.py

@@ -21,7 +21,14 @@ from lemur.endpoints.models import Endpoint
 manager = Manager(usage="Handles all endpoint related tasks.")
 
 
-@manager.option('-ttl', '--time-to-live', type=int, dest='ttl', default=2, help='Time in hours, which endpoint has not been refreshed to remove the endpoint.')
+@manager.option(
+    "-ttl",
+    "--time-to-live",
+    type=int,
+    dest="ttl",
+    default=2,
+    help="Time in hours, which endpoint has not been refreshed to remove the endpoint.",
+)
 def expire(ttl):
     """
     Removed all endpoints that have not been recently updated.
@@ -31,12 +38,18 @@ def expire(ttl):
     try:
         now = arrow.utcnow()
         expiration = now - timedelta(hours=ttl)
-        endpoints = database.session_query(Endpoint).filter(cast(Endpoint.last_updated, ArrowType) <= expiration)
+        endpoints = database.session_query(Endpoint).filter(
+            cast(Endpoint.last_updated, ArrowType) <= expiration
+        )
 
         for endpoint in endpoints:
-            print("[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(name=endpoint.name, last_updated=endpoint.last_updated))
+            print(
+                "[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(
+                    name=endpoint.name, last_updated=endpoint.last_updated
+                )
+            )
             database.delete(endpoint)
-            metrics.send('endpoint_expired', 'counter', 1)
+            metrics.send("endpoint_expired", "counter", 1)
 
         print("[+] Finished expiration.")
     except Exception as e:

lemur/endpoints/models.py

@@ -20,15 +20,11 @@ from lemur.database import db
 from lemur.models import policies_ciphers
 
 
-BAD_CIPHERS = [
-    'Protocol-SSLv3',
-    'Protocol-SSLv2',
-    'Protocol-TLSv1'
-]
+BAD_CIPHERS = ["Protocol-SSLv3", "Protocol-SSLv2", "Protocol-TLSv1"]
 
 
 class Cipher(db.Model):
-    __tablename__ = 'ciphers'
+    __tablename__ = "ciphers"
     id = Column(Integer, primary_key=True)
     name = Column(String(128), nullable=False)
 
@@ -38,23 +34,18 @@ class Cipher(db.Model):
 
     @deprecated.expression
     def deprecated(cls):
-        return case(
-            [
-                (cls.name in BAD_CIPHERS, True)
-            ],
-            else_=False
-        )
+        return case([(cls.name in BAD_CIPHERS, True)], else_=False)
 
 
 class Policy(db.Model):
-    ___tablename__ = 'policies'
+    ___tablename__ = "policies"
     id = Column(Integer, primary_key=True)
     name = Column(String(128), nullable=True)
-    ciphers = relationship('Cipher', secondary=policies_ciphers, backref='policy')
+    ciphers = relationship("Cipher", secondary=policies_ciphers, backref="policy")
 
 
 class Endpoint(db.Model):
-    __tablename__ = 'endpoints'
+    __tablename__ = "endpoints"
     id = Column(Integer, primary_key=True)
     owner = Column(String(128))
     name = Column(String(128))
@@ -62,16 +53,18 @@ class Endpoint(db.Model):
     type = Column(String(128))
     active = Column(Boolean, default=True)
     port = Column(Integer)
-    policy_id = Column(Integer, ForeignKey('policy.id'))
-    policy = relationship('Policy', backref='endpoint')
-    certificate_id = Column(Integer, ForeignKey('certificates.id'))
-    source_id = Column(Integer, ForeignKey('sources.id'))
+    policy_id = Column(Integer, ForeignKey("policy.id"))
+    policy = relationship("Policy", backref="endpoint")
+    certificate_id = Column(Integer, ForeignKey("certificates.id"))
+    source_id = Column(Integer, ForeignKey("sources.id"))
     sensitive = Column(Boolean, default=False)
-    source = relationship('Source', back_populates='endpoints')
+    source = relationship("Source", back_populates="endpoints")
     last_updated = Column(ArrowType, default=arrow.utcnow, nullable=False)
-    date_created = Column(ArrowType, default=arrow.utcnow, onupdate=arrow.utcnow, nullable=False)
+    date_created = Column(
+        ArrowType, default=arrow.utcnow, onupdate=arrow.utcnow, nullable=False
+    )
 
-    replaced = association_proxy('certificate', 'replaced')
+    replaced = association_proxy("certificate", "replaced")
 
     @property
     def issues(self):
@@ -79,13 +72,30 @@ class Endpoint(db.Model):
 
         for cipher in self.policy.ciphers:
             if cipher.deprecated:
-                issues.append({'name': 'deprecated cipher', 'value': '{0} has been deprecated consider removing it.'.format(cipher.name)})
+                issues.append(
+                    {
+                        "name": "deprecated cipher",
+                        "value": "{0} has been deprecated consider removing it.".format(
+                            cipher.name
+                        ),
+                    }
+                )
 
         if self.certificate.expired:
-            issues.append({'name': 'expired certificate', 'value': 'There is an expired certificate attached to this endpoint consider replacing it.'})
+            issues.append(
+                {
+                    "name": "expired certificate",
+                    "value": "There is an expired certificate attached to this endpoint consider replacing it.",
+                }
+            )
 
         if self.certificate.revoked:
-            issues.append({'name': 'revoked', 'value': 'There is a revoked certificate attached to this endpoint consider replacing it.'})
+            issues.append(
+                {
+                    "name": "revoked",
+                    "value": "There is a revoked certificate attached to this endpoint consider replacing it.",
+                }
+            )
 
         return issues
 

lemur/endpoints/service.py

@@ -46,7 +46,7 @@ def get_by_name(name):
     :param name:
     :return:
     """
-    return database.get(Endpoint, name, field='name')
+    return database.get(Endpoint, name, field="name")
 
 
 def get_by_dnsname(dnsname):
@@ -56,7 +56,7 @@ def get_by_dnsname(dnsname):
     :param dnsname:
     :return:
     """
-    return database.get(Endpoint, dnsname, field='dnsname')
+    return database.get(Endpoint, dnsname, field="dnsname")
 
 
 def get_by_dnsname_and_port(dnsname, port):
@@ -66,7 +66,11 @@ def get_by_dnsname_and_port(dnsname, port):
     :param port:
     :return:
     """
-    return Endpoint.query.filter(Endpoint.dnsname == dnsname).filter(Endpoint.port == port).scalar()
+    return (
+        Endpoint.query.filter(Endpoint.dnsname == dnsname)
+        .filter(Endpoint.port == port)
+        .scalar()
+    )
 
 
 def get_by_source(source_label):
@@ -95,12 +99,14 @@ def create(**kwargs):
     """
     endpoint = Endpoint(**kwargs)
     database.create(endpoint)
-    metrics.send('endpoint_added', 'counter', 1, metric_tags={'source': endpoint.source.label})
+    metrics.send(
+        "endpoint_added", "counter", 1, metric_tags={"source": endpoint.source.label}
+    )
     return endpoint
 
 
 def get_or_create_policy(**kwargs):
-    policy = database.get(Policy, kwargs['name'], field='name')
+    policy = database.get(Policy, kwargs["name"], field="name")
 
     if not policy:
         policy = Policy(**kwargs)
@@ -110,7 +116,7 @@ def get_or_create_policy(**kwargs):
 
 
 def get_or_create_cipher(**kwargs):
-    cipher = database.get(Cipher, kwargs['name'], field='name')
+    cipher = database.get(Cipher, kwargs["name"], field="name")
 
     if not cipher:
         cipher = Cipher(**kwargs)
@@ -122,11 +128,13 @@ def get_or_create_cipher(**kwargs):
 def update(endpoint_id, **kwargs):
     endpoint = database.get(Endpoint, endpoint_id)
 
-    endpoint.policy = kwargs['policy']
-    endpoint.certificate = kwargs['certificate']
-    endpoint.source = kwargs['source']
+    endpoint.policy = kwargs["policy"]
+    endpoint.certificate = kwargs["certificate"]
+    endpoint.source = kwargs["source"]
     endpoint.last_updated = arrow.utcnow()
-    metrics.send('endpoint_updated', 'counter', 1, metric_tags={'source': endpoint.source.label})
+    metrics.send(
+        "endpoint_updated", "counter", 1, metric_tags={"source": endpoint.source.label}
+    )
     database.update(endpoint)
     return endpoint
 
@@ -138,19 +146,17 @@ def render(args):
     :return:
     """
     query = database.session_query(Endpoint)
-    filt = args.pop('filter')
+    filt = args.pop("filter")
 
     if filt:
-        terms = filt.split(';')
-        if 'active' in filt:  # this is really weird but strcmp seems to not work here??
+        terms = filt.split(";")
+        if "active" in filt:  # this is really weird but strcmp seems to not work here??
             query = query.filter(Endpoint.active == truthiness(terms[1]))
-        elif 'port' in filt:
-            if terms[1] != 'null':  # ng-table adds 'null' if a number is removed
+        elif "port" in filt:
+            if terms[1] != "null":  # ng-table adds 'null' if a number is removed
                 query = query.filter(Endpoint.port == terms[1])
-        elif 'ciphers' in filt:
-            query = query.filter(
-                Cipher.name == terms[1]
-            )
+        elif "ciphers" in filt:
+            query = query.filter(Cipher.name == terms[1])
         else:
             query = database.filter(query, Endpoint, terms)
 
@@ -164,7 +170,7 @@ def stats(**kwargs):
     :param kwargs:
     :return:
     """
-    attr = getattr(Endpoint, kwargs.get('metric'))
+    attr = getattr(Endpoint, kwargs.get("metric"))
     query = database.db.session.query(attr, func.count(attr))
 
     items = query.group_by(attr).all()
@@ -175,4 +181,4 @@ def stats(**kwargs):
         keys.append(key)
         values.append(count)
 
-    return {'labels': keys, 'values': values}
+    return {"labels": keys, "values": values}

lemur/endpoints/views.py

@@ -16,12 +16,13 @@ from lemur.endpoints import service
 from lemur.endpoints.schemas import endpoint_output_schema, endpoints_output_schema
 
 
-mod = Blueprint('endpoints', __name__)
+mod = Blueprint("endpoints", __name__)
 api = Api(mod)
 
 
 class EndpointsList(AuthenticatedResource):
     """ Defines the 'endpoints' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(EndpointsList, self).__init__()
@@ -63,7 +64,7 @@ class EndpointsList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['user'] = g.current_user
+        args["user"] = g.current_user
         return service.render(args)
 
 
@@ -103,5 +104,5 @@ class Endpoints(AuthenticatedResource):
         return service.get(endpoint_id)
 
 
-api.add_resource(EndpointsList, '/endpoints', endpoint='endpoints')
-api.add_resource(Endpoints, '/endpoints/<int:endpoint_id>', endpoint='endpoint')
+api.add_resource(EndpointsList, "/endpoints", endpoint="endpoints")
+api.add_resource(Endpoints, "/endpoints/<int:endpoint_id>", endpoint="endpoint")

lemur/exceptions.py

@@ -21,7 +21,9 @@ class DuplicateError(LemurException):
 
 class InvalidListener(LemurException):
     def __str__(self):
-        return repr("Invalid listener, ensure you select a certificate if you are using a secure protocol")
+        return repr(
+            "Invalid listener, ensure you select a certificate if you are using a secure protocol"
+        )
 
 
 class AttrNotFound(LemurException):

lemur/extensions.py

@@ -15,25 +15,33 @@ class SQLAlchemy(SA):
 db = SQLAlchemy()
 
 from flask_migrate import Migrate
+
 migrate = Migrate()
 
 from flask_bcrypt import Bcrypt
+
 bcrypt = Bcrypt()
 
 from flask_principal import Principal
+
 principal = Principal(use_sessions=False)
 
 from flask_mail import Mail
+
 smtp_mail = Mail()
 
 from lemur.metrics import Metrics
+
 metrics = Metrics()
 
 from raven.contrib.flask import Sentry
+
 sentry = Sentry()
 
 from blinker import Namespace
+
 signals = Namespace()
 
 from flask_cors import CORS
+
 cors = CORS()

lemur/factory.py

@@ -13,20 +13,21 @@ import os
 import imp
 import errno
 import pkg_resources
+import socket
 
 from logging import Formatter, StreamHandler
 from logging.handlers import RotatingFileHandler
 
 from flask import Flask
+from flask_replicated import FlaskReplicated
+import logmatic
 
 from lemur.certificates.hooks import activate_debug_dump
 from lemur.common.health import mod as health
 from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry, cors
 
 
-DEFAULT_BLUEPRINTS = (
-    health,
-)
+DEFAULT_BLUEPRINTS = (health,)
 
 API_VERSION = 1
 
@@ -53,6 +54,7 @@ def create_app(app_name=None, blueprints=None, config=None):
     configure_blueprints(app, blueprints)
     configure_extensions(app)
     configure_logging(app)
+    configure_database(app)
     install_plugins(app)
 
     @app.teardown_appcontext
@@ -71,16 +73,17 @@ def from_file(file_path, silent=False):
     :param file_path:
     :param silent:
     """
-    d = imp.new_module('config')
+    d = imp.new_module("config")
     d.__file__ = file_path
     try:
         with open(file_path) as config_file:
-            exec(compile(config_file.read(),  # nosec: config file safe
-                 file_path, 'exec'), d.__dict__)
+            exec(  # nosec: config file safe
+                compile(config_file.read(), file_path, "exec"), d.__dict__
+            )
     except IOError as e:
         if silent and e.errno in (errno.ENOENT, errno.EISDIR):
             return False
-        e.strerror = 'Unable to load configuration file (%s)' % e.strerror
+        e.strerror = "Unable to load configuration file (%s)" % e.strerror
         raise
     return d
 
@@ -94,8 +97,8 @@ def configure_app(app, config=None):
     :return:
     """
     # respect the config first
-    if config and config != 'None':
-        app.config['CONFIG_PATH'] = config
+    if config and config != "None":
+        app.config["CONFIG_PATH"] = config
         app.config.from_object(from_file(config))
     else:
         try:
@@ -103,12 +106,21 @@ def configure_app(app, config=None):
         except RuntimeError:
             # look in default paths
             if os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
-                app.config.from_object(from_file(os.path.expanduser("~/.lemur/lemur.conf.py")))
+                app.config.from_object(
+                    from_file(os.path.expanduser("~/.lemur/lemur.conf.py"))
+                )
             else:
-                app.config.from_object(from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'default.conf.py')))
+                app.config.from_object(
+                    from_file(
+                        os.path.join(
+                            os.path.dirname(os.path.realpath(__file__)),
+                            "default.conf.py",
+                        )
+                    )
+                )
 
     # we don't use this
-    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
 
 
 def configure_extensions(app):
@@ -125,9 +137,15 @@ def configure_extensions(app):
     metrics.init_app(app)
     sentry.init_app(app)
 
-    if app.config['CORS']:
-        app.config['CORS_HEADERS'] = 'Content-Type'
-        cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True)
+    if app.config["CORS"]:
+        app.config["CORS_HEADERS"] = "Content-Type"
+        cors.init_app(
+            app,
+            resources=r"/api/*",
+            headers="Content-Type",
+            origin="*",
+            supports_credentials=True,
+        )
 
 
 def configure_blueprints(app, blueprints):
@@ -142,28 +160,41 @@ def configure_blueprints(app, blueprints):
         app.register_blueprint(blueprint, url_prefix="/api/{0}".format(API_VERSION))
 
 
+def configure_database(app):
+    if app.config.get("SQLALCHEMY_ENABLE_FLASK_REPLICATED"):
+        FlaskReplicated(app)
+
+
 def configure_logging(app):
     """
     Sets up application wide logging.
 
     :param app:
     """
-    handler = RotatingFileHandler(app.config.get('LOG_FILE', 'lemur.log'), maxBytes=10000000, backupCount=100)
+    handler = RotatingFileHandler(
+        app.config.get("LOG_FILE", "lemur.log"), maxBytes=10000000, backupCount=100
+    )
 
-    handler.setFormatter(Formatter(
-        '%(asctime)s %(levelname)s: %(message)s '
-        '[in %(pathname)s:%(lineno)d]'
-    ))
+    handler.setFormatter(
+        Formatter(
+            "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+        )
+    )
 
-    handler.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
-    app.logger.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
+    if app.config.get("LOG_JSON", False):
+        handler.setFormatter(
+            logmatic.JsonFormatter(extra={"hostname": socket.gethostname()})
+        )
+
+    handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG"))
+    app.logger.setLevel(app.config.get("LOG_LEVEL", "DEBUG"))
     app.logger.addHandler(handler)
 
     stream_handler = StreamHandler()
-    stream_handler.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
+    stream_handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG"))
     app.logger.addHandler(stream_handler)
 
-    if app.config.get('DEBUG_DUMP', False):
+    if app.config.get("DEBUG_DUMP", False):
         activate_debug_dump()
 
 
@@ -176,17 +207,21 @@ def install_plugins(app):
     """
     from lemur.plugins import plugins
     from lemur.plugins.base import register
+
     # entry_points={
     #    'lemur.plugins': [
     #         'verisign = lemur_verisign.plugin:VerisignPlugin'
     #     ],
     # },
-    for ep in pkg_resources.iter_entry_points('lemur.plugins'):
+    for ep in pkg_resources.iter_entry_points("lemur.plugins"):
         try:
             plugin = ep.load()
         except Exception:
             import traceback
-            app.logger.error("Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc()))
+
+            app.logger.error(
+                "Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc())
+            )
         else:
             register(plugin)
 
@@ -196,6 +231,9 @@ def install_plugins(app):
         try:
             plugins.get(slug)
         except KeyError:
-            raise Exception("Unable to location notification plugin: {slug}. Ensure that "
-                            "LEMUR_DEFAULT_NOTIFICATION_PLUGIN is set to a valid and installed notification plugin."
-                            .format(slug=slug))
+            raise Exception(
+                "Unable to location notification plugin: {slug}. Ensure that "
+                "LEMUR_DEFAULT_NOTIFICATION_PLUGIN is set to a valid and installed notification plugin.".format(
+                    slug=slug
+                )
+            )

lemur/logs/models.py

@@ -15,9 +15,19 @@ from lemur.database import db
 
 
 class Log(db.Model):
-    __tablename__ = 'logs'
+    __tablename__ = "logs"
     id = Column(Integer, primary_key=True)
-    certificate_id = Column(Integer, ForeignKey('certificates.id'))
-    log_type = Column(Enum('key_view', 'create_cert', 'update_cert', 'revoke_cert', name='log_type'), nullable=False)
+    certificate_id = Column(Integer, ForeignKey("certificates.id"))
+    log_type = Column(
+        Enum(
+            "key_view",
+            "create_cert",
+            "update_cert",
+            "revoke_cert",
+            "delete_cert",
+            name="log_type",
+        ),
+        nullable=False,
+    )
     logged_at = Column(ArrowType(), PassiveDefault(func.now()), nullable=False)
-    user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=False)

lemur/logs/service.py

@@ -24,7 +24,11 @@ def create(user, type, certificate=None):
     :param certificate:
     :return:
     """
-    current_app.logger.info("[lemur-audit] action: {0}, user: {1}, certificate: {2}.".format(type, user.email, certificate.name))
+    current_app.logger.info(
+        "[lemur-audit] action: {0}, user: {1}, certificate: {2}.".format(
+            type, user.email, certificate.name
+        )
+    )
     view = Log(user_id=user.id, log_type=type, certificate_id=certificate.id)
     database.add(view)
     database.commit()
@@ -50,20 +54,22 @@ def render(args):
     """
     query = database.session_query(Log)
 
-    filt = args.pop('filter')
+    filt = args.pop("filter")
 
     if filt:
-        terms = filt.split(';')
+        terms = filt.split(";")
 
-        if 'certificate.name' in terms:
-            sub_query = database.session_query(Certificate.id)\
-                .filter(Certificate.name.ilike('%{0}%'.format(terms[1])))
+        if "certificate.name" in terms:
+            sub_query = database.session_query(Certificate.id).filter(
+                Certificate.name.ilike("%{0}%".format(terms[1]))
+            )
 
             query = query.filter(Log.certificate_id.in_(sub_query))
 
-        elif 'user.email' in terms:
-            sub_query = database.session_query(User.id)\
-                .filter(User.email.ilike('%{0}%'.format(terms[1])))
+        elif "user.email" in terms:
+            sub_query = database.session_query(User.id).filter(
+                User.email.ilike("%{0}%".format(terms[1]))
+            )
 
             query = query.filter(Log.user_id.in_(sub_query))
 

lemur/logs/views.py

@@ -17,12 +17,13 @@ from lemur.logs.schemas import logs_output_schema
 from lemur.logs import service
 
 
-mod = Blueprint('logs', __name__)
+mod = Blueprint("logs", __name__)
 api = Api(mod)
 
 
 class LogsList(AuthenticatedResource):
     """ Defines the 'logs' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(LogsList, self).__init__()
@@ -65,10 +66,10 @@ class LogsList(AuthenticatedResource):
            :statuscode 200: no error
         """
         parser = paginated_parser.copy()
-        parser.add_argument('owner', type=str, location='args')
-        parser.add_argument('id', type=str, location='args')
+        parser.add_argument("owner", type=str, location="args")
+        parser.add_argument("id", type=str, location="args")
         args = parser.parse_args()
         return service.render(args)
 
 
-api.add_resource(LogsList, '/logs', endpoint='logs')
+api.add_resource(LogsList, "/logs", endpoint="logs")

lemur/manage.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 from __future__ import unicode_literals  # at top of module
 
 import os
@@ -49,25 +50,27 @@ from lemur.policies.models import RotationPolicy  # noqa
 from lemur.pending_certificates.models import PendingCertificate  # noqa
 from lemur.dns_providers.models import DnsProvider  # noqa
 
+from sqlalchemy.sql import text
+
 manager = Manager(create_app)
-manager.add_option('-c', '--config', dest='config')
+manager.add_option("-c", "--config", dest="config_path", required=False)
 
 migrate = Migrate(create_app)
 
 REQUIRED_VARIABLES = [
-    'LEMUR_SECURITY_TEAM_EMAIL',
-    'LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',
-    'LEMUR_DEFAULT_ORGANIZATION',
-    'LEMUR_DEFAULT_LOCATION',
-    'LEMUR_DEFAULT_COUNTRY',
-    'LEMUR_DEFAULT_STATE',
-    'SQLALCHEMY_DATABASE_URI'
+    "LEMUR_SECURITY_TEAM_EMAIL",
+    "LEMUR_DEFAULT_ORGANIZATIONAL_UNIT",
+    "LEMUR_DEFAULT_ORGANIZATION",
+    "LEMUR_DEFAULT_LOCATION",
+    "LEMUR_DEFAULT_COUNTRY",
+    "LEMUR_DEFAULT_STATE",
+    "SQLALCHEMY_DATABASE_URI",
 ]
 
 KEY_LENGTH = 40
-DEFAULT_CONFIG_PATH = '~/.lemur/lemur.conf.py'
-DEFAULT_SETTINGS = 'lemur.conf.server'
-SETTINGS_ENVVAR = 'LEMUR_CONF'
+DEFAULT_CONFIG_PATH = "~/.lemur/lemur.conf.py"
+DEFAULT_SETTINGS = "lemur.conf.server"
+SETTINGS_ENVVAR = "LEMUR_CONF"
 
 CONFIG_TEMPLATE = """
 # This is just Python which means you can inherit and tweak settings
@@ -142,8 +145,9 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
 
 @MigrateCommand.command
 def create():
+    database.db.engine.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
     database.db.create_all()
-    stamp(revision='head')
+    stamp(revision="head")
 
 
 @MigrateCommand.command
@@ -171,9 +175,9 @@ def generate_settings():
     output = CONFIG_TEMPLATE.format(
         # we use Fernet.generate_key to make sure that the key length is
         # compatible with Fernet
-        encryption_key=Fernet.generate_key().decode('utf-8'),
-        secret_token=base64.b64encode(os.urandom(KEY_LENGTH)).decode('utf-8'),
-        flask_secret_key=base64.b64encode(os.urandom(KEY_LENGTH)).decode('utf-8'),
+        encryption_key=Fernet.generate_key().decode("utf-8"),
+        secret_token=base64.b64encode(os.urandom(KEY_LENGTH)).decode("utf-8"),
+        flask_secret_key=base64.b64encode(os.urandom(KEY_LENGTH)).decode("utf-8"),
     )
 
     return output
@@ -187,39 +191,44 @@ class InitializeApp(Command):
     Additionally a Lemur user will be created as a default user
     and be used when certificates are discovered by Lemur.
     """
-    option_list = (
-        Option('-p', '--password', dest='password'),
-    )
+
+    option_list = (Option("-p", "--password", dest="password"),)
 
     def run(self, password):
         create()
         user = user_service.get_by_username("lemur")
 
-        admin_role = role_service.get_by_name('admin')
+        admin_role = role_service.get_by_name("admin")
 
         if admin_role:
             sys.stdout.write("[-] Admin role already created, skipping...!\n")
         else:
             # we create an admin role
-            admin_role = role_service.create('admin', description='This is the Lemur administrator role.')
+            admin_role = role_service.create(
+                "admin", description="This is the Lemur administrator role."
+            )
             sys.stdout.write("[+] Created 'admin' role\n")
 
-        operator_role = role_service.get_by_name('operator')
+        operator_role = role_service.get_by_name("operator")
 
         if operator_role:
             sys.stdout.write("[-] Operator role already created, skipping...!\n")
         else:
             # we create an operator role
-            operator_role = role_service.create('operator', description='This is the Lemur operator role.')
+            operator_role = role_service.create(
+                "operator", description="This is the Lemur operator role."
+            )
             sys.stdout.write("[+] Created 'operator' role\n")
 
-        read_only_role = role_service.get_by_name('read-only')
+        read_only_role = role_service.get_by_name("read-only")
 
         if read_only_role:
             sys.stdout.write("[-] Read only role already created, skipping...!\n")
         else:
             # we create an read only role
-            read_only_role = role_service.create('read-only', description='This is the Lemur read only role.')
+            read_only_role = role_service.create(
+                "read-only", description="This is the Lemur read only role."
+            )
             sys.stdout.write("[+] Created 'read-only' role\n")
 
         if not user:
@@ -232,34 +241,54 @@ class InitializeApp(Command):
                     sys.stderr.write("[!] Passwords do not match!\n")
                     sys.exit(1)
 
-            user_service.create("lemur", password, 'lemur@nobody.com', True, None, [admin_role])
-            sys.stdout.write("[+] Created the user 'lemur' and granted it the 'admin' role!\n")
+            user_service.create(
+                "lemur", password, "lemur@nobody.com", True, None, [admin_role]
+            )
+            sys.stdout.write(
+                "[+] Created the user 'lemur' and granted it the 'admin' role!\n"
+            )
 
         else:
-            sys.stdout.write("[-] Default user has already been created, skipping...!\n")
+            sys.stdout.write(
+                "[-] Default user has already been created, skipping...!\n"
+            )
 
-        intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", [])
+        intervals = current_app.config.get(
+            "LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", []
+        )
         sys.stdout.write(
             "[!] Creating {num} notifications for {intervals} days as specified by LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS\n".format(
-                num=len(intervals),
-                intervals=",".join([str(x) for x in intervals])
+                num=len(intervals), intervals=",".join([str(x) for x in intervals])
             )
         )
 
-        recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
+        recipients = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
         sys.stdout.write("[+] Creating expiration email notifications!\n")
-        sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format(recipients))
-        notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients)
+        sys.stdout.write(
+            "[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format(
+                recipients
+            )
+        )
+        notification_service.create_default_expiration_notifications(
+            "DEFAULT_SECURITY", recipients=recipients
+        )
 
-        _DEFAULT_ROTATION_INTERVAL = 'default'
-        default_rotation_interval = policy_service.get_by_name(_DEFAULT_ROTATION_INTERVAL)
+        _DEFAULT_ROTATION_INTERVAL = "default"
+        default_rotation_interval = policy_service.get_by_name(
+            _DEFAULT_ROTATION_INTERVAL
+        )
 
         if default_rotation_interval:
-            sys.stdout.write("[-] Default rotation interval policy already created, skipping...!\n")
+            sys.stdout.write(
+                "[-] Default rotation interval policy already created, skipping...!\n"
+            )
         else:
             days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30)
-            sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
-                days=days))
+            sys.stdout.write(
+                "[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
+                    days=days
+                )
+            )
             policy_service.create(days=days, name=_DEFAULT_ROTATION_INTERVAL)
 
         sys.stdout.write("[/] Done!\n")
@@ -269,14 +298,16 @@ class CreateUser(Command):
     """
     This command allows for the creation of a new user within Lemur.
     """
+
     option_list = (
-        Option('-u', '--username', dest='username', required=True),
-        Option('-e', '--email', dest='email', required=True),
-        Option('-a', '--active', dest='active', default=True),
-        Option('-r', '--roles', dest='roles', action='append', default=[])
+        Option("-u", "--username", dest="username", required=True),
+        Option("-e", "--email", dest="email", required=True),
+        Option("-a", "--active", dest="active", default=True),
+        Option("-r", "--roles", dest="roles", action="append", default=[]),
+        Option("-p", "--password", dest="password", default=None),
     )
 
-    def run(self, username, email, active, roles):
+    def run(self, username, email, active, roles, password):
         role_objs = []
         for r in roles:
             role_obj = role_service.get_by_name(r)
@@ -286,14 +317,16 @@ class CreateUser(Command):
                 sys.stderr.write("[!] Cannot find role {0}\n".format(r))
                 sys.exit(1)
 
+        if not password:
             password1 = prompt_pass("Password")
             password2 = prompt_pass("Confirm Password")
+            password = password1
 
             if password1 != password2:
                 sys.stderr.write("[!] Passwords do not match!\n")
                 sys.exit(1)
 
-        user_service.create(username, password1, email, active, None, role_objs)
+        user_service.create(username, password, email, active, None, role_objs)
         sys.stdout.write("[+] Created new user: {0}\n".format(username))
 
 
@@ -301,9 +334,8 @@ class ResetPassword(Command):
     """
     This command allows you to reset a user's password.
     """
-    option_list = (
-        Option('-u', '--username', dest='username', required=True),
-    )
+
+    option_list = (Option("-u", "--username", dest="username", required=True),)
 
     def run(self, username):
         user = user_service.get_by_username(username)
@@ -329,10 +361,11 @@ class CreateRole(Command):
     """
     This command allows for the creation of a new role within Lemur
     """
+
     option_list = (
-        Option('-n', '--name', dest='name', required=True),
-        Option('-u', '--users', dest='users', default=[]),
-        Option('-d', '--description', dest='description', required=True)
+        Option("-n", "--name", dest="name", required=True),
+        Option("-u", "--users", dest="users", default=[]),
+        Option("-d", "--description", dest="description", required=True),
     )
 
     def run(self, name, users, description):
@@ -363,7 +396,8 @@ class LemurServer(Command):
 
     Will start gunicorn with 4 workers bound to 127.0.0.0:8002
     """
-    description = 'Run the app within Gunicorn'
+
+    description = "Run the app within Gunicorn"
 
     def get_options(self):
         settings = make_settings()
@@ -371,8 +405,10 @@ class LemurServer(Command):
         for setting, klass in settings.items():
             if klass.cli:
                 if klass.action:
-                    if klass.action == 'store_const':
-                        options.append(Option(*klass.cli, const=klass.const, action=klass.action))
+                    if klass.action == "store_const":
+                        options.append(
+                            Option(*klass.cli, const=klass.const, action=klass.action)
+                        )
                     else:
                         options.append(Option(*klass.cli, action=klass.action))
                 else:
@@ -388,7 +424,9 @@ class LemurServer(Command):
         # run startup tasks on an app like object
         validate_conf(current_app, REQUIRED_VARIABLES)
 
-        app.app_uri = 'lemur:create_app(config="{0}")'.format(current_app.config.get('CONFIG_PATH'))
+        app.app_uri = 'lemur:create_app(config_path="{0}")'.format(
+            current_app.config.get("CONFIG_PATH")
+        )
 
         return app.run()
 
@@ -408,7 +446,7 @@ def create_config(config_path=None):
         os.makedirs(dir)
 
     config = generate_settings()
-    with open(config_path, 'w') as f:
+    with open(config_path, "w") as f:
         f.write(config)
 
     sys.stdout.write("[+] Created a new configuration file {0}\n".format(config_path))
@@ -430,7 +468,7 @@ def lock(path=None):
     :param: path
     """
     if not path:
-        path = os.path.expanduser('~/.lemur/keys')
+        path = os.path.expanduser("~/.lemur/keys")
 
     dest_dir = os.path.join(path, "encrypted")
     sys.stdout.write("[!] Generating a new key...\n")
@@ -441,15 +479,17 @@ def lock(path=None):
         sys.stdout.write("[+] Creating encryption directory: {0}\n".format(dest_dir))
         os.makedirs(dest_dir)
 
-    for root, dirs, files in os.walk(os.path.join(path, 'decrypted')):
+    for root, dirs, files in os.walk(os.path.join(path, "decrypted")):
         for f in files:
             source = os.path.join(root, f)
             dest = os.path.join(dest_dir, f + ".enc")
-            with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
+            with open(source, "rb") as in_file, open(dest, "wb") as out_file:
                 f = Fernet(key)
                 data = f.encrypt(in_file.read())
                 out_file.write(data)
-                sys.stdout.write("[+] Writing file: {0} Source: {1}\n".format(dest, source))
+                sys.stdout.write(
+                    "[+] Writing file: {0} Source: {1}\n".format(dest, source)
+                )
 
     sys.stdout.write("[+] Keys have been encrypted with key {0}\n".format(key))
 
@@ -469,7 +509,7 @@ def unlock(path=None):
     key = prompt_pass("[!] Please enter the encryption password")
 
     if not path:
-        path = os.path.expanduser('~/.lemur/keys')
+        path = os.path.expanduser("~/.lemur/keys")
 
     dest_dir = os.path.join(path, "decrypted")
     source_dir = os.path.join(path, "encrypted")
@@ -482,11 +522,13 @@ def unlock(path=None):
         for f in files:
             source = os.path.join(source_dir, f)
             dest = os.path.join(dest_dir, ".".join(f.split(".")[:-1]))
-            with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
+            with open(source, "rb") as in_file, open(dest, "wb") as out_file:
                 f = Fernet(key)
                 data = f.decrypt(in_file.read())
                 out_file.write(data)
-                sys.stdout.write("[+] Writing file: {0} Source: {1}\n".format(dest, source))
+                sys.stdout.write(
+                    "[+] Writing file: {0} Source: {1}\n".format(dest, source)
+                )
 
     sys.stdout.write("[+] Keys have been unencrypted!\n")
 
@@ -499,15 +541,16 @@ def publish_verisign_units():
     :return:
     """
     from lemur.plugins import plugins
-    v = plugins.get('verisign-issuer')
+
+    v = plugins.get("verisign-issuer")
     units = v.get_available_units()
 
     metrics = {}
     for item in units:
-        if item['@type'] in metrics.keys():
-            metrics[item['@type']] += int(item['@remaining'])
+        if item["@type"] in metrics.keys():
+            metrics[item["@type"]] += int(item["@remaining"])
         else:
-            metrics.update({item['@type']: int(item['@remaining'])})
+            metrics.update({item["@type"]: int(item["@remaining"])})
 
     for name, value in metrics.items():
         metric = [
@@ -516,16 +559,16 @@ def publish_verisign_units():
                 "type": "GAUGE",
                 "name": "Symantec {0} Unit Count".format(name),
                 "tags": {},
-                "value": value
+                "value": value,
             }
         ]
 
-        requests.post('http://localhost:8078/metrics', data=json.dumps(metric))
+        requests.post("http://localhost:8078/metrics", data=json.dumps(metric))
 
 
 def main():
     manager.add_command("start", LemurServer())
-    manager.add_command("runserver", Server(host='127.0.0.1', threaded=True))
+    manager.add_command("runserver", Server(host="127.0.0.1", threaded=True))
     manager.add_command("clean", Clean())
     manager.add_command("show_urls", ShowUrls())
     manager.add_command("db", MigrateCommand)

lemur/metrics.py

@@ -11,6 +11,7 @@ class Metrics(object):
     """
     :param app: The Flask application object. Defaults to None.
     """
+
     _providers = []
 
     def __init__(self, app=None):
@@ -22,11 +23,14 @@ class Metrics(object):
 
         :param app: The Flask application object.
         """
-        self._providers = app.config.get('METRIC_PROVIDERS', [])
+        self._providers = app.config.get("METRIC_PROVIDERS", [])
 
     def send(self, metric_name, metric_type, metric_value, *args, **kwargs):
         for provider in self._providers:
             current_app.logger.debug(
-                "Sending metric '{metric}' to the {provider} provider.".format(metric=metric_name, provider=provider))
+                "Sending metric '{metric}' to the {provider} provider.".format(
+                    metric=metric_name, provider=provider
+                )
+            )
             p = plugins.get(provider)
             p.submit(metric_name, metric_type, metric_value, *args, **kwargs)

lemur/migrations/env.py

@@ -19,8 +19,11 @@ fileConfig(config.config_file_name)
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 from flask import current_app
-config.set_main_option('sqlalchemy.url', current_app.config.get('SQLALCHEMY_DATABASE_URI'))
-target_metadata = current_app.extensions['migrate'].db.metadata
+
+config.set_main_option(
+    "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI")
+)
+target_metadata = current_app.extensions["migrate"].db.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -54,14 +57,18 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    engine = engine_from_config(config.get_section(config.config_ini_section),
-                                prefix='sqlalchemy.',
-                                poolclass=pool.NullPool)
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
 
     connection = engine.connect()
-    context.configure(connection=connection,
+    context.configure(
+        connection=connection,
         target_metadata=target_metadata,
-                      **current_app.extensions['migrate'].configure_args)
+        **current_app.extensions["migrate"].configure_args
+    )
 
     try:
         with context.begin_transaction():
@@ -69,8 +76,8 @@ def run_migrations_online():
     finally:
         connection.close()
 
+
 if context.is_offline_mode():
     run_migrations_offline()
 else:
     run_migrations_online()
-

lemur/migrations/versions/131ec6accff5_.py

@@ -7,8 +7,8 @@ Create Date: 2016-12-07 17:29:42.049986
 """
 
 # revision identifiers, used by Alembic.
-revision = '131ec6accff5'
-down_revision = 'e3691fc396e9'
+revision = "131ec6accff5"
+down_revision = "e3691fc396e9"
 
 from alembic import op
 import sqlalchemy as sa
@@ -16,13 +16,24 @@ import sqlalchemy as sa
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('certificates', sa.Column('rotation', sa.Boolean(), nullable=False, server_default=sa.false()))
-    op.add_column('endpoints', sa.Column('last_updated', sa.DateTime(), server_default=sa.text('now()'), nullable=False))
+    op.add_column(
+        "certificates",
+        sa.Column("rotation", sa.Boolean(), nullable=False, server_default=sa.false()),
+    )
+    op.add_column(
+        "endpoints",
+        sa.Column(
+            "last_updated",
+            sa.DateTime(),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+    )
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('endpoints', 'last_updated')
-    op.drop_column('certificates', 'rotation')
+    op.drop_column("endpoints", "last_updated")
+    op.drop_column("certificates", "rotation")
     # ### end Alembic commands ###

lemur/migrations/versions/1ae8e3104db8_.py

@@ -7,15 +7,19 @@ Create Date: 2017-07-13 12:32:09.162800
 """
 
 # revision identifiers, used by Alembic.
-revision = '1ae8e3104db8'
-down_revision = 'a02a678ddc25'
+revision = "1ae8e3104db8"
+down_revision = "a02a678ddc25"
 
 from alembic import op
 
 
 def upgrade():
-    op.sync_enum_values('public', 'log_type', ['key_view'], ['create_cert', 'key_view', 'update_cert'])
+    op.sync_enum_values(
+        "public", "log_type", ["key_view"], ["create_cert", "key_view", "update_cert"]
+    )
 
 
 def downgrade():
-    op.sync_enum_values('public', 'log_type', ['create_cert', 'key_view', 'update_cert'], ['key_view'])
+    op.sync_enum_values(
+        "public", "log_type", ["create_cert", "key_view", "update_cert"], ["key_view"]
+    )

lemur/migrations/versions/1db4f82bc780_.py

@@ -7,8 +7,8 @@ Create Date: 2018-08-03 12:56:44.565230
 """
 
 # revision identifiers, used by Alembic.
-revision = '1db4f82bc780'
-down_revision = '3adfdd6598df'
+revision = "1db4f82bc780"
+down_revision = "3adfdd6598df"
 
 import logging
 
@@ -20,12 +20,14 @@ log = logging.getLogger(__name__)
 def upgrade():
     connection = op.get_bind()
 
-    result = connection.execute("""\
+    result = connection.execute(
+        """\
        UPDATE certificates
            SET rotation_policy_id=(SELECT id FROM rotation_policies WHERE name='default')
          WHERE rotation_policy_id IS NULL
         RETURNING id
-    """)
+    """
+    )
     log.info("Filled rotation_policy for %d certificates" % result.rowcount)
 
 

lemur/migrations/versions/29d8c8455c86_.py

@@ -7,8 +7,8 @@ Create Date: 2016-06-28 16:05:25.720213
 """
 
 # revision identifiers, used by Alembic.
-revision = '29d8c8455c86'
-down_revision = '3307381f3b88'
+revision = "29d8c8455c86"
+down_revision = "3307381f3b88"
 
 from alembic import op
 import sqlalchemy as sa
@@ -17,46 +17,60 @@ from sqlalchemy.dialects import postgresql
 
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('ciphers',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('name', sa.String(length=128), nullable=False),
-    sa.PrimaryKeyConstraint('id')
+    op.create_table(
+        "ciphers",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(length=128), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
     )
-    op.create_table('policy',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('name', sa.String(length=128), nullable=True),
-    sa.PrimaryKeyConstraint('id')
+    op.create_table(
+        "policy",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(length=128), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
     )
-    op.create_table('policies_ciphers',
-    sa.Column('cipher_id', sa.Integer(), nullable=True),
-    sa.Column('policy_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['cipher_id'], ['ciphers.id'], ),
-    sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], )
+    op.create_table(
+        "policies_ciphers",
+        sa.Column("cipher_id", sa.Integer(), nullable=True),
+        sa.Column("policy_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(["cipher_id"], ["ciphers.id"]),
+        sa.ForeignKeyConstraint(["policy_id"], ["policy.id"]),
     )
-    op.create_index('policies_ciphers_ix', 'policies_ciphers', ['cipher_id', 'policy_id'], unique=False)
-    op.create_table('endpoints',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('owner', sa.String(length=128), nullable=True),
-    sa.Column('name', sa.String(length=128), nullable=True),
-    sa.Column('dnsname', sa.String(length=256), nullable=True),
-    sa.Column('type', sa.String(length=128), nullable=True),
-    sa.Column('active', sa.Boolean(), nullable=True),
-    sa.Column('port', sa.Integer(), nullable=True),
-    sa.Column('date_created', sa.DateTime(), server_default=sa.text(u'now()'), nullable=False),
-    sa.Column('policy_id', sa.Integer(), nullable=True),
-    sa.Column('certificate_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ),
-    sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], ),
-    sa.PrimaryKeyConstraint('id')
+    op.create_index(
+        "policies_ciphers_ix",
+        "policies_ciphers",
+        ["cipher_id", "policy_id"],
+        unique=False,
+    )
+    op.create_table(
+        "endpoints",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("owner", sa.String(length=128), nullable=True),
+        sa.Column("name", sa.String(length=128), nullable=True),
+        sa.Column("dnsname", sa.String(length=256), nullable=True),
+        sa.Column("type", sa.String(length=128), nullable=True),
+        sa.Column("active", sa.Boolean(), nullable=True),
+        sa.Column("port", sa.Integer(), nullable=True),
+        sa.Column(
+            "date_created",
+            sa.DateTime(),
+            server_default=sa.text(u"now()"),
+            nullable=False,
+        ),
+        sa.Column("policy_id", sa.Integer(), nullable=True),
+        sa.Column("certificate_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(["certificate_id"], ["certificates.id"]),
+        sa.ForeignKeyConstraint(["policy_id"], ["policy.id"]),
+        sa.PrimaryKeyConstraint("id"),
     )
     ### end Alembic commands ###
 
 
 def downgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('endpoints')
-    op.drop_index('policies_ciphers_ix', table_name='policies_ciphers')
-    op.drop_table('policies_ciphers')
-    op.drop_table('policy')
-    op.drop_table('ciphers')
+    op.drop_table("endpoints")
+    op.drop_index("policies_ciphers_ix", table_name="policies_ciphers")
+    op.drop_table("policies_ciphers")
+    op.drop_table("policy")
+    op.drop_table("ciphers")
     ### end Alembic commands ###

lemur/migrations/versions/318b66568358_.py

@@ -0,0 +1,23 @@
+""" Set 'deleted' flag from null to false on all certificates once
+
+Revision ID: 318b66568358
+Revises: 9f79024fe67b
+Create Date: 2019-02-05 15:42:25.477587
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "318b66568358"
+down_revision = "9f79024fe67b"
+
+from alembic import op
+
+
+def upgrade():
+    connection = op.get_bind()
+    # Delete duplicate entries
+    connection.execute("UPDATE certificates SET deleted = false WHERE deleted IS NULL")
+
+
+def downgrade():
+    pass

lemur/migrations/versions/3307381f3b88_.py

@@ -12,8 +12,8 @@ Create Date: 2016-05-20 17:33:04.360687
 """
 
 # revision identifiers, used by Alembic.
-revision = '3307381f3b88'
-down_revision = '412b22cb656a'
+revision = "3307381f3b88"
+down_revision = "412b22cb656a"
 
 from alembic import op
 import sqlalchemy as sa
@@ -23,109 +23,165 @@ from sqlalchemy.dialects import postgresql
 
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.alter_column('authorities', 'owner',
-               existing_type=sa.VARCHAR(length=128),
-               nullable=True)
-    op.drop_column('authorities', 'not_after')
-    op.drop_column('authorities', 'bits')
-    op.drop_column('authorities', 'cn')
-    op.drop_column('authorities', 'not_before')
-    op.add_column('certificates', sa.Column('root_authority_id', sa.Integer(), nullable=True))
-    op.alter_column('certificates', 'body',
-               existing_type=sa.TEXT(),
-               nullable=False)
-    op.alter_column('certificates', 'owner',
-               existing_type=sa.VARCHAR(length=128),
-               nullable=True)
-    op.drop_constraint(u'certificates_authority_id_fkey', 'certificates', type_='foreignkey')
-    op.create_foreign_key(None, 'certificates', 'authorities', ['authority_id'], ['id'], ondelete='CASCADE')
-    op.create_foreign_key(None, 'certificates', 'authorities', ['root_authority_id'], ['id'], ondelete='CASCADE')
+    op.alter_column(
+        "authorities", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
+    )
+    op.drop_column("authorities", "not_after")
+    op.drop_column("authorities", "bits")
+    op.drop_column("authorities", "cn")
+    op.drop_column("authorities", "not_before")
+    op.add_column(
+        "certificates", sa.Column("root_authority_id", sa.Integer(), nullable=True)
+    )
+    op.alter_column("certificates", "body", existing_type=sa.TEXT(), nullable=False)
+    op.alter_column(
+        "certificates", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
+    )
+    op.drop_constraint(
+        u"certificates_authority_id_fkey", "certificates", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        None,
+        "certificates",
+        "authorities",
+        ["authority_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.create_foreign_key(
+        None,
+        "certificates",
+        "authorities",
+        ["root_authority_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
     ### end Alembic commands ###
 
     # link existing certificate to their authority certificates
     conn = op.get_bind()
-    for id, body, owner in conn.execute(text('select id, body, owner from authorities')):
+    for id, body, owner in conn.execute(
+        text("select id, body, owner from authorities")
+    ):
         if not owner:
             owner = "lemur@nobody"
 
         # look up certificate by body, if duplications are found, pick one
-        stmt = text('select id from certificates where body=:body')
+        stmt = text("select id from certificates where body=:body")
         stmt = stmt.bindparams(body=body)
         root_certificate = conn.execute(stmt).fetchone()
         if root_certificate:
-            stmt = text('update certificates set root_authority_id=:root_authority_id where id=:id')
+            stmt = text(
+                "update certificates set root_authority_id=:root_authority_id where id=:id"
+            )
             stmt = stmt.bindparams(root_authority_id=id, id=root_certificate[0])
             op.execute(stmt)
 
         # link owner roles to their authorities
-        stmt = text('select id from roles where name=:name')
+        stmt = text("select id from roles where name=:name")
         stmt = stmt.bindparams(name=owner)
         owner_role = conn.execute(stmt).fetchone()
 
         if not owner_role:
-            stmt = text('insert into roles (name, description) values (:name, :description)')
-            stmt = stmt.bindparams(name=owner, description='Lemur generated role or existing owner.')
+            stmt = text(
+                "insert into roles (name, description) values (:name, :description)"
+            )
+            stmt = stmt.bindparams(
+                name=owner, description="Lemur generated role or existing owner."
+            )
             op.execute(stmt)
 
-        stmt = text('select id from roles where name=:name')
+        stmt = text("select id from roles where name=:name")
         stmt = stmt.bindparams(name=owner)
         owner_role = conn.execute(stmt).fetchone()
 
-        stmt = text('select * from roles_authorities where role_id=:role_id and authority_id=:authority_id')
+        stmt = text(
+            "select * from roles_authorities where role_id=:role_id and authority_id=:authority_id"
+        )
         stmt = stmt.bindparams(role_id=owner_role[0], authority_id=id)
         exists = conn.execute(stmt).fetchone()
 
         if not exists:
-            stmt = text('insert into roles_authorities (role_id, authority_id) values (:role_id, :authority_id)')
+            stmt = text(
+                "insert into roles_authorities (role_id, authority_id) values (:role_id, :authority_id)"
+            )
             stmt = stmt.bindparams(role_id=owner_role[0], authority_id=id)
             op.execute(stmt)
 
     # link owner roles to their certificates
-    for id, owner in conn.execute(text('select id, owner from certificates')):
+    for id, owner in conn.execute(text("select id, owner from certificates")):
         if not owner:
             owner = "lemur@nobody"
 
-        stmt = text('select id from roles where name=:name')
+        stmt = text("select id from roles where name=:name")
         stmt = stmt.bindparams(name=owner)
         owner_role = conn.execute(stmt).fetchone()
 
         if not owner_role:
-            stmt = text('insert into roles (name, description) values (:name, :description)')
-            stmt = stmt.bindparams(name=owner, description='Lemur generated role or existing owner.')
+            stmt = text(
+                "insert into roles (name, description) values (:name, :description)"
+            )
+            stmt = stmt.bindparams(
+                name=owner, description="Lemur generated role or existing owner."
+            )
             op.execute(stmt)
 
         # link owner roles to their authorities
-        stmt = text('select id from roles where name=:name')
+        stmt = text("select id from roles where name=:name")
         stmt = stmt.bindparams(name=owner)
         owner_role = conn.execute(stmt).fetchone()
 
-        stmt = text('select * from roles_certificates where role_id=:role_id and certificate_id=:certificate_id')
+        stmt = text(
+            "select * from roles_certificates where role_id=:role_id and certificate_id=:certificate_id"
+        )
         stmt = stmt.bindparams(role_id=owner_role[0], certificate_id=id)
         exists = conn.execute(stmt).fetchone()
 
         if not exists:
-            stmt = text('insert into roles_certificates (role_id, certificate_id) values (:role_id, :certificate_id)')
+            stmt = text(
+                "insert into roles_certificates (role_id, certificate_id) values (:role_id, :certificate_id)"
+            )
             stmt = stmt.bindparams(role_id=owner_role[0], certificate_id=id)
             op.execute(stmt)
 
 
 def downgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.drop_constraint(None, 'certificates', type_='foreignkey')
-    op.drop_constraint(None, 'certificates', type_='foreignkey')
-    op.create_foreign_key(u'certificates_authority_id_fkey', 'certificates', 'authorities', ['authority_id'], ['id'])
-    op.alter_column('certificates', 'owner',
-               existing_type=sa.VARCHAR(length=128),
-               nullable=True)
-    op.alter_column('certificates', 'body',
-               existing_type=sa.TEXT(),
-               nullable=True)
-    op.drop_column('certificates', 'root_authority_id')
-    op.add_column('authorities', sa.Column('not_before', postgresql.TIMESTAMP(), autoincrement=False, nullable=True))
-    op.add_column('authorities', sa.Column('cn', sa.VARCHAR(length=128), autoincrement=False, nullable=True))
-    op.add_column('authorities', sa.Column('bits', sa.INTEGER(), autoincrement=False, nullable=True))
-    op.add_column('authorities', sa.Column('not_after', postgresql.TIMESTAMP(), autoincrement=False, nullable=True))
-    op.alter_column('authorities', 'owner',
-               existing_type=sa.VARCHAR(length=128),
-               nullable=True)
+    op.drop_constraint(None, "certificates", type_="foreignkey")
+    op.drop_constraint(None, "certificates", type_="foreignkey")
+    op.create_foreign_key(
+        u"certificates_authority_id_fkey",
+        "certificates",
+        "authorities",
+        ["authority_id"],
+        ["id"],
+    )
+    op.alter_column(
+        "certificates", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
+    )
+    op.alter_column("certificates", "body", existing_type=sa.TEXT(), nullable=True)
+    op.drop_column("certificates", "root_authority_id")
+    op.add_column(
+        "authorities",
+        sa.Column(
+            "not_before", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
+        ),
+    )
+    op.add_column(
+        "authorities",
+        sa.Column("cn", sa.VARCHAR(length=128), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "authorities",
+        sa.Column("bits", sa.INTEGER(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "authorities",
+        sa.Column(
+            "not_after", postgresql.TIMESTAMP(), autoincrement=False, nullable=True
+        ),
+    )
+    op.alter_column(
+        "authorities", "owner", existing_type=sa.VARCHAR(length=128), nullable=True
+    )
     ### end Alembic commands ###

lemur/migrations/versions/33de094da890_.py

@@ -7,25 +7,31 @@ Create Date: 2015-11-30 15:40:19.827272
 """
 
 # revision identifiers, used by Alembic.
-revision = '33de094da890'
+revision = "33de094da890"
 down_revision = None
 
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('certificate_replacement_associations',
-    sa.Column('replaced_certificate_id', sa.Integer(), nullable=True),
-    sa.Column('certificate_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['replaced_certificate_id'], ['certificates.id'], ondelete='cascade')
+    op.create_table(
+        "certificate_replacement_associations",
+        sa.Column("replaced_certificate_id", sa.Integer(), nullable=True),
+        sa.Column("certificate_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["certificate_id"], ["certificates.id"], ondelete="cascade"
+        ),
+        sa.ForeignKeyConstraint(
+            ["replaced_certificate_id"], ["certificates.id"], ondelete="cascade"
+        ),
     )
     ### end Alembic commands ###
 
 
 def downgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('certificate_replacement_associations')
+    op.drop_table("certificate_replacement_associations")
     ### end Alembic commands ###

lemur/migrations/versions/3adfdd6598df_.py

@@ -7,8 +7,8 @@ Create Date: 2018-04-10 13:25:47.007556
 """
 
 # revision identifiers, used by Alembic.
-revision = '3adfdd6598df'
-down_revision = '556ceb3e3c3e'
+revision = "3adfdd6598df"
+down_revision = "556ceb3e3c3e"
 
 import sqlalchemy as sa
 from alembic import op
@@ -22,84 +22,90 @@ def upgrade():
     # create provider table
     print("Creating dns_providers table")
     op.create_table(
-        'dns_providers',
-        sa.Column('id', sa.Integer(), nullable=False),
-        sa.Column('name', sa.String(length=256), nullable=True),
-        sa.Column('description', sa.String(length=1024), nullable=True),
-        sa.Column('provider_type', sa.String(length=256), nullable=True),
-        sa.Column('credentials', Vault(), nullable=True),
-        sa.Column('api_endpoint', sa.String(length=256), nullable=True),
-        sa.Column('date_created', ArrowType(), server_default=sa.text('now()'), nullable=False),
-        sa.Column('status', sa.String(length=128), nullable=True),
-        sa.Column('options', JSON),
-        sa.Column('domains', sa.JSON(), nullable=True),
-        sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('name')
+        "dns_providers",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(length=256), nullable=True),
+        sa.Column("description", sa.String(length=1024), nullable=True),
+        sa.Column("provider_type", sa.String(length=256), nullable=True),
+        sa.Column("credentials", Vault(), nullable=True),
+        sa.Column("api_endpoint", sa.String(length=256), nullable=True),
+        sa.Column(
+            "date_created", ArrowType(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column("status", sa.String(length=128), nullable=True),
+        sa.Column("options", JSON),
+        sa.Column("domains", sa.JSON(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
     )
     print("Adding dns_provider_id column to certificates")
-    op.add_column('certificates', sa.Column('dns_provider_id', sa.Integer(), nullable=True))
+    op.add_column(
+        "certificates", sa.Column("dns_provider_id", sa.Integer(), nullable=True)
+    )
     print("Adding dns_provider_id column to pending_certs")
-    op.add_column('pending_certs', sa.Column('dns_provider_id', sa.Integer(), nullable=True))
+    op.add_column(
+        "pending_certs", sa.Column("dns_provider_id", sa.Integer(), nullable=True)
+    )
     print("Adding options column to pending_certs")
-    op.add_column('pending_certs', sa.Column('options', JSON))
+    op.add_column("pending_certs", sa.Column("options", JSON))
 
     print("Creating pending_dns_authorizations table")
     op.create_table(
-        'pending_dns_authorizations',
-        sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
-        sa.Column('account_number', sa.String(length=128), nullable=True),
-        sa.Column('domains', JSON, nullable=True),
-        sa.Column('dns_provider_type', sa.String(length=128), nullable=True),
-        sa.Column('options', JSON, nullable=True),
+        "pending_dns_authorizations",
+        sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
+        sa.Column("account_number", sa.String(length=128), nullable=True),
+        sa.Column("domains", JSON, nullable=True),
+        sa.Column("dns_provider_type", sa.String(length=128), nullable=True),
+        sa.Column("options", JSON, nullable=True),
     )
 
     print("Creating certificates_dns_providers_fk foreign key")
-    op.create_foreign_key('certificates_dns_providers_fk', 'certificates', 'dns_providers', ['dns_provider_id'], ['id'],
-                          ondelete='cascade')
+    op.create_foreign_key(
+        "certificates_dns_providers_fk",
+        "certificates",
+        "dns_providers",
+        ["dns_provider_id"],
+        ["id"],
+        ondelete="cascade",
+    )
 
     print("Altering column types in the api_keys table")
-    op.alter_column('api_keys', 'issued_at',
-                    existing_type=sa.BIGINT(),
-                    nullable=True)
-    op.alter_column('api_keys', 'revoked',
-                    existing_type=sa.BOOLEAN(),
-                    nullable=True)
-    op.alter_column('api_keys', 'ttl',
-                    existing_type=sa.BIGINT(),
-                    nullable=True)
-    op.alter_column('api_keys', 'user_id',
-                    existing_type=sa.INTEGER(),
-                    nullable=True)
+    op.alter_column("api_keys", "issued_at", existing_type=sa.BIGINT(), nullable=True)
+    op.alter_column("api_keys", "revoked", existing_type=sa.BOOLEAN(), nullable=True)
+    op.alter_column("api_keys", "ttl", existing_type=sa.BIGINT(), nullable=True)
+    op.alter_column("api_keys", "user_id", existing_type=sa.INTEGER(), nullable=True)
 
     print("Creating dns_providers_id foreign key on pending_certs table")
-    op.create_foreign_key(None, 'pending_certs', 'dns_providers', ['dns_provider_id'], ['id'], ondelete='CASCADE')
+    op.create_foreign_key(
+        None,
+        "pending_certs",
+        "dns_providers",
+        ["dns_provider_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+
 
 def downgrade():
     print("Removing dns_providers_id foreign key on pending_certs table")
-    op.drop_constraint(None, 'pending_certs', type_='foreignkey')
+    op.drop_constraint(None, "pending_certs", type_="foreignkey")
     print("Reverting column types in the api_keys table")
-    op.alter_column('api_keys', 'user_id',
-                    existing_type=sa.INTEGER(),
-                    nullable=False)
-    op.alter_column('api_keys', 'ttl',
-                    existing_type=sa.BIGINT(),
-                    nullable=False)
-    op.alter_column('api_keys', 'revoked',
-                    existing_type=sa.BOOLEAN(),
-                    nullable=False)
-    op.alter_column('api_keys', 'issued_at',
-                    existing_type=sa.BIGINT(),
-                    nullable=False)
+    op.alter_column("api_keys", "user_id", existing_type=sa.INTEGER(), nullable=False)
+    op.alter_column("api_keys", "ttl", existing_type=sa.BIGINT(), nullable=False)
+    op.alter_column("api_keys", "revoked", existing_type=sa.BOOLEAN(), nullable=False)
+    op.alter_column("api_keys", "issued_at", existing_type=sa.BIGINT(), nullable=False)
     print("Reverting certificates_dns_providers_fk foreign key")
-    op.drop_constraint('certificates_dns_providers_fk', 'certificates', type_='foreignkey')
+    op.drop_constraint(
+        "certificates_dns_providers_fk", "certificates", type_="foreignkey"
+    )
 
     print("Dropping pending_dns_authorizations table")
-    op.drop_table('pending_dns_authorizations')
+    op.drop_table("pending_dns_authorizations")
     print("Undoing modifications to pending_certs table")
-    op.drop_column('pending_certs', 'options')
-    op.drop_column('pending_certs', 'dns_provider_id')
+    op.drop_column("pending_certs", "options")
+    op.drop_column("pending_certs", "dns_provider_id")
     print("Undoing modifications to certificates table")
-    op.drop_column('certificates', 'dns_provider_id')
+    op.drop_column("certificates", "dns_provider_id")
 
     print("Deleting dns_providers table")
-    op.drop_table('dns_providers')
+    op.drop_table("dns_providers")

lemur/migrations/versions/412b22cb656a_.py

@@ -7,8 +7,8 @@ Create Date: 2016-05-17 17:37:41.210232
 """
 
 # revision identifiers, used by Alembic.
-revision = '412b22cb656a'
-down_revision = '4c50b903d1ae'
+revision = "412b22cb656a"
+down_revision = "4c50b903d1ae"
 
 from alembic import op
 import sqlalchemy as sa
@@ -17,47 +17,102 @@ from sqlalchemy.sql import text
 
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('roles_authorities',
-    sa.Column('authority_id', sa.Integer(), nullable=True),
-    sa.Column('role_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['authority_id'], ['authorities.id'], ),
-    sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
+    op.create_table(
+        "roles_authorities",
+        sa.Column("authority_id", sa.Integer(), nullable=True),
+        sa.Column("role_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(["authority_id"], ["authorities.id"]),
+        sa.ForeignKeyConstraint(["role_id"], ["roles.id"]),
     )
-    op.create_index('roles_authorities_ix', 'roles_authorities', ['authority_id', 'role_id'], unique=True)
-    op.create_table('roles_certificates',
-    sa.Column('certificate_id', sa.Integer(), nullable=True),
-    sa.Column('role_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ),
-    sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
+    op.create_index(
+        "roles_authorities_ix",
+        "roles_authorities",
+        ["authority_id", "role_id"],
+        unique=True,
+    )
+    op.create_table(
+        "roles_certificates",
+        sa.Column("certificate_id", sa.Integer(), nullable=True),
+        sa.Column("role_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(["certificate_id"], ["certificates.id"]),
+        sa.ForeignKeyConstraint(["role_id"], ["roles.id"]),
+    )
+    op.create_index(
+        "roles_certificates_ix",
+        "roles_certificates",
+        ["certificate_id", "role_id"],
+        unique=True,
+    )
+    op.create_index(
+        "certificate_associations_ix",
+        "certificate_associations",
+        ["domain_id", "certificate_id"],
+        unique=True,
+    )
+    op.create_index(
+        "certificate_destination_associations_ix",
+        "certificate_destination_associations",
+        ["destination_id", "certificate_id"],
+        unique=True,
+    )
+    op.create_index(
+        "certificate_notification_associations_ix",
+        "certificate_notification_associations",
+        ["notification_id", "certificate_id"],
+        unique=True,
+    )
+    op.create_index(
+        "certificate_replacement_associations_ix",
+        "certificate_replacement_associations",
+        ["certificate_id", "certificate_id"],
+        unique=True,
+    )
+    op.create_index(
+        "certificate_source_associations_ix",
+        "certificate_source_associations",
+        ["source_id", "certificate_id"],
+        unique=True,
+    )
+    op.create_index(
+        "roles_users_ix", "roles_users", ["user_id", "role_id"], unique=True
     )
-    op.create_index('roles_certificates_ix', 'roles_certificates', ['certificate_id', 'role_id'], unique=True)
-    op.create_index('certificate_associations_ix', 'certificate_associations', ['domain_id', 'certificate_id'], unique=True)
-    op.create_index('certificate_destination_associations_ix', 'certificate_destination_associations', ['destination_id', 'certificate_id'], unique=True)
-    op.create_index('certificate_notification_associations_ix', 'certificate_notification_associations', ['notification_id', 'certificate_id'], unique=True)
-    op.create_index('certificate_replacement_associations_ix', 'certificate_replacement_associations', ['certificate_id', 'certificate_id'], unique=True)
-    op.create_index('certificate_source_associations_ix', 'certificate_source_associations', ['source_id', 'certificate_id'], unique=True)
-    op.create_index('roles_users_ix', 'roles_users', ['user_id', 'role_id'], unique=True)
 
     ### end Alembic commands ###
 
     # migrate existing authority_id relationship to many_to_many
     conn = op.get_bind()
-    for id, authority_id in conn.execute(text('select id, authority_id from roles where authority_id is not null')):
-        stmt = text('insert into roles_authoritties (role_id, authority_id) values (:role_id, :authority_id)')
+    for id, authority_id in conn.execute(
+        text("select id, authority_id from roles where authority_id is not null")
+    ):
+        stmt = text(
+            "insert into roles_authoritties (role_id, authority_id) values (:role_id, :authority_id)"
+        )
         stmt = stmt.bindparams(role_id=id, authority_id=authority_id)
         op.execute(stmt)
 
 
 def downgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index('roles_users_ix', table_name='roles_users')
-    op.drop_index('certificate_source_associations_ix', table_name='certificate_source_associations')
-    op.drop_index('certificate_replacement_associations_ix', table_name='certificate_replacement_associations')
-    op.drop_index('certificate_notification_associations_ix', table_name='certificate_notification_associations')
-    op.drop_index('certificate_destination_associations_ix', table_name='certificate_destination_associations')
-    op.drop_index('certificate_associations_ix', table_name='certificate_associations')
-    op.drop_index('roles_certificates_ix', table_name='roles_certificates')
-    op.drop_table('roles_certificates')
-    op.drop_index('roles_authorities_ix', table_name='roles_authorities')
-    op.drop_table('roles_authorities')
+    op.drop_index("roles_users_ix", table_name="roles_users")
+    op.drop_index(
+        "certificate_source_associations_ix",
+        table_name="certificate_source_associations",
+    )
+    op.drop_index(
+        "certificate_replacement_associations_ix",
+        table_name="certificate_replacement_associations",
+    )
+    op.drop_index(
+        "certificate_notification_associations_ix",
+        table_name="certificate_notification_associations",
+    )
+    op.drop_index(
+        "certificate_destination_associations_ix",
+        table_name="certificate_destination_associations",
+    )
+    op.drop_index("certificate_associations_ix", table_name="certificate_associations")
+    op.drop_index("roles_certificates_ix", table_name="roles_certificates")
+    op.drop_table("roles_certificates")
+    op.drop_index("roles_authorities_ix", table_name="roles_authorities")
+    op.drop_table("roles_authorities")
     ### end Alembic commands ###

lemur/migrations/versions/449c3d5c7299_.py

@@ -7,8 +7,8 @@ Create Date: 2018-02-24 22:51:35.369229
 """
 
 # revision identifiers, used by Alembic.
-revision = '449c3d5c7299'
-down_revision = '5770674184de'
+revision = "449c3d5c7299"
+down_revision = "5770674184de"
 
 from alembic import op
 from flask_sqlalchemy import SQLAlchemy
@@ -21,6 +21,16 @@ COLUMNS = ["notification_id", "certificate_id"]
 
 
 def upgrade():
+    connection = op.get_bind()
+    # Delete duplicate entries
+    connection.execute(
+        """\
+        DELETE FROM certificate_notification_associations WHERE ctid NOT IN (
+            -- Select the first tuple ID for each (notification_id, certificate_id) combination and keep that
+            SELECT min(ctid) FROM certificate_notification_associations GROUP BY notification_id, certificate_id
+        )
+    """
+    )
     op.create_unique_constraint(CONSTRAINT_NAME, TABLE, COLUMNS)
 
 

lemur/migrations/versions/4c50b903d1ae_.py

@@ -7,20 +7,21 @@ Create Date: 2015-12-30 10:19:30.057791
 """
 
 # revision identifiers, used by Alembic.
-revision = '4c50b903d1ae'
-down_revision = '33de094da890'
+revision = "4c50b903d1ae"
+down_revision = "33de094da890"
 
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+
 def upgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('domains', sa.Column('sensitive', sa.Boolean(), nullable=True))
+    op.add_column("domains", sa.Column("sensitive", sa.Boolean(), nullable=True))
     ### end Alembic commands ###
 
 
 def downgrade():
     ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('domains', 'sensitive')
+    op.drop_column("domains", "sensitive")
     ### end Alembic commands ###

lemur/migrations/versions/556ceb3e3c3e_.py

@@ -7,8 +7,8 @@ Create Date: 2018-01-05 01:18:45.571595
 """
 
 # revision identifiers, used by Alembic.
-revision = '556ceb3e3c3e'
-down_revision = '449c3d5c7299'
+revision = "556ceb3e3c3e"
+down_revision = "449c3d5c7299"
 
 from alembic import op
 import sqlalchemy as sa
@@ -16,84 +16,150 @@ from lemur.utils import Vault
 from sqlalchemy.dialects import postgresql
 from sqlalchemy_utils import ArrowType
 
+
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('pending_certs',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('external_id', sa.String(length=128), nullable=True),
-    sa.Column('owner', sa.String(length=128), nullable=False),
-    sa.Column('name', sa.String(length=256), nullable=True),
-    sa.Column('description', sa.String(length=1024), nullable=True),
-    sa.Column('notify', sa.Boolean(), nullable=True),
-    sa.Column('number_attempts', sa.Integer(), nullable=True),
-    sa.Column('rename', sa.Boolean(), nullable=True),
-    sa.Column('cn', sa.String(length=128), nullable=True),
-    sa.Column('csr', sa.Text(), nullable=False),
-    sa.Column('chain', sa.Text(), nullable=True),
-    sa.Column('private_key', Vault(), nullable=True),
-    sa.Column('date_created', ArrowType(), server_default=sa.text('now()'), nullable=False),
-    sa.Column('status', sa.String(length=128), nullable=True),
-    sa.Column('rotation', sa.Boolean(), nullable=True),
-    sa.Column('user_id', sa.Integer(), nullable=True),
-    sa.Column('authority_id', sa.Integer(), nullable=True),
-    sa.Column('root_authority_id', sa.Integer(), nullable=True),
-    sa.Column('rotation_policy_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['authority_id'], ['authorities.id'], ondelete='CASCADE'),
-    sa.ForeignKeyConstraint(['root_authority_id'], ['authorities.id'], ondelete='CASCADE'),
-    sa.ForeignKeyConstraint(['rotation_policy_id'], ['rotation_policies.id'], ),
-    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('name')
+    op.create_table(
+        "pending_certs",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("external_id", sa.String(length=128), nullable=True),
+        sa.Column("owner", sa.String(length=128), nullable=False),
+        sa.Column("name", sa.String(length=256), nullable=True),
+        sa.Column("description", sa.String(length=1024), nullable=True),
+        sa.Column("notify", sa.Boolean(), nullable=True),
+        sa.Column("number_attempts", sa.Integer(), nullable=True),
+        sa.Column("rename", sa.Boolean(), nullable=True),
+        sa.Column("cn", sa.String(length=128), nullable=True),
+        sa.Column("csr", sa.Text(), nullable=False),
+        sa.Column("chain", sa.Text(), nullable=True),
+        sa.Column("private_key", Vault(), nullable=True),
+        sa.Column(
+            "date_created", ArrowType(), server_default=sa.text("now()"), nullable=False
+        ),
+        sa.Column("status", sa.String(length=128), nullable=True),
+        sa.Column("rotation", sa.Boolean(), nullable=True),
+        sa.Column("user_id", sa.Integer(), nullable=True),
+        sa.Column("authority_id", sa.Integer(), nullable=True),
+        sa.Column("root_authority_id", sa.Integer(), nullable=True),
+        sa.Column("rotation_policy_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["authority_id"], ["authorities.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["root_authority_id"], ["authorities.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(["rotation_policy_id"], ["rotation_policies.id"]),
+        sa.ForeignKeyConstraint(["user_id"], ["users.id"]),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
     )
-    op.create_table('pending_cert_destination_associations',
-    sa.Column('destination_id', sa.Integer(), nullable=True),
-    sa.Column('pending_cert_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['destination_id'], ['destinations.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade')
+    op.create_table(
+        "pending_cert_destination_associations",
+        sa.Column("destination_id", sa.Integer(), nullable=True),
+        sa.Column("pending_cert_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["destination_id"], ["destinations.id"], ondelete="cascade"
+        ),
+        sa.ForeignKeyConstraint(
+            ["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
+        ),
     )
-    op.create_index('pending_cert_destination_associations_ix', 'pending_cert_destination_associations', ['destination_id', 'pending_cert_id'], unique=False)
-    op.create_table('pending_cert_notification_associations',
-    sa.Column('notification_id', sa.Integer(), nullable=True),
-    sa.Column('pending_cert_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['notification_id'], ['notifications.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade')
+    op.create_index(
+        "pending_cert_destination_associations_ix",
+        "pending_cert_destination_associations",
+        ["destination_id", "pending_cert_id"],
+        unique=False,
     )
-    op.create_index('pending_cert_notification_associations_ix', 'pending_cert_notification_associations', ['notification_id', 'pending_cert_id'], unique=False)
-    op.create_table('pending_cert_replacement_associations',
-    sa.Column('replaced_certificate_id', sa.Integer(), nullable=True),
-    sa.Column('pending_cert_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['replaced_certificate_id'], ['certificates.id'], ondelete='cascade')
+    op.create_table(
+        "pending_cert_notification_associations",
+        sa.Column("notification_id", sa.Integer(), nullable=True),
+        sa.Column("pending_cert_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["notification_id"], ["notifications.id"], ondelete="cascade"
+        ),
+        sa.ForeignKeyConstraint(
+            ["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
+        ),
     )
-    op.create_index('pending_cert_replacement_associations_ix', 'pending_cert_replacement_associations', ['replaced_certificate_id', 'pending_cert_id'], unique=False)
-    op.create_table('pending_cert_role_associations',
-    sa.Column('pending_cert_id', sa.Integer(), nullable=True),
-    sa.Column('role_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ),
-    sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
+    op.create_index(
+        "pending_cert_notification_associations_ix",
+        "pending_cert_notification_associations",
+        ["notification_id", "pending_cert_id"],
+        unique=False,
     )
-    op.create_index('pending_cert_role_associations_ix', 'pending_cert_role_associations', ['pending_cert_id', 'role_id'], unique=False)
-    op.create_table('pending_cert_source_associations',
-    sa.Column('source_id', sa.Integer(), nullable=True),
-    sa.Column('pending_cert_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['pending_cert_id'], ['pending_certs.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='cascade')
+    op.create_table(
+        "pending_cert_replacement_associations",
+        sa.Column("replaced_certificate_id", sa.Integer(), nullable=True),
+        sa.Column("pending_cert_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
+        ),
+        sa.ForeignKeyConstraint(
+            ["replaced_certificate_id"], ["certificates.id"], ondelete="cascade"
+        ),
+    )
+    op.create_index(
+        "pending_cert_replacement_associations_ix",
+        "pending_cert_replacement_associations",
+        ["replaced_certificate_id", "pending_cert_id"],
+        unique=False,
+    )
+    op.create_table(
+        "pending_cert_role_associations",
+        sa.Column("pending_cert_id", sa.Integer(), nullable=True),
+        sa.Column("role_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(["pending_cert_id"], ["pending_certs.id"]),
+        sa.ForeignKeyConstraint(["role_id"], ["roles.id"]),
+    )
+    op.create_index(
+        "pending_cert_role_associations_ix",
+        "pending_cert_role_associations",
+        ["pending_cert_id", "role_id"],
+        unique=False,
+    )
+    op.create_table(
+        "pending_cert_source_associations",
+        sa.Column("source_id", sa.Integer(), nullable=True),
+        sa.Column("pending_cert_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["pending_cert_id"], ["pending_certs.id"], ondelete="cascade"
+        ),
+        sa.ForeignKeyConstraint(["source_id"], ["sources.id"], ondelete="cascade"),
+    )
+    op.create_index(
+        "pending_cert_source_associations_ix",
+        "pending_cert_source_associations",
+        ["source_id", "pending_cert_id"],
+        unique=False,
     )
-    op.create_index('pending_cert_source_associations_ix', 'pending_cert_source_associations', ['source_id', 'pending_cert_id'], unique=False)
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_index('pending_cert_source_associations_ix', table_name='pending_cert_source_associations')
-    op.drop_table('pending_cert_source_associations')
-    op.drop_index('pending_cert_role_associations_ix', table_name='pending_cert_role_associations')
-    op.drop_table('pending_cert_role_associations')
-    op.drop_index('pending_cert_replacement_associations_ix', table_name='pending_cert_replacement_associations')
-    op.drop_table('pending_cert_replacement_associations')
-    op.drop_index('pending_cert_notification_associations_ix', table_name='pending_cert_notification_associations')
-    op.drop_table('pending_cert_notification_associations')
-    op.drop_index('pending_cert_destination_associations_ix', table_name='pending_cert_destination_associations')
-    op.drop_table('pending_cert_destination_associations')
-    op.drop_table('pending_certs')
+    op.drop_index(
+        "pending_cert_source_associations_ix",
+        table_name="pending_cert_source_associations",
+    )
+    op.drop_table("pending_cert_source_associations")
+    op.drop_index(
+        "pending_cert_role_associations_ix", table_name="pending_cert_role_associations"
+    )
+    op.drop_table("pending_cert_role_associations")
+    op.drop_index(
+        "pending_cert_replacement_associations_ix",
+        table_name="pending_cert_replacement_associations",
+    )
+    op.drop_table("pending_cert_replacement_associations")
+    op.drop_index(
+        "pending_cert_notification_associations_ix",
+        table_name="pending_cert_notification_associations",
+    )
+    op.drop_table("pending_cert_notification_associations")
+    op.drop_index(
+        "pending_cert_destination_associations_ix",
+        table_name="pending_cert_destination_associations",
+    )
+    op.drop_table("pending_cert_destination_associations")
+    op.drop_table("pending_certs")
     # ### end Alembic commands ###

lemur/migrations/versions/5770674184de_.py

@@ -7,8 +7,8 @@ Create Date: 2018-02-23 15:27:30.335435
 """
 
 # revision identifiers, used by Alembic.
-revision = '5770674184de'
-down_revision = 'ce547319f7be'
+revision = "5770674184de"
+down_revision = "ce547319f7be"
 
 from flask_sqlalchemy import SQLAlchemy
 from lemur.models import certificate_notification_associations
@@ -32,7 +32,9 @@ def upgrade():
         # If we've seen a pair already, delete the duplicates
         if seen.get("{}-{}".format(x.certificate_id, x.notification_id)):
             print("Deleting duplicate: {}".format(x))
-            d = session.query(certificate_notification_associations).filter(certificate_notification_associations.c.id==x.id)
+            d = session.query(certificate_notification_associations).filter(
+                certificate_notification_associations.c.id == x.id
+            )
             d.delete(synchronize_session=False)
         seen["{}-{}".format(x.certificate_id, x.notification_id)] = True
     db.session.commit()

lemur/migrations/versions/5ae0ecefb01f_.py

@@ -7,8 +7,8 @@ Create Date: 2018-08-14 08:16:43.329316
 """
 
 # revision identifiers, used by Alembic.
-revision = '5ae0ecefb01f'
-down_revision = '1db4f82bc780'
+revision = "5ae0ecefb01f"
+down_revision = "1db4f82bc780"
 
 from alembic import op
 import sqlalchemy as sa
@@ -16,17 +16,14 @@ import sqlalchemy as sa
 
 def upgrade():
     op.alter_column(
-        table_name='pending_certs',
-        column_name='status',
-        nullable=True,
-        type_=sa.TEXT()
+        table_name="pending_certs", column_name="status", nullable=True, type_=sa.TEXT()
     )
 
 
 def downgrade():
     op.alter_column(
-        table_name='pending_certs',
-        column_name='status',
+        table_name="pending_certs",
+        column_name="status",
         nullable=True,
-        type_=sa.VARCHAR(128)
+        type_=sa.VARCHAR(128),
     )

lemur/migrations/versions/5bc47fa7cac4_.py

@@ -7,16 +7,18 @@ Create Date: 2017-12-08 14:19:11.903864
 """
 
 # revision identifiers, used by Alembic.
-revision = '5bc47fa7cac4'
-down_revision = 'c05a8998b371'
+revision = "5bc47fa7cac4"
+down_revision = "c05a8998b371"
 
 from alembic import op
 import sqlalchemy as sa
 
 
 def upgrade():
-    op.add_column('roles', sa.Column('third_party', sa.Boolean(), nullable=True, default=False))
+    op.add_column(
+        "roles", sa.Column("third_party", sa.Boolean(), nullable=True, default=False)
+    )
 
 
 def downgrade():
-    op.drop_column('roles', 'third_party')
+    op.drop_column("roles", "third_party")

lemur/migrations/versions/5e680529b666_.py

@@ -7,20 +7,20 @@ Create Date: 2017-01-26 05:05:25.168125
 """
 
 # revision identifiers, used by Alembic.
-revision = '5e680529b666'
-down_revision = '131ec6accff5'
+revision = "5e680529b666"
+down_revision = "131ec6accff5"
 
 from alembic import op
 import sqlalchemy as sa
 
 
 def upgrade():
-    op.add_column('endpoints', sa.Column('sensitive', sa.Boolean(), nullable=True))
-    op.add_column('endpoints', sa.Column('source_id', sa.Integer(), nullable=True))
-    op.create_foreign_key(None, 'endpoints', 'sources', ['source_id'], ['id'])
+    op.add_column("endpoints", sa.Column("sensitive", sa.Boolean(), nullable=True))
+    op.add_column("endpoints", sa.Column("source_id", sa.Integer(), nullable=True))
+    op.create_foreign_key(None, "endpoints", "sources", ["source_id"], ["id"])
 
 
 def downgrade():
-    op.drop_constraint(None, 'endpoints', type_='foreignkey')
-    op.drop_column('endpoints', 'source_id')
-    op.drop_column('endpoints', 'sensitive')
+    op.drop_constraint(None, "endpoints", type_="foreignkey")
+    op.drop_column("endpoints", "source_id")
+    op.drop_column("endpoints", "sensitive")