Merge pull request #3471 from hosseinsh/aut0mate-bump-version

Automate bump version

.github/workflows/lemur-publish-release-pypi.yml

@@ -0,0 +1,41 @@
+# This workflow will upload a Python Package using Twine when a Lemur release is created via github
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Publish Lemur's latest package to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Autobump version
+      run: |
+        # from refs/tags/v0.8.1 get 0.8.1
+        VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+        PLACEHOLDER='__version__ = "develop"'
+        VERSION_FILE='lemur/__about__.py'
+        # in case placeholder is missing, exists with code 1 and github actions aborts the build
+        grep "$PLACEHOLDER" "$VERSION_FILE"
+        sed -i "s/$PLACEHOLDER/__version__ = \"${VERSION}\"/g" "$VERSION_FILE"
+      shell: bash
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.LEMUR_PYPI_API_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.LEMUR_PYPI_API_TOKEN }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.gitignore

@@ -10,6 +10,7 @@
 *.db
 *.pid
 *.enc
+*.env
 MANIFEST
 test.conf
 pip-log.txt
@@ -25,10 +26,17 @@ package-lock.json
 /lemur/static/dist/
 /lemur/static/app/vendor/
 /wheelhouse
+/lemur/lib
+/lemur/bin
+/lemur/lib64
+/lemur/include
+
 docs/_build
 .editorconfig
 .idea
+.pytest_cache
 lemur/tests/tmp
 
 /lemur/plugins/lemur_email/tests/expiration-rendered.html
 /lemur/plugins/lemur_email/tests/rotation-rendered.html
+.celerybeat-schedule

.pre-commit-config.yaml

@@ -8,3 +8,17 @@
     sha: v2.9.5
     hooks:
     -   id: jshint
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.7
+
+-   repo: local
+    hooks:
+    -   id: python-bandit-vulnerability-check
+        name: bandit
+        entry: bandit
+        args: ['--ini', 'tox.ini', '-r', 'consoleme']
+        language: system
+        pass_filenames: false

.readthedocs.yml

@@ -0,0 +1,23 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/conf.py
+   fail_on_warning: true
+
+# Build docs in all formats (html, pdf, epub)
+formats: all
+
+# Set the version of Python and requirements required to build the docs
+python:
+   version: 3.7
+   install:
+   - requirements: requirements-docs.txt
+   - method: setuptools
+     path: .
+   system_packages: true

.travis.yml

@@ -1,17 +1,50 @@
-language: python
-sudo: required
-dist: trusty
-
 node_js:
-  - "6.2.0"
+  - "10"
 
+jobs:
+  include:
+    - name: "python3.7-postgresql-9.4-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
+      env: TOXENV=py37
       addons:
         postgresql: "9.4"
-
-matrix:
-  include:
-    - python: "3.5"
-      env: TOXENV=py35
+        chrome: stable
+      services:
+        - xvfb
+    - name: "python3.7-postgresql-10-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
+      env: TOXENV=py37
+      addons:
+        postgresql: '10'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-10
+            - postgresql-client-10
+            - postgresql-server-dev-10
+      services:
+        - postgresql
+        - xvfb
+    - name: "python3.8-postgresql-12-focal"
+      dist: focal
+      language: python
+      python: "3.8"
+      env: TOXENV=py38
+      addons:
+        postgresql: '12'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-12
+            - postgresql-client-12
+            - postgresql-server-dev-12
+      services:
+        - postgresql
+        - xvfb
 
 cache:
   directories:
@@ -21,16 +54,29 @@ cache:
 env:
   global:
     - PIP_DOWNLOAD_CACHE=".pip_download_cache"
+    # The following line is a temporary workaround for this issue: https://github.com/pypa/setuptools/issues/2230
+    - SETUPTOOLS_USE_DISTUTILS=stdlib
     # do not load /etc/boto.cfg with Python 3 incompatible plugin
     # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
     - BOTO_CONFIG=/doesnotexist
 
+before_install:
+  - export CHROME_BIN=/usr/bin/google-chrome
+
 before_script:
+  - sudo systemctl stop postgresql
+  # the port may have been auto-configured to use 5433 if it thought 5422 was already in use,
+  # for some reason it happens very often
+  # https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_setup_postgresql.bash#L52
+  - sudo sed -i -e 's/5433/5432/' /etc/postgresql/*/main/postgresql.conf
+  - sudo systemctl restart postgresql
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
+  - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
   - npm config set registry https://registry.npmjs.org
-  - npm install -g bower
+  - npm install -g npm@latest bower
   - pip install --upgrade setuptools
+  - export DISPLAY=:99.0
 
 install:
   - pip install coveralls
@@ -39,10 +85,15 @@ install:
 script:
   - make test
   - bandit -r . -ll -ii -x lemur/tests/,docs
+  - make test-js
 
 after_success:
   - coveralls
 
 notifications:
   email:
-    kglisson@netflix.com
+    recipients:
+      - lemur@netflix.com
+    on_success: never
+    on_failure: always
+    on_cancel: never # Dependbot cancels Travis before rebase and triggers too many emails

CHANGELOG.rst

@@ -1,9 +1,121 @@
 Changelog
 =========
 
+0.8.1 - `2021-03-12`
+~~~~~~~~~~~~~~~~~~~~
+
+This release includes improvements on many fronts, such as:
+
+- Notifications:
+    - Enhanced SNS flow
+    - Expiration Summary
+    - CA expiration email
+- EC algorithm as the default
+- Improved revocation flow
+- Localized AWS STS option
+- Improved Lemur doc building
+- ACME:
+    - reduced failed attempts to 3x trials
+    - support for selecting the chain (Let's Encrypt X1 transition)
+    - revocation
+    - http01 documentation
+- Entrust:
+    - Support for cross-signed intermediate CA
+- Revised disclosure process
+- Dependency updates and conflict resolutions
+
+Special thanks to all who contributed to this release, notably:
+
+- `peschmae  <https://github.com/peschmae>`_
+- `atugushev  <https://github.com/atugushev>`_
+- `sirferl   <https://github.com/sirferl>`_
+
+
+
+0.8.0 - `2020-11-13`
+~~~~~~~~~~~~~~~~~~~~
+
+This release comes after more than two years and contains many interesting new features and improvements.
+In addition to multiple new plugins, such as ACME-http01, ADCS, PowerDNS, UltraDNS, Entrust, SNS, many of Lemur's existing
+flows have improved.
+
+In the future, we plan to do frequent releases.
+
+
+Summary of notable changes:
+
+- AWS S3 plugin: added delete, get methods, and support for uploading/deleting acme tokens
+- ACME plugin:
+    - revamp of the plugin
+    - support for http01 domain validation, via S3 and SFTP as destination for the acme token
+    - support for CNAME delegated domain validation
+    - store-acme-account-details
+- PowerDNS plugin
+- UltraDNS plugin
+- ADCS plugin
+- SNS plugin
+- Entrust plugin
+- Rotation:
+    - respecting keyType and extensions
+    - region-by-region rotation option
+    - default to auto-rotate when cert attached to endpoint
+    - default to 1y validity during rotation for multi-year browser-trusted certs
+- Certificate: search_by_name, and important performance improvements
+- UI
+    - reducing the EC curve options to the relevant ones
+    - edit option for notifications, destinations and sources
+    - showing 13 month validity as default
+    - option to hide certs expired since 3month
+    - faster Permalink (no search involved)
+    - commonName Auto Added as DNS in the UI
+    - improved search and cert lookup
+- celery tasks instead of crone, for better logging and monitoring
+- countless bugfixes
+    - group-lookup-fix-referral
+    - url_context_path
+    - duplicate notification
+    - digicert-time-bug-fix
+    - improved-csr-support
+    - fix-cryptography-intermediate-ca
+    - enhanced logging
+    - vault-k8s-auth
+    - cfssl-key-fix
+    - cert-sync-endpoint-find-by-hash
+    - nlb-naming-bug
+    - fix_vault_api_v2_append
+    - aid_openid_roles_provider_integration
+    - rewrite-java-keystore-use-pyjks
+    - vault_kv2
+
+
+To see the full list of changes, you can run
+
+    $ git log --merges --first-parent master         --pretty=format:"%h %<(10,trunc)%aN %C(white)%<(15)%ar%Creset %C(red bold)%<(15)%D%Creset %s" | grep -v "depend"
+
+
+Special thanks to all who contributed to this release, notably:
+
+- `peschmae  <https://github.com/peschmae>`_
+- `sirferl   <https://github.com/sirferl>`_
+- `lukasmrtvy  <https://github.com/lukasmrtvy>`_
+- `intgr  <https://github.com/intgr>`_
+- `kush-bavishi  <https://github.com/kush-bavishi>`_
+- `alwaysjolley  <https://github.com/alwaysjolley>`_
+- `jplana <https://github.com/jplana>`_
+- `explody <https://github.com/explody>`_
+- `titouanc <https://github.com/titouanc>`_
+- `jramosf <https://github.com/jramosf>`_
+
+
+Upgrading
+---------
+
+.. note:: This release will need a migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
 
 0.7 - `2018-05-07`
-~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality.
 The linux_dst plugin will also be deprecated and removed.
@@ -40,8 +152,7 @@ Happy Holidays! This is a big release with lots of bug fixes and features. Below
 
 Features:
 
-* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates.
-is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used.
+* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used.
 * Adds per-user API Keys, allows users to issue multiple long-lived API tokens with the same permission as the user creating them.
 * Adds the ability to revoke certificates from the Lemur UI/API, this is currently only supported for the digicert CIS and cfssl plugins.
 * Allow destinations to support an export function. Useful for file system destinations e.g. S3 to specify the export plugin you wish to run before being sent to the destination.
@@ -85,13 +196,9 @@ Big thanks to neilschelly for quite a lot of improvements to the `lemur-cryptogr
 
 Other Highlights:
 
-* Closed `#501 <https://github.com/Netflix/lemur/issues/501>`_ - Endpoint resource as now kept in sync via an
-expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never
-removed from Lemur.
-* Closed `#551 <https://github.com/Netflix/lemur/pull/551>`_ - Added the ability to create a 4096 bit key during certificate
-creation. Closed `#528 <https://github.com/Netflix/lemur/pull/528>`_ to ensure that issuer plugins supported the new 4096 bit keys.
-* Closed `#566 <https://github.com/Netflix/lemur/issues/566>`_ - Fixed an issue changing the notification status for  certificates
-without private keys.
+* Closed `#501 <https://github.com/Netflix/lemur/issues/501>`_ - Endpoint resource as now kept in sync via an expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never removed from Lemur.
+* Closed `#551 <https://github.com/Netflix/lemur/pull/551>`_ - Added the ability to create a 4096 bit key during certificate creation. Closed `#528 <https://github.com/Netflix/lemur/pull/528>`_ to ensure that issuer plugins supported the new 4096 bit keys.
+* Closed `#566 <https://github.com/Netflix/lemur/issues/566>`_ - Fixed an issue changing the notification status for  certificates without private keys.
 * Closed `#594 <https://github.com/Netflix/lemur/issues/594>`_ - Added `replaced` field indicating if a certificate has been superseded.
 * Closed `#602 <https://github.com/Netflix/lemur/issues/602>`_ - AWS plugin added support for ALBs for endpoint tracking.
 
@@ -115,12 +222,8 @@ Upgrading
 
 There have been quite a few issues closed in this release. Some notables:
 
-* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated
-AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for
-future enhancements of Lemur's certificate 'deployment' capabilities.
-
-* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability
-to restrict certificate expiration dates to weekdays.
+* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for future enhancements of Lemur's certificate 'deployment' capabilities.
+* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability to restrict certificate expiration dates to weekdays.
 
 Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!)
 
@@ -175,7 +278,7 @@ these keys should be fairly trivial, additionally pull requests have been submit
     should be easier to determine what authorities are available and when an authority has actually been selected.
 * Closed `#254 <https://github.com/Netflix/lemur/issues/254>`_ - Forces certificate names to be generally unique. If a certificate name
     (generated or otherwise) is found to be a duplicate we increment by appending a counter.
-* Closed `#254 <https://github.com/Netflix/lemur/issues/275>`_ - Switched to using Fernet generated passphrases for exported items.
+* Closed `#275 <https://github.com/Netflix/lemur/issues/275>`_ - Switched to using Fernet generated passphrases for exported items.
     These are more sounds that pseudo random passphrases generated before and have the nice property of being in base64.
 * Closed `#278 <https://github.com/Netflix/lemur/issues/278>`_ - Added ability to specify a custom name to certificate creation, previously
     this was only available in the certificate import wizard.

Dockerfile

@@ -1,9 +1,10 @@
-FROM python:3.5
+FROM python:3.7
 RUN apt-get update
-RUN apt-get install -y make python-software-properties curl
+RUN apt-get install -y make software-properties-common curl
 RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
 RUN apt-get update
-RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN pip install pip==20.0.2
 RUN pip install -U setuptools
 RUN pip install coveralls bandit
 WORKDIR /app

MANIFEST.in

@@ -1,4 +1,4 @@
-include setup.py version.py package.json bower.json gulpfile.js README.rst MANIFEST.in LICENSE AUTHORS
+include setup.py version.py package.json bower.json gulpfile.js README.rst MANIFEST.in LICENSE AUTHORS requirements*.txt
 recursive-include lemur/plugins/lemur_email/templates *
 recursive-include lemur/static *
 global-exclude *~

Makefile

@@ -36,20 +36,24 @@ endif
 	@echo ""
 
 dev-docs:
-	pip install -r docs/requirements.txt
+	pip install -r requirements-docs.txt
 
 reset-db:
 	@echo "--> Dropping existing 'lemur' database"
 	dropdb lemur || true
 	@echo "--> Creating 'lemur' database"
 	createdb -E utf-8 lemur
+	@echo "--> Enabling pg_trgm extension"
+	psql lemur -c "create extension IF NOT EXISTS pg_trgm;"
 	@echo "--> Applying migrations"
-	lemur db upgrade
+	cd lemur && lemur db upgrade
 
 setup-git:
 	@echo "--> Installing git hooks"
-	git config branch.autosetuprebase always
-	cd .git/hooks && ln -sf ../../hooks/* ./
+	if [ -d .git/hooks ]; then \
+		git config branch.autosetuprebase always; \
+		cd .git/hooks && ln -sf ../../hooks/* ./; \
+	fi
 	@echo ""
 
 clean:
@@ -111,10 +115,10 @@ endif
 	@echo "--> Updating Python requirements"
 	pip install --upgrade pip
 	pip install --upgrade pip-tools
-	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
-	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
-	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
-	pip-compile --output-file requirements.txt requirements.in -U --no-index
+	pip-compile --output-file requirements.txt requirements.in -U --no-emit-index-url
+	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-emit-index-url
+	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-emit-index-url
+	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-emit-index-url
 	@echo "--> Done updating Python requirements"
 	@echo "--> Removing python-ldap from requirements-docs.txt"
 	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt
@@ -123,5 +127,9 @@ endif
 	@echo "--> Done installing new dependencies"
 	@echo ""
 
+# Execute with make checkout-pr pr=<pr number>
+checkout-pr:
+	git fetch upstream pull/$(pr)/head:pr-$(pr)
+
 
 .PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish release

README.rst

@@ -22,7 +22,7 @@ Lemur
 Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs
 and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults.
 
-It works on CPython 3.5. We deploy on Ubuntu and develop on OS X.
+It works on Python 3.7. We deploy on Ubuntu and develop on OS X.
 
 
 Project resources

bower.json

@@ -11,12 +11,12 @@
     "angular": "1.4.9",
     "json3": "~3.3",
     "es5-shim": "~4.5.0",
-    "bootstrap": "~3.3.6",
     "angular-bootstrap": "~1.1.1",
     "angular-animate": "~1.4.9",
     "restangular": "~1.5.1",
     "ng-table": "~0.8.3",
     "moment": "~2.11.1",
+    "bootstrap": "~3.4.1",
     "angular-loading-bar": "~0.8.0",
     "angular-moment": "~0.10.3",
     "moment-range": "~2.1.0",
@@ -24,12 +24,12 @@
     "angularjs-toaster": "~1.0.0",
     "angular-chart.js": "~0.8.8",
     "ngletteravatar": "~4.0.0",
-    "bootswatch": "~3.3.6",
+    "bootswatch": "3.4.1+1",
     "fontawesome": "~4.5.0",
     "satellizer": "~0.13.4",
     "angular-ui-router": "~0.2.15",
     "font-awesome": "~4.5.0",
-    "lodash": "~4.0.1",
+    "lodash": "~4.17.20",
     "underscore": "~1.8.3",
     "angular-smart-table": "2.1.8",
     "angular-strap": ">= 2.2.2",

docker-compose.yml

@@ -13,7 +13,15 @@ services:
       VIRTUAL_ENV: 'true'
 
   postgres:
-    image: postgres:9.4
+    image: postgres
+    restart: always
     environment:
       POSTGRES_USER: lemur
       POSTGRES_PASSWORD: lemur
+    ports:
+      - "5432:5432"
+
+  redis:
+    image: "redis:alpine"
+    ports:
+      - "6379:6379"

docker/.dockerignore

@@ -0,0 +1,3 @@
+*-env
+docker-compose.yml
+Dockerfile

docker/Dockerfile

@@ -0,0 +1,69 @@
+FROM python:3.7.9-alpine3.12
+
+ARG VERSION
+ENV VERSION master
+
+ARG URLCONTEXT
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk add --no-cache --update python3 py-pip libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                libtool \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    mkdir -p /opt/lemur /home/lemur/.lemur/ && \
+    curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
+    chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
+
+WORKDIR /opt/lemur
+
+RUN echo "Running with python:" && python -c 'import platform; print(platform.python_version())' && \
+    echo "Running with nodejs:" && node -v && \
+    npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
+    apk del build-dependencies
+
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/Dockerfile-src

@@ -0,0 +1,67 @@
+FROM alpine:3.8
+
+ARG VERSION
+ENV VERSION master
+
+ARG URLCONTEXT
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                libtool \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    mkdir -p /home/lemur/.lemur/ && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/
+
+COPY ./ /opt/lemur    
+WORKDIR /opt/lemur
+
+RUN chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \
+    npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
+    apk del build-dependencies
+
+COPY docker/entrypoint /
+COPY docker/src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY docker/supervisor.conf /
+COPY docker/nginx/default.conf /etc/nginx/conf.d/
+COPY docker/nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '3'
+
+volumes:
+  pg_data: { }
+
+services:
+  postgres:
+    image: "postgres:13.1-alpine"
+    restart: on-failure
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    env_file:
+      - pgsql-env
+
+  lemur:
+    # image: "netlix-lemur:latest"
+    restart: on-failure
+    build:
+      context: .
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - lemur-env
+      - pgsql-env
+    ports:
+      - 87:80
+      - 447:443
+
+  redis:
+    image: "redis:alpine3.12"
+    restart: on-failure

docker/entrypoint

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
+  echo "Database vars not set"
+  exit 1
+fi
+
+export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
+
+export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}"
+
+export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
+
+
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'select 1;'
+
+echo " # Create Postgres trgm extension"
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
+echo " # Done"
+
+if [ -z "${SKIP_SSL}" ]; then
+  if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
+    openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
+  fi
+  [ -f "/etc/nginx/conf.d/default-ssl.conf.a" ] && mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  [ -f "/etc/nginx/conf.d/default.conf" ] && mv -f /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+fi
+
+# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
+# echo "Creating config"
+# https://github.com/Netflix/lemur/issues/2257
+# python3 /opt/lemur/lemur/manage.py create_config
+# echo "Done"
+# fi
+
+echo " # Running init"
+su lemur -s /bin/bash -c "cd /opt/lemur/lemur; lemur init -p ${LEMUR_ADMIN_PASSWORD}"
+echo " # Done"
+
+# echo "Creating user"
+# https://github.com/Netflix/lemur/issues/
+# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
+# echo "Done"
+
+cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
+cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
+cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
+cron_reissue="${CRON_REISSUE:-"0 23 * * *"}"
+
+echo " # Populating crontab"
+echo "${cron_notify} lemur notify expirations" > /etc/crontabs/lemur
+echo "${cron_sync} lemur source sync -s all" >> /etc/crontabs/lemur
+echo "${cron_revoked} lemur certificate check_revoked" >> /etc/crontabs/lemur
+echo "${cron_reissue} lemur certificate reissue -c" >> /etc/crontabs/lemur
+echo " # Done"
+
+exec "$@"

docker/lemur-env

@@ -0,0 +1,25 @@
+# SKIP_SSL=1
+# LEMUR_TOKEN_SECRET=
+# LEMUR_DEFAULT_COUNTRY=
+# LEMUR_DEFAULT_STATE=
+# LEMUR_DEFAULT_LOCATION=
+# LEMUR_DEFAULT_ORGANIZATION=
+# LEMUR_DEFAULT_ORGANIZATIONAL_UNIT=
+# LEMUR_DEFAULT_ISSUER_PLUGIN=cryptography-issuer
+# LEMUR_DEFAULT_AUTHORITY=cryptography
+# MAIL_SERVER=mail.example.com
+# MAIL_PORT=25
+# LEMUR_EMAIL=lemur@example.com
+# LEMUR_SECURITY_TEAM_EMAIL=['team@example.com']
+# LEMUR_TOKEN_SECRET=
+# LEMUR_ENCRYPTION_KEYS=['']
+# DEBUG=True
+# LDAP_DEBUG=True
+# LDAP_AUTH=True
+# LDAP_BIND_URI=ldap://example.com
+# LDAP_BASE_DN=DC=example,DC=com
+# LDAP_EMAIL_DOMAIN=example.com
+# LDAP_USE_TLS=False
+# LDAP_REQUIRED_GROUP=certificate-management-admins
+# LDAP_GROUPS_TO_ROLES={'certificate-management-admins': 'admin', 'Team': 'team@example.com'}
+# LDAP_IS_ACTIVE_DIRECTORY=False

docker/nginx/default-ssl.conf

@@ -0,0 +1,37 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+    listen 80;
+    server_name  _;
+    return 301 https://$host$request_uri;
+}
+
+server {
+   listen 443 ssl;
+   server_name  _;
+   access_log /dev/stdout;
+   error_log /dev/stderr;
+   ssl_certificate /etc/nginx/ssl/server.crt;
+   ssl_certificate_key /etc/nginx/ssl/server.key;
+   ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+   ssl_ciphers HIGH:!aNULL:!MD5;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/nginx/default.conf

@@ -0,0 +1,26 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+   listen      80;
+   access_log  /dev/stdout;
+   error_log   /dev/stderr;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/pgsql-env

@@ -0,0 +1,4 @@
+POSTGRES_USER=lemur
+POSTGRES_PASSWORD=12345
+POSTGRES_DB=lemur
+POSTGRES_HOST=postgres

docker/src/lemur.conf.py

@@ -0,0 +1,237 @@
+import os.path
+import random
+import string
+from celery.schedules import crontab
+
+import base64
+
+_basedir = os.path.abspath(os.path.dirname(__file__))
+
+# See the Lemur docs (https://lemur.readthedocs.org) for more information on configuration
+
+LOG_LEVEL = str(os.environ.get('LOG_LEVEL', 'DEBUG'))
+LOG_FILE = str(os.environ.get('LOG_FILE', '/home/lemur/.lemur/lemur.log'))
+LOG_JSON = True
+
+CORS = os.environ.get("CORS") == "True"
+debug = os.environ.get("DEBUG") == "True"
+
+
+def get_random_secret(length):
+    secret_key = ''.join(random.choice(string.ascii_uppercase) for x in range(round(length / 4)))
+    secret_key = secret_key + ''.join(random.choice("~!@#$%^&*()_+") for x in range(round(length / 4)))
+    secret_key = secret_key + ''.join(random.choice(string.ascii_lowercase) for x in range(round(length / 4)))
+    return secret_key + ''.join(random.choice(string.digits) for x in range(round(length / 4)))
+
+
+# This is the secret key used by Flask session management
+SECRET_KEY = repr(os.environ.get('SECRET_KEY', get_random_secret(32).encode('utf8')))
+
+# You should consider storing these separately from your config
+LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET',
+                                         base64.b64encode(get_random_secret(32).encode('utf8'))))
+# This must match the key for whichever DB the container is using - this could be a dump of dev or test, or a unique key
+LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS',
+                                            base64.b64encode(get_random_secret(32).encode('utf8')).decode('utf8')))
+
+REDIS_HOST = 'redis'
+REDIS_PORT = 6379
+REDIS_DB = 0
+CELERY_RESULT_BACKEND = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_BROKER_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_IMPORTS = ('lemur.common.celery')
+CELERYBEAT_SCHEDULE = {
+    # All tasks are disabled by default. Enable any tasks you wish to run.
+    # 'fetch_all_pending_acme_certs': {
+    #     'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'remove_old_acme_certs': {
+    #     'task': 'lemur.common.celery.remove_old_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=8, minute=0, day_of_week=5),
+    # },
+    # 'clean_all_sources': {
+    #     'task': 'lemur.common.celery.clean_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=5, minute=0, day_of_week=5),
+    # },
+    # 'sync_all_sources': {
+    #     'task': 'lemur.common.celery.sync_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=0),
+    #     # this job is running 30min before endpoints_expire which deletes endpoints which were not updated
+    # },
+    # 'sync_source_destination': {
+    #     'task': 'lemur.common.celery.sync_source_destination',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=15),
+    # },
+    # 'report_celery_last_success_metrics': {
+    #     'task': 'lemur.common.celery.report_celery_last_success_metrics',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'certificate_reissue': {
+    #     'task': 'lemur.common.celery.certificate_reissue',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=9, minute=0),
+    # },
+    # 'certificate_rotate': {
+    #     'task': 'lemur.common.celery.certificate_rotate',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'endpoints_expire': {
+    #     'task': 'lemur.common.celery.endpoints_expire',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=30),
+    #     # this job is running 30min after sync_all_sources which updates endpoints
+    # },
+    # 'get_all_zones': {
+    #     'task': 'lemur.common.celery.get_all_zones',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*/30"),
+    # },
+    # 'check_revoked': {
+    #     'task': 'lemur.common.celery.check_revoked',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'enable_autorotate_for_certs_attached_to_endpoint': {
+    #     'task': 'lemur.common.celery.enable_autorotate_for_certs_attached_to_endpoint',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'notify_expirations': {
+    #     'task': 'lemur.common.celery.notify_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    #  },
+    # 'notify_authority_expirations': {
+    #     'task': 'lemur.common.celery.notify_authority_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'send_security_expiration_summary': {
+    #     'task': 'lemur.common.celery.send_security_expiration_summary',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0, day_of_week='mon-fri'),
+    # }
+}
+CELERY_TIMEZONE = 'UTC'
+
+SQLALCHEMY_ENABLE_FLASK_REPLICATED = False
+SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://lemur:lemur@localhost:5432/lemur')
+
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+SQLALCHEMY_ECHO = True
+SQLALCHEMY_POOL_RECYCLE = 499
+SQLALCHEMY_POOL_TIMEOUT = 20
+
+LEMUR_EMAIL = 'lemur@example.com'
+LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
+LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
+LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
+LEMUR_EMAIL_SENDER = 'smtp'
+
+# mail configuration
+# MAIL_SERVER = 'mail.example.com'
+
+PUBLIC_CA_MAX_VALIDITY_DAYS = 397
+DEFAULT_VALIDITY_DAYS = 365
+
+LEMUR_OWNER_EMAIL_IN_SUBJECT = False
+
+LEMUR_DEFAULT_COUNTRY = str(os.environ.get('LEMUR_DEFAULT_COUNTRY', 'US'))
+LEMUR_DEFAULT_STATE = str(os.environ.get('LEMUR_DEFAULT_STATE', 'California'))
+LEMUR_DEFAULT_LOCATION = str(os.environ.get('LEMUR_DEFAULT_LOCATION', 'Los Gatos'))
+LEMUR_DEFAULT_ORGANIZATION = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATION', 'Example, Inc.'))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT', ''))
+
+LEMUR_DEFAULT_AUTHORITY = str(os.environ.get('LEMUR_DEFAULT_AUTHORITY', 'ExampleCa'))
+
+LEMUR_DEFAULT_ROLE = 'operator'
+
+ACTIVE_PROVIDERS = []
+METRIC_PROVIDERS = []
+
+# Authority Settings - These will change depending on which authorities you are
+# using
+current_path = os.path.dirname(os.path.realpath(__file__))
+
+# DNS Settings
+
+# exclude logging missing SAN, since we can have certs from private CAs with only cn, prod parity
+LOG_SSL_SUBJ_ALT_NAME_ERRORS = False
+
+ACME_DNS_PROVIDER_TYPES = {"items": [
+    {
+        'name': 'route53',
+        'requirements': [
+            {
+                'name': 'account_id',
+                'type': 'int',
+                'required': True,
+                'helpMessage': 'AWS Account number'
+            },
+        ]
+    },
+    {
+        'name': 'cloudflare',
+        'requirements': [
+            {
+                'name': 'email',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Email'
+            },
+            {
+                'name': 'key',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Key'
+            },
+        ]
+    },
+    {
+        'name': 'dyn',
+    },
+    {
+        'name': 'ultradns',
+    },
+]}
+
+# Authority plugins which support revocation
+SUPPORTED_REVOCATION_AUTHORITY_PLUGINS = ['acme-issuer']

docker/supervisor.conf

@@ -0,0 +1,33 @@
+[supervisord]
+nodaemon=true
+user=root
+logfile=/dev/stdout
+logfile_maxbytes=0
+pidfile = /tmp/supervisord.pid
+
+[program:lemur]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
+command=lemur start -b 0.0.0.0:8000
+user=lemur
+directory=/opt/lemur/lemur
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:cron]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
+command=/usr/sbin/crond -f
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docs/administration.rst

@@ -28,6 +28,13 @@ Basic Configuration
 
         LOG_FILE = "/logs/lemur/lemur-test.log"
 
+.. data:: LOG_UPGRADE_FILE
+    :noindex:
+
+    ::
+
+        LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log"
+
 .. data:: DEBUG
     :noindex:
 
@@ -100,7 +107,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
 
         Specifies whether to allow certificates created by Lemur to expire on weekends. Default is True.
 
-.. data:: LEMUR_WHITELISTED_DOMAINS
+.. data:: LEMUR_ALLOWED_DOMAINS
     :noindex:
 
         List of regular expressions for domain restrictions; if the list is not empty, normal users can only issue
@@ -144,6 +151,15 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
         to start. Multiple keys can be provided to facilitate key rotation. The first key in the list is used for
         encryption and all keys are tried for decryption until one works. Each key must be 32 URL safe base-64 encoded bytes.
 
+        Only fields of type ``Vault`` will be encrypted. At present, only the following fields are encrypted:
+
+        * ``certificates.private_key``
+        * ``pending_certificates.private_key``
+        * ``dns_providers.credentials``
+        * ``roles.password``
+
+        For implementation details, see ``Vault`` in ``utils.py``.
+
         Running lemur create_config will securely generate a key for your configuration file.
         If you would like to generate your own, we recommend the following method:
 
@@ -156,11 +172,48 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
         LEMUR_ENCRYPTION_KEYS = ['1YeftooSbxCiX2zo8m1lXtpvQjy27smZcUUaGmffhMY=', 'LAfQt6yrkLqOK5lwpvQcT4jf2zdeTQJV1uYeh9coT5s=']
 
 
+.. data:: PUBLIC_CA_MAX_VALIDITY_DAYS
+    :noindex:
+
+        Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities.
+        The authorities with cab_compliant option set to true will use this config. The example below overrides the default validity
+        of 397 days and sets it to 365 days.
+
+    ::
+
+        PUBLIC_CA_MAX_VALIDITY_DAYS = 365
+
+
+.. data:: DEFAULT_VALIDITY_DAYS
+    :noindex:
+
+        Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which
+        is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please
+        note that this config is used for cert issuance only through Lemur UI. The example below overrides the default validity
+        of 365 days and sets it to 1095 days (3 years).
+
+    ::
+
+        DEFAULT_VALIDITY_DAYS = 1095
+
+
 .. data:: DEBUG_DUMP
     :noindex:
 
         Dump all imported or generated CSR and certificate details to stdout using OpenSSL. (default: `False`)
 
+.. data:: ALLOW_CERT_DELETION
+    :noindex:
+
+        When set to True, certificates can be marked as deleted via the API and deleted certificates will not be displayed
+        in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed"
+        and deleted certificates will always be visible in the UI. (default: `False`)
+
+.. data:: LEMUR_AWS_REGION
+    :noindex:
+
+        This is an optional config applicable for settings where Lemur is deployed in AWS. For accessing regionalized
+        STS endpoints, LEMUR_AWS_REGION defines the region where Lemur is deployed.
 
 Certificate Default Options
 ---------------------------
@@ -206,7 +259,7 @@ and are used when Lemur creates the CSR for your certificates.
 
     ::
 
-        LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = "Operations"
+        LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = ""
 
 
 .. data:: LEMUR_DEFAULT_ISSUER_PLUGIN
@@ -225,22 +278,123 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_AUTHORITY = "verisign"
 
 
+.. _NotificationOptions:
+
 Notification Options
 --------------------
 
-Lemur currently has very basic support for notifications. Currently only expiration notifications are supported. Actual notification
-is handled by the notification plugins that you have configured. Lemur ships with the 'Email' notification that allows expiration emails
-to be sent to subscribers.
+Lemur supports a small variety of notification types through a set of notification plugins.
+By default, Lemur configures a standard set of email notifications for all certificates.
 
-Templates for expiration emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
-Notifications are sent to the certificate creator, owner and security team as specified by the `LEMUR_SECURITY_TEAM_EMAIL` configuration parameter.
+**Plugin-capable notifications**
 
-Certificates marked as inactive will **not** be notified of upcoming expiration. This enables a user to essentially
-silence the expiration. If a certificate is active and is expiring the above will be notified according to the `LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS` or
-30, 15, 2 days before expiration if no intervals are set.
+These notifications can be configured to use all available notification plugins.
 
-Lemur supports sending certification expiration notifications through SES and SMTP.
+Supported types:
 
+* Certificate expiration
+
+**Email-only notifications**
+
+These notifications can only be sent via email and cannot use other notification plugins.
+
+Supported types:
+
+* CA certificate expiration
+* Pending ACME certificate failure
+* Certificate rotation
+* Security certificate expiration summary
+
+**Default notifications**
+
+When a certificate is created, the following email notifications are created for it if they do not exist.
+If these notifications already exist, they will be associated with the new certificate.
+
+* ``DEFAULT_<OWNER>_X_DAY``, where X is the set of values specified in ``LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS`` and defaults to 30, 15, and 2 if not specified. The owner's username will replace ``<OWNER>``.
+* ``DEFAULT_SECURITY_X_DAY``, where X is the set of values specified in ``LEMUR_SECURITY_TEAM_EMAIL_INTERVALS`` and defaults to ``LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS`` if not specified (which also defaults to 30, 15, and 2 if not specified).
+
+These notifications can be disabled if desired. They can also be unassociated with a specific certificate.
+
+**Disabling notifications**
+
+Notifications can be disabled either for an individual certificate (which disables all notifications for that certificate)
+or for an individual notification object (which disables that notification for all associated certificates).
+At present, disabling a notification object will only disable certificate expiration notifications, and not other types,
+since other notification types don't use notification objects.
+
+**Certificate expiration**
+
+Certificate expiration notifications are sent when the scheduled task to send certificate expiration notifications runs
+(see :ref:`PeriodicTasks`). Specific patterns of certificate names may be excluded using ``--exclude`` (when using
+cron; you may specify this multiple times for multiple patterns) or via the config option ``EXCLUDE_CN_FROM_NOTIFICATION``
+(when using celery; this is a list configuration option, meaning you specify multiple values, such as
+``['exclude', 'also exclude']``). The specified exclude pattern will match if found anywhere in the certificate name.
+
+When the periodic task runs, Lemur checks for certificates meeting the following conditions:
+
+* Certificate has notifications enabled
+* Certificate is not expired
+* Certificate is not revoked
+* Certificate name does not match the `exclude` parameter
+* Certificate has at least one associated notification object
+* That notification is active
+* That notification's configured interval and unit match the certificate's remaining lifespan
+
+All eligible certificates are then grouped by owner and applicable notification. For each notification and certificate group,
+Lemur will send the expiration notification using whichever plugin was configured for that notification object.
+In addition, Lemur will send an email to the certificate owner and security team (as specified by the
+``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter).
+
+**CA certificate expiration**
+
+Certificate authority certificate expiration notifications are sent when the scheduled task to send authority certificate
+expiration notifications runs (see :ref:`PeriodicTasks`). Notifications are sent via the intervals configured in the
+configuration parameter ``LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS``, with a default of 365 and 180 days.
+
+When the periodic task runs, Lemur checks for certificates meeting the following conditions:
+
+* Certificate has notifications enabled
+* Certificate is not expired
+* Certificate is not revoked
+* Certificate is associated with a CA
+* Certificate's remaining lifespan matches one of the configured intervals
+
+All eligible certificates are then grouped by owner and expiration interval. For each interval and certificate group,
+Lemur will send the CA certificate expiration notification via email to the certificate owner and security team
+(as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter).
+
+**Pending ACME certificate failure**
+
+Whenever a pending ACME certificate fails to be issued, Lemur will send a notification via email to the certificate owner
+and security team (as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter). This email is not sent if
+the pending certificate had notifications disabled.
+
+Lemur will attempt 3x times to resolve a pending certificate.
+This can at times result into 3 duplicate certificates, if all certificate attempts get resolved.
+
+**Certificate rotation**
+
+Whenever a cert is rotated, Lemur will send a notification via email to the certificate owner. This notification is
+disabled by default; to enable it, you must set the option ``--notify`` (when using cron) or the configuration parameter
+``ENABLE_ROTATION_NOTIFICATION`` (when using celery).
+
+**Security certificate expiration summary**
+
+If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all
+certificates with upcoming expiration date that occurs within the number of days specified by the
+``LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS`` configuration parameter (with a fallback of 14 days).
+Note that certificates will be included in this summary even if they do not have any associated notifications.
+
+This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails.
+
+NOTE: At present, this summary email essentially duplicates the certificate expiration notifications, since all
+certificate expiration notifications are also sent to the security team. This issue will be fixed in the future.
+
+**Email notifications**
+
+Templates for emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
+
+The following configuration options are supported:
 
 .. data:: LEMUR_EMAIL_SENDER
     :noindex:
@@ -255,6 +409,25 @@ Lemur supports sending certification expiration notifications through SES and SM
         you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
 
 
+.. data:: LEMUR_SES_SOURCE_ARN
+    :noindex:
+
+    Specifies an ARN to use as the SourceArn when sending emails via SES.
+
+    .. note::
+        This parameter is only required if you're using a sending authorization with SES.
+        See: `Using sending authorization with Amazon SES <https://docs.aws.amazon.com/ses/latest/DeveloperGuide/sending-authorization.html>`_
+
+
+.. data:: LEMUR_SES_REGION
+    :noindex:
+
+    Specifies a region for sending emails via SES.
+
+    .. note::
+        This parameter defaults to us-east-1 and is only required if you wish to use a different region.
+
+
 .. data:: LEMUR_EMAIL
     :noindex:
 
@@ -262,7 +435,7 @@ Lemur supports sending certification expiration notifications through SES and SM
 
         ::
 
-            LEMUR_EMAIL = 'lemur.example.com'
+            LEMUR_EMAIL = 'lemur@example.com'
 
 
 .. data:: LEMUR_SECURITY_TEAM_EMAIL
@@ -274,16 +447,81 @@ Lemur supports sending certification expiration notifications through SES and SM
 
             LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
 
-
 .. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS
     :noindex:
 
-        Lemur notification intervals
+        Lemur notification intervals. If unspecified, the value [30, 15, 2] is used.
 
         ::
 
             LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
 
+.. data:: LEMUR_SECURITY_TEAM_EMAIL_INTERVALS
+    :noindex:
+
+       Alternate notification interval set for security team notifications. Use this if you would like the default security team notification interval for new certificates to differ from the global default as specified in LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS. If unspecified, the value of LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS is used. Security team default notifications for new certificates can effectively be disabled by setting this value to an empty array.
+
+       ::
+
+          LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
+
+.. data:: LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS
+    :noindex:
+
+       Notification interval set for CA certificate expiration notifications. If unspecified, the value [365, 180] is used (roughly one year and 6 months).
+
+       ::
+
+          LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS = [365, 180]
+
+
+Celery Options
+---------------
+To make use of automated tasks within lemur (e.g. syncing source/destinations, or reissuing ACME certificates), you
+need to configure celery. See :ref:`Periodic Tasks <PeriodicTasks>` for more in depth documentation.
+
+.. data:: CELERY_RESULT_BACKEND
+    :noindex:
+
+        The url to your redis backend (needs to be in the format `redis://<host>:<port>/<database>`)
+
+.. data:: CELERY_BROKER_URL
+    :noindex:
+
+        The url to your redis broker (needs to be in the format `redis://<host>:<port>/<database>`)
+
+.. data:: CELERY_IMPORTS
+    :noindex:
+
+        The module that celery needs to import, in our case thats `lemur.common.celery`
+
+.. data:: CELERY_TIMEZONE
+    :noindex:
+
+        The timezone for celery to work with
+
+
+.. data:: CELERYBEAT_SCHEDULE
+    :noindex:
+
+        This defines the schedule, with which the celery beat makes the worker run the specified tasks.
+
+Since the celery module, relies on the RedisHandler, the following options also need to be set.
+
+.. data:: REDIS_HOST
+    :noindex:
+
+        Hostname of your redis instance
+
+.. data:: REDIS_PORT
+    :noindex:
+
+        Port on which redis is running (default: 6379)
+
+.. data:: REDIS_DB
+    :noindex:
+
+        Which redis database to be used, by default redis offers databases 0-15 (default: 0)
 
 Authentication Options
 ----------------------
@@ -316,6 +554,7 @@ Here is an example LDAP configuration stanza you can add to your config. Adjust
         LDAP_CACERT_FILE = '/opt/lemur/trusted.pem'
         LDAP_REQUIRED_GROUP = 'certificate-management-access'
         LDAP_GROUPS_TO_ROLES = {'certificate-management-admin': 'admin', 'certificate-management-read-only': 'read-only'}
+        LDAP_IS_ACTIVE_DIRECTORY = True
 
 
 The lemur ldap module uses the `user principal name` (upn) of the authenticating user to bind. This is done once for each user at login time. The UPN is effectively the email address in AD/LDAP of the user. If the user doesn't provide the email address, it constructs one based on the username supplied (which should normally match the samAccountName) and the value provided by the config LDAP_EMAIL_DOMAIN.
@@ -398,6 +637,17 @@ The following LDAP options are not required, however TLS is always recommended.
             LDAP_GROUPS_TO_ROLES = {'lemur_admins': 'admin', 'Lemur Team DL Group': 'team@example.com'}
 
 
+.. data:: LDAP_IS_ACTIVE_DIRECTORY
+    :noindex:
+
+        When set to True, nested group memberships are supported, by searching for groups with the member:1.2.840.113556.1.4.1941 attribute set to the user DN.
+        When set to False, the list of groups will be determined by the 'memberof' attribute of the LDAP user logging in.
+
+        ::
+
+            LDAP_IS_ACTIVE_DIRECTORY = False
+
+
 Authentication Providers
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -469,6 +719,33 @@ For more information about how to use social logins, see: `Satellizer <https://g
 
             PING_AUTH_ENDPOINT = "https://<yourpingserver>/oauth2/authorize"
 
+.. data:: PING_USER_MEMBERSHIP_URL
+    :noindex:
+
+        An optional additional endpoint to learn membership details post the user validation.
+
+        ::
+
+            PING_USER_MEMBERSHIP_URL = "https://<yourmembershipendpoint>"
+
+.. data:: PING_USER_MEMBERSHIP_TLS_PROVIDER
+    :noindex:
+
+        A custom TLS session provider plugin name
+
+        ::
+
+            PING_USER_MEMBERSHIP_TLS_PROVIDER = "slug-name"
+
+.. data:: PING_USER_MEMBERSHIP_SERVICE
+    :noindex:
+
+        Membership service name used by PING_USER_MEMBERSHIP_TLS_PROVIDER to create a session
+
+        ::
+
+            PING_USER_MEMBERSHIP_SERVICE = "yourmembershipservice"
+
 .. data:: OAUTH2_SECRET
     :noindex:
 
@@ -566,8 +843,190 @@ If you are not using a metric provider you do not need to configure any of these
 Plugin Specific Options
 -----------------------
 
+ACME Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. data:: ACME_DNS_PROVIDER_TYPES
+    :noindex:
+
+        Dictionary of ACME DNS Providers and their requirements.
+
+.. data:: ACME_ENABLE_DELEGATED_CNAME
+    :noindex:
+
+        Enables delegated DNS domain validation using CNAMES.  When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges.
+
+
+The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
+account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
+
+
+.. data:: ACME_PRIVATE_KEY
+    :noindex:
+
+            This is the private key, the account was registered with (in JWK format)
+
+.. data:: ACME_REGR
+    :noindex:
+
+            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
+
+.. data:: ACME_PREFERRED_ISSUER
+    :noindex:
+
+            This is an optional parameter to indicate the preferred chain to retrieve from ACME when finalizing the order.
+            This is applicable to Let's Encrypts recent `migration <https://letsencrypt.org/certificates/>`_ to their
+            own root, where they provide two distinct certificate chains (fullchain_pem vs. alternative_fullchains_pem);
+            the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain
+            is rooted in X1 root CA.
+            Select "X1" to get the shorter chain (currently alternative), leave blank or "DST Root CA X3" for the longer chain.
+
+
+Active Directory Certificate Services Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+.. data:: ADCS_SERVER
+    :noindex:
+
+        FQDN of your ADCS Server
+
+
+.. data:: ADCS_AUTH_METHOD
+    :noindex:
+
+        The chosen authentication method. Either ‘basic’ (the default), ‘ntlm’ or ‘cert’ (SSL client certificate). The next 2 variables are interpreted differently for different methods.
+
+
+.. data:: ADCS_USER
+    :noindex:
+
+        The username (basic) or the path to the public cert (cert) of the user accessing PKI
+
+
+.. data:: ADCS_PWD
+    :noindex:
+
+        The passwd (basic) or the path to the private key (cert) of the user accessing PKI
+
+
+.. data:: ADCS_TEMPLATE
+    :noindex:
+
+        Template to be used for certificate issuing. Usually display name w/o spaces
+        
+.. data:: ADCS_TEMPLATE_<upper(authority.name)>
+    :noindex:
+
+        If there is a config variable ADCS_TEMPLATE_<upper(authority.name)> take the value as Cert template else default to ADCS_TEMPLATE to be compatible with former versions. Template to be used for certificate issuing. Usually display name w/o spaces
+
+.. data:: ADCS_START
+    :noindex:
+
+        Used in ADCS-Sourceplugin. Minimum id of the first certificate to be returned. ID is increased by one until ADCS_STOP. Missing cert-IDs are ignored
+
+.. data:: ADCS_STOP
+    :noindex:
+
+        Used for ADCS-Sourceplugin. Maximum id of the certificates returned. 
+        
+
+.. data:: ADCS_ISSUING
+    :noindex:
+
+        Contains the issuing cert of the CA
+
+
+.. data:: ADCS_ROOT
+    :noindex:
+
+        Contains the root cert of the CA
+
+Entrust Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enables the creation of Entrust certificates. You need to set the API access up with Entrust support. Check the information in the Entrust Portal as well. 
+Certificates are created as "SERVER_AND_CLIENT_AUTH".
+Caution: Sometimes the entrust API does not respond in a timely manner. This error is handled and reported by the plugin. Should this happen you just have to hit the create button again after to create a valid certificate. 
+The following parameters have to be set in the configuration files.
+
+.. data:: ENTRUST_URL
+    :noindex:
+    
+       This is the url for the Entrust API. Refer to the API documentation.
+       
+.. data:: ENTRUST_API_CERT
+    :noindex:
+    
+       Path to the certificate file in PEM format. This certificate is created in the onboarding process. Refer to the API documentation.
+       
+.. data:: ENTRUST_API_KEY
+    :noindex:
+    
+       Path to the key file in RSA format. This certificate is created in the onboarding process. Refer to the API documentation. Caution: the request library cannot handle encrypted keys. The keyfile therefore has to contain the unencrypted key. Please put this in a secure location on the server.
+       
+.. data:: ENTRUST_API_USER
+    :noindex:
+    
+       String with the API user. This user is created in the onboarding process. Refer to the API documentation.   
+       
+.. data:: ENTRUST_API_PASS
+    :noindex:
+    
+       String with the password for the API user. This password is created in the onboarding process. Refer to the API documentation.
+
+.. data:: ENTRUST_NAME
+    :noindex:
+    
+        String with the name that should appear as certificate owner in the Entrust portal. Refer to the API documentation.
+
+.. data:: ENTRUST_EMAIL
+    :noindex:
+    
+        String with the email address that should appear as certificate contact email in the Entrust portal. Refer to the API documentation.       
+
+.. data:: ENTRUST_PHONE
+    :noindex:
+    
+        String with the phone number that should appear as certificate contact in the Entrust portal. Refer to the API documentation.        
+
+.. data:: ENTRUST_ISSUING
+    :noindex:
+    
+        Contains the issuing cert of the CA
+
+.. data:: ENTRUST_ROOT
+    :noindex:
+    
+        Contains the root cert of the CA
+
+.. data:: ENTRUST_PRODUCT_<upper(authority.name)>
+    :noindex:
+
+        If there is a config variable ENTRUST_PRODUCT_<upper(authority.name)> take the value as cert product name else default to "STANDARD_SSL". Refer to the API documentation for valid products names.
+
+
+.. data:: ENTRUST_CROSS_SIGNED_RSA_L1K
+    :noindex:
+
+        This is optional. Entrust provides support for cross-signed subCAS. One can set ENTRUST_CROSS_SIGNED_RSA_L1K to the respective cross-signed RSA-based subCA PEM and Lemur will replace the retrieved subCA with ENTRUST_CROSS_SIGNED_RSA_L1K.
+
+
+.. data:: ENTRUST_CROSS_SIGNED_ECC_L1F
+    :noindex:
+
+        This is optional. Entrust provides support for cross-signed subCAS. One can set ENTRUST_CROSS_SIGNED_ECC_L1F to the respective cross-signed EC-based subCA PEM and Lemur will replace the retrieved subCA with ENTRUST_CROSS_SIGNED_ECC_L1F.
+
+
+.. data:: ENTRUST_USE_DEFAULT_CLIENT_ID
+    :noindex:
+
+        If set to True, Entrust will use the primary client ID of 1, which applies to most use-case.
+        Otherwise, Entrust will first lookup the clientId before ordering the certificate.
+
+
 Verisign Issuer Plugin
-^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~
 
 Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur,
 Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation
@@ -650,10 +1109,16 @@ The following configuration properties are required to use the Digicert issuer p
             This is the root to be used for your CA chain
 
 
-.. data:: DIGICERT_DEFAULT_VALIDITY
+.. data:: DIGICERT_DEFAULT_VALIDITY_DAYS
     :noindex:
 
-            This is the default validity (in years), if no end date is specified. (Default: 1)
+            This is the default validity (in days), if no end date is specified. (Default: 397)
+
+
+.. data:: DIGICERT_MAX_VALIDITY_DAYS
+    :noindex:
+
+            This is the maximum validity (in days). (Default: value of DIGICERT_DEFAULT_VALIDITY_DAYS)
 
 
 .. data:: DIGICERT_PRIVATE
@@ -663,7 +1128,7 @@ The following configuration properties are required to use the Digicert issuer p
 
 
 CFSSL Issuer Plugin
-^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~
 
 The following configuration properties are required to use the CFSSL issuer plugin.
 
@@ -682,9 +1147,36 @@ The following configuration properties are required to use the CFSSL issuer plug
 
         This is the intermediate to be used for your CA chain
 
+.. data:: CFSSL_KEY
+    :noindex:
+
+        This is the hmac key to authenticate to the CFSSL service. (Optional)
+
+
+Hashicorp Vault Source/Destination Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Lemur can import and export certificate data to and from a Hashicorp Vault secrets store. Lemur can connect to a different Vault service per source/destination.
+
+.. note:: This plugin does not supersede or overlap the 3rd party Vault Issuer plugin.
+
+.. note:: Vault does not have any configuration properties however it does read from a file on disk for a vault access token. The Lemur service account needs read access to this file.
+
+Vault Source
+""""""""""""
+
+The Vault Source Plugin will read from one Vault object location per source defined. There is expected to be one or more certificates defined in each object in Vault.
+
+Vault Destination
+"""""""""""""""""
+
+A Vault destination can be one object in Vault or a directory where all certificates will be stored as their own object by CN.
+
+Vault Destination supports a regex filter to prevent certificates with SAN that do not match the regex filter from being deployed. This is an optional feature per destination defined.
+
 
 AWS Source/Destination Plugin
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions.
 
@@ -867,6 +1359,53 @@ Will be the sender of all notifications, so ensure that it is verified with AWS.
 SES if the default notification gateway and will be used unless SMTP settings are configured in the application configuration
 settings.
 
+PowerDNS ACME Plugin
+~~~~~~~~~~~~~~~~~~~~~~
+
+The following configuration properties are required to use the PowerDNS ACME Plugin for domain validation.
+
+
+.. data:: ACME_POWERDNS_DOMAIN
+    :noindex:
+
+            This is the FQDN for the PowerDNS API (without path)
+
+
+.. data:: ACME_POWERDNS_SERVERID
+    :noindex:
+
+            This is the ServerID attribute of the PowerDNS API Server (i.e. "localhost")
+
+
+.. data:: ACME_POWERDNS_APIKEYNAME
+    :noindex:
+
+            This is the Key name to use for authentication (i.e. "X-API-Key")
+
+
+.. data:: ACME_POWERDNS_APIKEY
+    :noindex:
+
+            This is the API Key to use for authentication (i.e. "Password")
+
+
+.. data:: ACME_POWERDNS_RETRIES
+    :noindex:
+
+            This is the number of times DNS Verification should be attempted (i.e. 20)
+
+
+.. data:: ACME_POWERDNS_VERIFY
+    :noindex:
+
+            This configures how TLS certificates on the PowerDNS API target are validated.  The PowerDNS Plugin depends on the PyPi requests library, which supports the following options for the verify parameter:
+
+            True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default)
+
+            False: Disables certificate validation (Not Recommended)
+
+            File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle.
+
 .. _CommandLineInterface:
 
 Command Line Interface
@@ -965,6 +1504,15 @@ All commands default to `~/.lemur/lemur.conf.py` if a configuration is not speci
         lemur notify
 
 
+.. data:: acme
+
+    Handles all ACME related tasks, like ACME plugin testing.
+
+    ::
+
+        lemur acme
+
+
 Sub-commands
 ------------
 
@@ -1036,7 +1584,9 @@ Verisign/Symantec
 -----------------
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Issuer
 :Description:
@@ -1062,18 +1612,23 @@ Acme
 
 :Authors:
     Kevin Glisson <kglisson@netflix.com>,
-    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>,
+    Chad Sine <csine@netflix.com>
 :Type:
     Issuer
 :Description:
-    Adds support for the ACME protocol (including LetsEncrypt) with domain validation being handled Route53.
+    Adds support for the ACME protocol (including LetsEncrypt) with domain validation using several providers.
 
 
 Atlas
 -----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Metric
 :Description:
@@ -1084,7 +1639,9 @@ Email
 -----
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Notification
 :Description:
@@ -1102,28 +1659,45 @@ Slack
     Adds support for slack notifications.
 
 
-AWS
-----
+AWS (Source)
+------------
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Source
 :Description:
     Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment.
 
 
-AWS
-----
+AWS (Destination)
+-----------------
 
 :Authors:
-    Kevin Glisson <kglisson@netflix.com>
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
 :Type:
     Destination
 :Description:
     Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment.
 
 
+AWS (SNS Notification)
+----------------------
+
+:Authors:
+    Jasmine Schladen <jschladen@netflix.com>
+:Type:
+    Notification
+:Description:
+    Adds support for SNS notifications. SNS notifications (like other notification plugins) are currently only supported
+    for certificate expiration. Configuration requires a region, account number, and SNS topic name; these elements
+    are then combined to build the topic ARN. Lemur must have access to publish messages to the specified SNS topic.
+
+
 Kubernetes
 ----------
 
@@ -1167,6 +1741,26 @@ CFSSL
 :Description:
     Basic support for generating certificates from the private certificate authority CFSSL
 
+Vault
+-----
+
+:Authors:
+    Christopher Jolley <chris@alwaysjolley.com>
+:Type:
+    Source
+:Description:
+    Source plugin imports certificates from Hashicorp Vault secret store.
+
+Vault
+-----
+
+:Authors:
+    Christopher Jolley <chris@alwaysjolley.com>
+:Type:
+    Destination
+:Description:
+    Destination plugin to deploy certificates to Hashicorp Vault secret store.
+
 
 3rd Party Plugins
 =================

docs/conf.py

@@ -18,19 +18,23 @@ from unittest.mock import MagicMock
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath(".."))
 
 # Mock packages that cannot be installed on rtd
-on_rtd = os.environ.get('READTHEDOCS') == 'True'
+on_rtd = os.environ.get("READTHEDOCS") == "True"
 if on_rtd:
+
     class Mock(MagicMock):
         @classmethod
         def __getattr__(cls, name):
             return MagicMock()
 
-    MOCK_MODULES = ['ldap']
+    MOCK_MODULES = ["ldap"]
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
+autodoc_mock_imports = ["python-ldap", "acme", "certsrv", "dnspython3", "dyn", "factory-boy", "flask_replicated",
+                        "josepy", "logmatic", "pem"]
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -39,27 +43,23 @@ if on_rtd:
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinxcontrib.autohttp.flask',
-    'sphinx.ext.todo',
-]
+extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autohttp.flask", "sphinx.ext.todo"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'lemur'
-copyright = u'2018, Netflix Inc.'
+project = u"lemur"
+copyright = u"2018, Netflix Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -68,7 +68,7 @@ copyright = u'2018, Netflix Inc.'
 base_dir = os.path.join(os.path.dirname(__file__), os.pardir)
 about = {}
 with open(os.path.join(base_dir, "lemur", "__about__.py")) as f:
-    exec(f.read(), about)
+    exec(f.read(), about)  # nosec
 
 version = release = about["__version__"]
 
@@ -84,7 +84,7 @@ version = release = about["__version__"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ["_build"]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -102,7 +102,7 @@ exclude_patterns = ['_build']
 # show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
 # modindex_common_prefix = []
@@ -114,11 +114,12 @@ pygments_style = 'sphinx'
 # -- Options for HTML output ----------------------------------------------
 
 # on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+on_rtd = os.environ.get("READTHEDOCS", None) == "True"
 
 if not on_rtd:  # only import and set the theme if we're building docs locally
     import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
+
+    html_theme = "sphinx_rtd_theme"
     html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
@@ -148,7 +149,7 @@ if not on_rtd:  # only import and set the theme if we're building docs locally
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
@@ -197,7 +198,7 @@ html_static_path = ['_static']
 # html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'lemurdoc'
+htmlhelp_basename = "lemurdoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -205,10 +206,8 @@ htmlhelp_basename = 'lemurdoc'
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #'preamble': '',
 }
@@ -217,8 +216,7 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'lemur.tex', u'Lemur Documentation',
-   u'Kevin Glisson', 'manual'),
+    ("index", "lemur.tex", u"Lemur Documentation", u"Netflix Security", "manual")
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -246,10 +244,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'Lemur', u'Lemur Documentation',
-     [u'Kevin Glisson'], 1)
-]
+man_pages = [("index", "Lemur", u"Lemur Documentation", [u"Netflix Security"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -261,9 +256,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Lemur', u'Lemur Documentation',
-   u'Kevin Glisson', 'Lemur', 'SSL Certificate Management',
-   'Miscellaneous'),
+    (
+        "index",
+        "Lemur",
+        u"Lemur Documentation",
+        u"Netflix Security",
+        "Lemur",
+        "SSL Certificate Management",
+        "Miscellaneous",
+    )
 ]
 
 # Documents to append as an appendix to all manuals.

docs/developer/index.rst

@@ -22,12 +22,18 @@ Once you've got all that, the rest is simple:
     # If you have a fork, you'll want to clone it instead
     git clone git://github.com/netflix/lemur.git
 
-    # Create a python virtualenv
-    mkvirtualenv lemur
+    # Create and activate python virtualenv from within the lemur repo
+    python3 -m venv env
+    . env/bin/activate
+
+    # Install doc requirements
 
-    # Make the magic happen
     make dev-docs
 
+    # Make the docs
+    cd docs
+    make html
+
 Running ``make dev-docs`` will install the basic requirements to get Sphinx running.
 
 
@@ -37,18 +43,35 @@ Building Documentation
 Inside the ``docs`` directory, you can run ``make`` to build the documentation.
 See ``make help`` for available options and the `Sphinx Documentation <http://sphinx-doc.org/contents.html>`_ for more information.
 
+Adding New Modules to Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When a new module is added, it will need to be added to the documentation.
+Ideally, we might rely on `sphinx-apidoc <https://www.sphinx-doc.org/en/master/man/sphinx-apidoc.html>`_ to autogenerate our documentation.
+Unfortunately, this causes some build problems.
+Instead, you'll need to add new modules by hand.
 
 Developing Against HEAD
 -----------------------
 
 We try to make it easy to get up and running in a development environment using a git checkout
-of Lemur. You'll want to make sure you have a few things on your local system first:
+of Lemur. There are two ways to run Lemur locally: directly on your development machine, or
+in a Docker container.
+
+**Running in a Docker container**
+
+Look at the `lemur-docker <https://github.com/Netflix/lemur-docker>`_ project.
+Usage instructions are self-contained in the README for that project.
+
+**Running directly on your development machine**
+
+You'll want to make sure you have a few things on your local system first:
 
 * python-dev (if you're on OS X, you already have this)
 * pip
 * virtualenv (ideally virtualenvwrapper)
 * node.js (for npm and building css/javascript)
-+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
 
 Once you've got all that, the rest is simple:
 
@@ -58,7 +81,7 @@ Once you've got all that, the rest is simple:
     git clone git://github.com/lemur/lemur.git
 
     # Create a python virtualenv
-    mkvirtualenv lemur
+    python3 -m venv env
 
     # Make the magic happen
     make
@@ -93,7 +116,9 @@ You'll likely want to make some changes to the default configuration (we recomme
 Running tests with Docker and docker-compose
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Alternatively you can use Docker and docker-compose for running the tests with ``docker-compose run test``.
+If you just want to run tests in a Docker container, you can use Docker and docker-compose for running the tests with ``docker-compose run test`` directly in the ``lemur`` project.
+
+(For running the Lemur service in Docker, see `lemur-docker <https://github.com/Netflix/lemur-docker>`_.)
 
 
 Coding Standards
@@ -135,7 +160,7 @@ The test suite consists of multiple parts, testing both the Python and JavaScrip
 
     make test
 
-If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``test-js`` for the JavaScript tests.
+If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``make test-js`` for the JavaScript tests.
 
 
 You'll notice that the test suite is structured based on where the code lives, and strongly encourages using the mock library to drive more accurate individual tests.
@@ -146,7 +171,7 @@ You'll notice that the test suite is structured based on where the code lives, a
 Static Media
 ------------
 
-Lemur uses a library that compiles it's static media assets (LESS and JS files) automatically. If you're developing using
+Lemur uses a library that compiles its static media assets (LESS and JS files) automatically. If you're developing using
 runserver you'll see changes happen not only in the original files, but also the minified or processed versions of the file.
 
 If you've made changes and need to compile them by hand for any reason, you can do so by running:

docs/developer/internals/lemur.defaults.rst

@@ -0,0 +1,29 @@
+defaults Package
+================
+
+:mod:`defaults` Module
+----------------------------------------
+
+.. automodule:: lemur.defaults
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module	
+-----------------------------	
+
+.. automodule:: lemur.defaults.schemas	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module	
+---------------------------	
+
+.. automodule:: lemur.defaults.views	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.deployment.rst

@@ -0,0 +1,20 @@
+deployment Package
+===================
+
+:mod:`deployment` Module
+----------------------------------------
+
+.. automodule:: lemur.deployment
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module	
+------------------------------	
+
+.. automodule:: lemur.deployment.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.endpoints.rst

@@ -0,0 +1,56 @@
+endpoints Package
+===================
+
+:mod:`endpoints` Module
+----------------------------------------
+
+.. automodule:: lemur.endpoints
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module	
+--------------------------	
+
+.. automodule:: lemur.endpoints.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module	
+-----------------------------	
+
+.. automodule:: lemur.endpoints.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module	
+------------------------------	
+
+.. automodule:: lemur.endpoints.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module	
+------------------------------	
+
+.. automodule:: lemur.endpoints.service	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module	
+----------------------------	
+
+.. automodule:: lemur.endpoints.views	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.logs.rst

@@ -0,0 +1,47 @@
+logs Package
+===================
+
+:mod:`logs` Module
+--------------------
+
+.. automodule:: lemur.logs
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module
+------------------------------	
+
+.. automodule:: lemur.logs.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module
+------------------------------
+
+.. automodule:: lemur.logs.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.logs.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.logs.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_acme.rst

@@ -0,0 +1,83 @@
+lemur_acme package	
+=================================	
+
+:mod:`lemur_acme` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_acme
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`acme_handlers` Module
+-----------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.acme_handlers	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`challenge_types` Module
+-------------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.challenge_types	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cloudflare` Module
+-------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.cloudflare	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`dyn` Module
+------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.dyn	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+---------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.plugin	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`powerdns` Module
+-----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.powerdns	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`route53` Module
+----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.route53	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`ultradns` Module
+-----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.ultradns	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_atlas.rst

@@ -0,0 +1,20 @@
+lemur_atlas package
+==================================
+
+:mod:`lemur_atlas` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_atlas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_atlas.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_cryptography.rst

@@ -0,0 +1,20 @@
+lemur_cryptography package
+==================================
+
+:mod:`lemur_cryptography` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_cryptography
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_cryptography.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_digicert.rst

@@ -0,0 +1,20 @@
+lemur_digicert package
+==================================
+
+:mod:`lemur_digicert` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_digicert
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_digicert.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_jks.rst

@@ -0,0 +1,20 @@
+lemur_jks package
+==================================
+
+:mod:`lemur_jks` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_jks
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_jks.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_kubernetes.rst

@@ -0,0 +1,20 @@
+lemur_kubernetes package
+==================================
+
+:mod:`lemur_kubernetes` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_kubernetes
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_kubernetes.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_openssl.rst

@@ -0,0 +1,20 @@
+lemur_openssl package
+==================================
+
+:mod:`lemur_openssl` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_openssl
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_openssl.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_slack.rst

@@ -0,0 +1,20 @@
+lemur_slack package
+==================================
+
+:mod:`lemur_slack` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_slack
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_slack.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.reporting.rst

@@ -0,0 +1,38 @@
+reporting Package
+===================
+
+:mod:`reporting` Module
+----------------------------------------
+
+.. automodule:: lemur.reporting
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module
+------------------------------	
+
+.. automodule:: lemur.reporting.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.reporting.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.reporting.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.rst

@@ -28,15 +28,6 @@ lemur Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`decorators` Module
-------------------------
-
-.. automodule:: lemur.decorators
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`exceptions` Module
 ------------------------
 
@@ -108,7 +99,7 @@ Subpackages
     lemur.plugins.lemur_atlas
     lemur.plugins.lemur_cryptography
     lemur.plugins.lemur_digicert
-    lemur.plugins.lemur_java
+    lemur.plugins.lemur_jks
     lemur.plugins.lemur_kubernetes
     lemur.plugins.lemur_openssl
     lemur.plugins.lemur_slack

docs/developer/internals/lemur.sources.rst

@@ -0,0 +1,56 @@
+sources Package
+===================
+
+:mod:`sources` Module
+----------------------
+
+.. automodule:: lemur.sources
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module
+------------------------------	
+
+.. automodule:: lemur.sources.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module
+------------------------------
+
+.. automodule:: lemur.sources.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module
+------------------------------
+
+.. automodule:: lemur.sources.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.sources.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.sources.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.tests.rst

@@ -0,0 +1,11 @@
+tests Package
+=============
+
+:mod:`tests` Module
+--------------------
+
+.. automodule:: lemur.tests
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/plugins/index.rst

@@ -104,7 +104,7 @@ The `IssuerPlugin` exposes four functions functions::
 
     def create_certificate(self, csr, issuer_options):
         # requests.get('a third party')
-    def revoke_certificate(self, certificate, comments):
+    def revoke_certificate(self, certificate, reason):
         # requests.put('a third party')
     def get_ordered_certificate(self, order_id):
         # requests.get('already existing certificate')
@@ -145,8 +145,7 @@ The `IssuerPlugin` doesn't have any options like Destination, Source, and Notifi
 any fields you might need to submit a request to a third party. If there are additional options you need
 in your plugin feel free to open an issue, or look into adding additional options to issuers yourself.
 
-Asynchronous Certificates
-^^^^^^^^^^^^^^^^^^^^^^^^^
+**Asynchronous Certificates**
 An issuer may take some time to actually issue a certificate for an order.  In this case, a `PendingCertificate` is returned, which holds information to recreate a `Certificate` object at a later time.  Then, `get_ordered_certificate()` should be run periodically via `python manage.py pending_certs fetch -i all` to attempt to retrieve an ordered certificate::
 
     def get_ordered_ceriticate(self, order_id):
@@ -154,6 +153,7 @@ An issuer may take some time to actually issue a certificate for an order.  In t
         # retrieve an order, and check if there is an issued certificate attached to it
 
 `cancel_ordered_certificate()` should be implemented to allow an ordered certificate to be canceled before it is issued::
+
         def cancel_ordered_certificate(self, pending_cert, **kwargs):
             # pending_cert should contain the necessary information to match an order
             # kwargs can be given to provide information to the issuer for canceling
@@ -215,18 +215,22 @@ Notification
 ------------
 
 Lemur includes the ability to create Email notifications by **default**. These notifications
-currently come in the form of expiration notices. Lemur periodically checks certifications expiration dates and
+currently come in the form of expiration and rotation notices for all certificates, expiration notices for CA certificates,
+and ACME certificate creation failure notices. Lemur periodically checks certificate expiration dates and
 determines if a given certificate is eligible for notification. There are currently only two parameters used to
 determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
 of days the current date (UTC) is from that expiration date.
 
-There are currently two objects that available for notification plugins the first is `NotficationPlugin`. This is the base object for
-any notification within Lemur. Currently the only support notification type is an certificate expiration notification. If you
+Certificate expiration notifications can also be configured for Slack or AWS SNS. Other notifications are not configurable.
+Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email.
+
+There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for
+any notification within Lemur. Currently the only supported notification type is a certificate expiration notification. If you
 are trying to create a new notification type (audit, failed logins, etc.) this would be the object to base your plugin on.
 You would also then need to build additional code to trigger the new notification type.
 
-The second is `ExpirationNotificationPlugin`, this object inherits from `NotificationPlugin` object.
-You will most likely want to base your plugin on, if you want to add new channels for expiration notices (Slack, HipChat, Jira, etc.). It adds default options that are required by
+The second is `ExpirationNotificationPlugin`, which inherits from the `NotificationPlugin` object.
+You will most likely want to base your plugin on this object if you want to add new channels for expiration notices (HipChat, Jira, etc.). It adds default options that are required by
 all expiration notifications (interval, unit). This interface expects for the child to define the following function::
 
     def send(self, notification_type, message, targets, options, **kwargs):
@@ -281,6 +285,17 @@ The `ExportPlugin` object requires the implementation of one function::
     Support of various formats sometimes relies on external tools system calls. Always be mindful of sanitizing any input to these calls.
 
 
+Custom TLS Provider
+-------------------
+
+Managing TLS at the enterprise scale could be hard and often organizations offer custom wrapper implementations. It could
+be ideal to use those while making calls to internal services. The `TLSPlugin` would help to achieve this. It requires the
+implementation of one function which creates a TLS session::
+
+     def session(self, server_application):
+        # return active session
+
+
 Testing
 =======
 

docs/doing-a-release.rst

@@ -11,22 +11,47 @@ software.
 
 * Update the version number in ``lemur/__about__.py``.
 * Set the release date in the :doc:`/changelog`.
-* Do a commit indicating this.
-* Send a pull request with this.
+* Do a commit indicating this, and raise a pull request with this.
 * Wait for it to be merged.
 
 Performing the release
 ----------------------
 
 The commit that merged the version number bump is now the official release
-commit for this release. You will need to have ``gpg`` installed and a ``gpg``
-key in order to do a release. Once this has happened:
+commit for this release. You need an `API key <https://pypi.org/manage/account/#api-tokens>`_,
+which requires permissions to maintain the Lemur `project  <https://pypi.org/project/lemur/>`_.
 
-* Run ``invoke release {version}``.
+For creating the release, follow these steps (more details `here <https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives>`_)
 
-The release should now be available on PyPI and a tag should be available in
+* Make sure you have the latest versions of setuptools and wheel installed:
+
+    ``python3 -m pip install --user --upgrade setuptools wheel``
+
+* Now run this command from the same directory where setup.py is located:
+
+    ``python3 setup.py sdist bdist_wheel``
+
+* Once completed it should generate two files in the dist directory:
+
+.. code-block:: pycon
+
+    $ ls dist/
+    lemur-0.8.0-py2.py3-none-any.whl	lemur-0.8.0.tar.gz
+
+
+* In this step, the distribution will be uploaded. You’ll need to install Twine:
+
+    ``python3 -m pip install --user --upgrade twine``
+
+* Once installed, run Twine to upload all of the archives under dist. Once installed, run Twine to upload all of the archives under dist:
+
+    ``python3 -m twine upload --repository pypi dist/*``
+
+The release should now be available on `PyPI Lemur <https://pypi.org/project/lemur/>`_ and a tag should be available in
 the repository.
 
+Make sure to also make a github `release <https://github.com/Netflix/lemur/releases>`_ which will pick up the latest version.
+
 Verifying the release
 ---------------------
 

docs/guide/index.rst

@@ -37,18 +37,20 @@ Create a New Certificate
 
 .. figure:: create_certificate.png
 
-    Enter an owner, short description and the authority you wish to issue this certificate.
-    Enter a common name into the certificate, if no validity range is selected two years is
-    the default.
+    Enter an owner, common name, short description and certificate authority you wish to issue this certificate.
+    Depending upon the selected CA, the UI displays default validity of the certificate. You can select different
+    validity by entering a custom date, if supported by the CA.
+
+    You can also add `Subject Alternate Names` or SAN for certificates that need to include more than one domains,
+    The first domain is the Common Name and all other domains are added here as DNSName entries.
 
     You can add notification options and upload the created certificate to a destination, both
     of these are editable features and can be changed after the certificate has been created.
 
 .. figure:: certificate_extensions.png
 
-    These options are typically for advanced users, the one exception is the `Subject Alternate Names` or SAN.
-    For certificates that need to include more than one domains, the first domain is the Common Name and all
-    other domains are added here as DNSName entries.
+    These options are typically for advanced users. Lemur creates ECC based certificate (ECCPRIME256V1 in particular)
+    by default. One can change the key type using the dropdown option listed here.
 
 
 Import an Existing Certificate
@@ -58,11 +60,12 @@ Import an Existing Certificate
 
     Enter an owner, short description and public certificate. If there are intermediates and private keys
     Lemur will track them just as it does if the certificate were created through Lemur. Lemur generates
-    a certificate name but you can override that by passing a value to the `Custom Name` field.
+    a certificate name but you can override that by passing a value to the `Custom Certificate Name` field.
 
     You can add notification options and upload the created certificate to a destination, both
     of these are editable features and can be changed after the certificate has been created.
 
+.. _CreateANewUser:
 
 Create a New User
 ~~~~~~~~~~~~~~~~~

docs/license/index.rst

@@ -18,3 +18,4 @@ Lemur License
 -------------
 
 .. include:: ../../LICENSE
+    :literal:

docs/production/index.rst

@@ -49,9 +49,11 @@ The amount of effort you wish to expend ensuring that Lemur has good entropy to
 
 If you wish to generate more entropy for your system we would suggest you take a look at the following resources:
 
-- `WES-entropy-client <https://github.com/WhitewoodCrypto/WES-entropy-client>`_
+- `WES-entropy-client <https://github.com/Virginian/WES-entropy-client>`_
 - `haveged <http://www.issihosts.com/haveged/>`_
 
+The original *WES-entropy-client* repository by WhitewoodCrypto was removed, the link now points to a fork of it.
+
 For additional information about OpenSSL entropy issues:
 
 - `Managing and Understanding Entropy Usage <https://www.blackhat.com/docs/us-15/materials/us-15-Potter-Understanding-And-Managing-Entropy-Usage.pdf>`_
@@ -313,24 +315,288 @@ It will start a shell from which you can start/stop/restart the service.
 
 You can read all errors that might occur from /tmp/lemur.log.
 
+.. _PeriodicTasks:
 
 Periodic Tasks
 ==============
 
 Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
-a cron job that runs the commands.
+celery tasks or cron jobs that run these commands.
 
-There are currently three commands that could/should be run on a periodic basis:
+The following commands that could/should be run on a periodic basis:
 
-- `notify`
+- `notify expirations` `notify authority_expirations`, and `notify security_expiration_summary` (see :ref:`NotificationOptions` for configuration info)
 - `check_revoked`
 - `sync`
 
-How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
-`sync` is typically run every 15 minutes.
+If you are using LetsEncrypt, you must also run the following:
+
+- `fetch_all_pending_acme_certs`
+- `remove_old_acme_certs`
+
+How often you run these commands is largely up to the user. `notify` should be run once a day (more often will result in
+duplicate notifications). `check_revoked` is typically run at least once a day.
+`sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
+`remove_old_acme_certs` can be ran more rarely, such as once every week.
 
 Example cron entries::
 
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify security_expiration_summary
     */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
+
+
+Example Celery configuration (To be placed in your configuration file)::
+
+    CELERYBEAT_SCHEDULE = {
+        'fetch_all_pending_acme_certs': {
+            'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(minute="*"),
+        },
+        'remove_old_acme_certs': {
+            'task': 'lemur.common.celery.remove_old_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=7, minute=30, day_of_week=1),
+        },
+        'clean_all_sources': {
+            'task': 'lemur.common.celery.clean_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=1, minute=0, day_of_week=1),
+        },
+        'sync_all_sources': {
+            'task': 'lemur.common.celery.sync_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*/3", minute=5),
+        },
+        'sync_source_destination': {
+            'task': 'lemur.common.celery.sync_source_destination',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*"),
+        },
+        'notify_expirations': {
+            'task': 'lemur.common.celery.notify_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        },
+        'notify_authority_expirations': {
+            'task': 'lemur.common.celery.notify_authority_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        },
+        'send_security_expiration_summary': {
+            'task': 'lemur.common.celery.send_security_expiration_summary',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        }
+    }
+
+To enable celery support, you must also have configuration values that tell Celery which broker and backend to use.
+Here are the Celery configuration variables that should be set::
+
+    CELERY_RESULT_BACKEND = 'redis://your_redis_url:6379'
+    CELERY_BROKER_URL = 'redis://your_redis_url:6379/0'
+    CELERY_IMPORTS = ('lemur.common.celery')
+    CELERY_TIMEZONE = 'UTC'
+
+    REDIS_HOST="your_redis_url"
+    REDIS_PORT=6379
+    REDIS_DB=0
+
+Out of the box, every Redis instance supports 16 databases. The default database (`REDIS_DB`) is  set to 0, however, you can use any of the databases from 0-15. Via `redis.conf` more databases can be supported.
+In the `redis://` url, the database number can be added with a slash after the port. (defaults to 0, if omitted)
+
+Do not forget to import crontab module in your configuration file::
+
+    from celery.task.schedules import crontab
+
+You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks.
+The scheduler can be started with::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery beat
+
+And the worker can be started with desired options such as the following::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery worker --concurrency 10 -E -n lemurworker1@%%h
+
+supervisor or systemd configurations should be created for these in production environments as appropriate.
+
+Add support for LetsEncrypt/ACME
+================================
+
+LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
+for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
+LetsEncrypt requires that we prove ownership of a domain before we're able to issue a certificate for that domain, each
+time we want a certificate.
+
+The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
+through the creation of DNS TXT records as well as HTTP validation, reusing the destination concept.
+
+ACME DNS Challenge
+------------------
+
+In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
+token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
+prove ownership of all of these domains through this method. The token is typically written to a TXT record at
+-acme_challenge.domain.com. Once we create the appropriate TXT record(s), Lemur will try to validate propagation
+before requesting that LetsEncrypt finalize the certificate request and send us the certificate.
+
+.. figure:: letsencrypt_flow.png
+
+To start issuing certificates through LetsEncrypt, you must enable Celery support within Lemur first. After doing so,
+you need to create a LetsEncrypt authority. To do this, visit
+Authorities -> Create. Set the applicable attributes and click "More Options".
+
+.. figure:: letsencrypt_authority_1.png
+
+You will need to set "Certificate" to LetsEncrypt's active chain of trust for the authority you want to use. To find
+the active chain of trust at the time of writing, please visit `LetsEncrypt
+<https://letsencrypt.org/certificates/>`_.
+
+Under Acme_url, enter in the appropriate endpoint URL. Lemur supports LetsEncrypt's V2 API, and we recommend you to use
+this. At the time of writing, the staging and production URLs for LetsEncrypt V2 are
+https://acme-staging-v02.api.letsencrypt.org/directory and https://acme-v02.api.letsencrypt.org/directory.
+
+.. figure:: letsencrypt_authority_2.png
+
+After creating the authorities, we will need to create a DNS provider. Visit `Admin` -> `DNS Providers` and click
+`Create`. Lemur comes with a few provider plugins built in, with different options. Create a DNS provider with the
+appropriate choices.
+
+.. figure:: create_dns_provider.png
+
+By default, users will need to select the DNS provider that is authoritative over their domain in order for the
+LetsEncrypt flow to function. However, Lemur will attempt to automatically determine the appropriate provider if
+possible. To enable this functionality, periodically (or through Cron/Celery) run `lemur dns_providers get_all_zones`.
+This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
+Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.
+
+ACME HTTP Challenge
+-------------------
+
+The flow for requesting a certificate using the HTTP challenge is not that different from the one described for the DNS
+challenge. The only difference is, that instead of creating a DNS TXT record, a file is uploaded to a Webserver which
+serves the file at `http://<domain>/.well-known/acme-challenge/<token>`
+
+Currently the HTTP challenge also works without Celery, since it's done while creating the certificate, and doesn't
+rely on celery to create the DNS record. This will change when we implement mix & match of acme challenge types.
+
+To create a HTTP compatible Authority, you first need to create a new destination that will be used to deploy the
+challenge token. Visit `Admin` -> `Destination` and click `Create`. The path you provide for the destination needs to
+be the exact path that is called when the ACME providers calls `http://<domain>/.well-known/acme-challenge/`. The
+token part will be added dynamically by the acme_upload.
+Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge.
+
+Afterwards you can create a new certificate authority as described in the DNS challenge, but need to choose
+`Acme HTTP-01` as the plugin type, and then the destination you created beforehand.
+
+LetsEncrypt: pinning to cross-signed ICA
+----------------------------------------
+
+Let's Encrypt has been using a `cross-signed <https://letsencrypt.org/certificates/>`_ intermediate CA by DST Root CA X3,
+which is included in many older devices' TrustStore.
+
+
+Let's Encrypt is `transitioning <https://letsencrypt.org/2019/04/15/transitioning-to-isrg-root.html>`_ to use
+the intermediate CA issued by their own root (ISRG X1) starting from September 29th 2020.
+This is in preparation of concluding the initial bootstrapping of their CA, by having it cross-signed by an older CA.
+
+
+Lemur can temporarily pin to the cross-signed intermediate CA (same public/private key pair as the ICA signed by ISRG X1).
+This will prolong support for incompatible devices.
+
+The following must be added to the config file to activate the pinning (the pinning will be removed by September 2021)::
+
+    # remove or update after Mar 17 16:40:46 2021 GMT
+    IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE = "17/03/21"
+    IDENTRUST_CROSS_SIGNED_LE_ICA = """
+    -----BEGIN CERTIFICATE-----
+    MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+    MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+    DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+    SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+    GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+    AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+    q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+    SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+    Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+    a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+    /PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+    AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+    CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+    bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+    c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+    VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+    ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+    MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+    Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+    AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+    uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+    wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+    X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+    PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+    KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+    -----END CERTIFICATE-----
+    """
+
+
+.. _AcmeAccountReuse:
+
+LetsEncrypt: Using a pre-existing ACME account
+-----------------------------------------------
+
+Let's Encrypt allows reusing an existing ACME account, to create and especially revoke certificates. The current
+implementation in the acme plugin, only allows for a single account for all ACME authorities, which might be an issue,
+when you try to use Let's Encrypt together with another certificate authority that uses the ACME protocol.
+
+To use an existing account, you need to configure the `ACME_PRIVATE_KEY` and `ACME_REGR` variables in the lemur
+configuration.
+
+`ACME_PRIVATE_KEY` needs to be in the JWK format::
+
+    {
+        "kty": "RSA",
+        "n": "yr1qBwHizA7ME_iV32bY10ILp.....",
+        "e": "AQAB",
+        "d": "llBlYhil3I.....",
+        "p": "-5LW2Lewogo.........",
+        "q": "zk6dHqHfHksd.........",
+        "dp": "qfe9fFIu3mu.......",
+        "dq": "cXFO-loeOyU.......",
+        "qi": "AfK1sh0_8sLTb..........."
+    }
+
+
+Using `python-jwt` converting an existing private key in PEM format is quite easy::
+
+    import python_jwt as jwt, jwcrypto.jwk as jwk
+
+    priv_key = jwk.JWK.from_pem(b"""-----BEGIN RSA PRIVATE KEY-----
+    ...
+    -----END RSA PRIVATE KEY-----""")
+
+    print(priv_key.export())
+
+`ACME_REGR` needs to be a valid JSON with a `body` and a `uri` attribute, similar to this::
+
+    {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/<ACCOUNT_NUMBER>"}
+
+The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key.

docs/quickstart/index.rst

@@ -1,9 +1,10 @@
 Quickstart
 **********
 
-This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.  This guide assumes a clean Ubuntu 14.04 instance, commands may differ based on the OS and configuration being used.
+This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.
+This guide assumes a clean Ubuntu 18.04/20.04 instance, commands may differ based on the OS and configuration being used.
 
-Pressed for time? See the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
+For a quicker alternative, see the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
 
 
 Dependencies
@@ -11,34 +12,36 @@ Dependencies
 
 Some basic prerequisites which you'll need in order to run Lemur:
 
-* A UNIX-based operating system (we test on Ubuntu, develop on OS X)
-* Python 3.5 or greater
+* A UNIX-based operating system (we test on Ubuntu, develop on macOS)
+* Python 3.7 or greater
 * PostgreSQL 9.4 or greater
 * Nginx
+* Node v10.x (LTS)
 
-.. note:: Lemur was built with in AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
+.. note:: Ubuntu 18.04 supports by default Python 3.6.x and Node v8.x
+.. note:: Lemur was built with AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
 
 
 Installing Build Dependencies
 -----------------------------
 
-If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
+If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
 
 .. code-block:: bash
 
-    $ sudo apt-get update
-    $ sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql
+    sudo apt-get update
+    sudo apt-get install nodejs npm python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor postgresql
 
 .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver.  npm is needed if you're installing Lemur from the source (e.g., from git).
 
-.. note:: Installing node from a package manager may creat the nodejs bin at  /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
-    $ sudo ln -s /user/bin/nodejs /usr/bin/node
+.. note:: Installing node from a package manager may create the nodejs bin at  /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
+    sudo ln -s /user/bin/nodejs /usr/bin/node
 
 Now, install Python ``virtualenv`` package:
 
 .. code-block:: bash
 
-    $ sudo pip install -U virtualenv
+    sudo pip install -U virtualenv
 
 
 Setting up an Environment
@@ -48,28 +51,28 @@ In this guide, Lemur will be installed in ``/www``, so you need to create that s
 
 .. code-block:: bash
 
-    $ sudo mkdir /www
-    $ cd /www
+    sudo mkdir /www
+    cd /www
 
 Clone Lemur inside the just created directory and give yourself write permission (we assume ``lemur`` is the user):
 
 .. code-block:: bash
 
-    $ sudo useradd lemur
-    $ sudo passwd lemur
-    $ sudo mkdir /home/lemur
-    $ sudo chown lemur:lemur /home/lemur
-    $ sudo git clone https://github.com/Netflix/lemur
-    $ sudo chown -R lemur lemur/
+    sudo useradd lemur
+    sudo passwd lemur
+    sudo mkdir /home/lemur
+    sudo chown lemur:lemur /home/lemur
+    sudo git clone https://github.com/Netflix/lemur
+    sudo chown -R lemur lemur/
 
 Create the virtual environment, activate it and enter the Lemur's directory:
 
 .. code-block:: bash
 
-    $ su lemur
-    $ virtualenv -p python3 lemur
-    $ source /www/lemur/bin/activate
-    $ cd lemur
+    su lemur
+    virtualenv -p python3 lemur
+    source /www/lemur/bin/activate
+    cd lemur
 
 .. note:: Activating the environment adjusts your PATH, so that things like pip now install into the virtualenv by default.
 
@@ -81,13 +84,13 @@ Once your system is prepared, ensure that you are in the virtualenv:
 
 .. code-block:: bash
 
-  $ which python
+  which python
 
 And then run:
 
 .. code-block:: bash
 
-  $ make release
+  make release
 
 .. note:: This command will install npm dependencies as well as compile static assets.
 
@@ -101,7 +104,7 @@ You may also run with the urlContextPath variable set. If this is set it will ad
 
 .. code-block:: bash
 
-  $ make release urlContextPath={desired context path}
+  make release urlContextPath={desired context path}
 
 
 Creating a configuration
@@ -113,7 +116,7 @@ Simply run:
 
 .. code-block:: bash
 
-  $ lemur create_config
+  lemur create_config
 
 .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command.
 
@@ -127,10 +130,10 @@ Once created, you will need to update the configuration file with information ab
 
 .. code-block:: bash
 
-    $ vi ~/.lemur/lemur.conf.py
+    vi ~/.lemur/lemur.conf.py
 
 .. note:: If you are unfamiliar with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
-      ``postgresql://userame:password@<database-fqdn>:<database-port>/<database-name>``
+      ``postgresql://username:password@<database-fqdn>:<database-port>/<database-name>``
 
 Before Lemur will run you need to fill in a few required variables in the configuration file:
 
@@ -144,8 +147,8 @@ Before Lemur will run you need to fill in a few required variables in the config
     LEMUR_DEFAULT_ORGANIZATION
     LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
 
-Setup Postgres
---------------
+Set Up Postgres
+---------------
 
 For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
 
@@ -153,8 +156,8 @@ First, set a password for the postgres user.  For this guide, we will use ``lemu
 
 .. code-block:: bash
 
-    $ sudo -u postgres -i
-    $ psql
+    sudo -u postgres -i
+    psql
     postgres=# CREATE USER lemur WITH PASSWORD 'lemur';
 
 Once successful, type CTRL-D to exit the Postgres shell.
@@ -163,7 +166,7 @@ Next, we will create our new database:
 
 .. code-block:: bash
 
-    $ sudo -u postgres createdb lemur
+    sudo -u postgres createdb lemur
 
 .. _InitializingLemur:
 
@@ -180,23 +183,38 @@ Lemur provides a helpful command that will initialize your database for you. It
 
 In addition to creating a new user, Lemur also creates a few default email notifications.  These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``.  They basically guarantee that every certificate within Lemur will send one expiration notification to the security team.
 
-Additional notifications can be created through the UI or API.  See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user.
+
+.. code-block:: bash
+
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH SUPERUSER
+
+Additional notifications can be created through the UI or API.  See :ref:`Notification Options <NotificationOptions>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 **Make note of the password used as this will be used during first login to the Lemur UI.**
 
 .. code-block:: bash
 
-    $ cd /www/lemur/lemur
-    $ lemur init
+    cd /www/lemur/lemur
+    lemur init
+
+.. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now.
+
+.. code-block:: bash
+
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH NOSUPERUSER
 
 
-.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating a New User <CreateANewUser>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
+Set Up a Reverse Proxy
+----------------------
 
-Setup a Reverse Proxy
----------------------
-
-By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
+By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
 
 
 Proxying with Nginx
@@ -228,7 +246,7 @@ After making these changes, restart Nginx service to apply them:
 
 .. code-block:: bash
 
-    $ sudo service nginx restart
+    sudo service nginx restart
 
 
 Starting the Web Service
@@ -284,7 +302,7 @@ Lemur uses periodic sync tasks to make sure it is up-to-date with its environmen
 
 .. code-block:: bash
 
-  $ crontab -e
+  crontab -e
   */15 * * * * lemur sync -s all
   0 22 * * * lemur check_revoked
   0 22 * * * lemur notify
@@ -310,6 +328,12 @@ unlock
 Decrypts sensitive key material - used to decrypt the secrets stored in source during deployment.
 
 
+Automated celery tasks
+~~~~~~~~~~~~~~~~~~~~~~
+
+Please refer to :ref:`Periodic Tasks <PeriodicTasks>` to learn more about task scheduling in Lemur.
+
+
 What's Next?
 ------------
 

docs/security.rst

@@ -22,7 +22,7 @@ Supported Versions
 ------------------
 
 At any given time, we will provide security support for the `master`_ branch
-as well as the 2 most recent releases.
+as well as the most recent release.
 
 Disclosure Process
 ------------------
@@ -30,20 +30,15 @@ Disclosure Process
 Our process for taking a security issue from private discussion to public
 disclosure involves multiple steps.
 
-Approximately one week before full public disclosure, we will send advance
-notification of the issue to a list of people and organizations, primarily
-composed of operating-system vendors and other distributors of
-``lemur``.  This notification will consist of an email message
-containing:
+Approximately one week before full public disclosure, we will provide advanced notification that a security issue exists. Depending on the severity of the issue, we may choose to either send a targeted email to known Lemur users and contributors or post an issue to the Lemur repository.  In either case, the notification should contain the following.
 
-* A full description of the issue and the affected versions of
-  ``lemur``.
+* A description of the potential impact
+* The affected versions of ``lemur``.
 * The steps we will be taking to remedy the issue.
-* The patches, if any, that will be applied to ``lemur``.
 * The date on which the ``lemur`` team will apply these patches, issue
   new releases, and publicly disclose the issue.
 
-Simultaneously, the reporter of the issue will receive notification of the date
+If the issue was disclosed to us, the reporter will receive notification of the date
 on which we plan to make the issue public.
 
 On the day of disclosure, we will take the following steps:
@@ -52,7 +47,7 @@ On the day of disclosure, we will take the following steps:
   messages for these patches will indicate that they are for security issues,
   but will not describe the issue in any detail; instead, they will warn of
   upcoming disclosure.
-* Issue the relevant releases.
+* Issue an updated release.
 
 If a reported issue is believed to be particularly time-sensitive – due to a
 known exploit in the wild, for example – the time between advance notification

gulp/build.js

@@ -21,7 +21,6 @@ var gulp = require('gulp'),
   useref = require('gulp-useref'),
   filter = require('gulp-filter'),
   rev = require('gulp-rev'),
-  revReplace = require('gulp-rev-replace'),
   imagemin = require('gulp-imagemin'),
   minifyHtml = require('gulp-minify-html'),
   bowerFiles = require('main-bower-files'),
@@ -29,52 +28,77 @@ var gulp = require('gulp'),
   replace = require('gulp-replace'),
   argv = require('yargs').argv;
 
-gulp.task('default', ['clean'], function () {
-  gulp.start('fonts', 'styles');
+// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
+function escapeRegExp(string) {
+  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
+}
+
+function replaceAll(string, find, replace) {
+  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
+}
+
+function stringSrc(filename, string) {
+  let src = require('stream').Readable({objectMode: true});
+  src._read = function () {
+    this.push(new gutil.File({cwd: '', base: '', path: filename, contents: Buffer.from(string)}));
+    this.push(null);
+  };
+  return src;
+}
+
+gulp.task('clean', function (done) {
+  del(['.tmp', 'lemur/static/dist'], done);
+  done();
 });
 
-gulp.task('clean', function (cb) {
-  del(['.tmp', 'lemur/static/dist'], cb);
-});
+gulp.task('default', gulp.series(['clean'], function () {
+  gulp.start('fonts', 'styles');
+}));
 
 gulp.task('test', function (done) {
   new karma.Server({
     configFile: __dirname + '/karma.conf.js',
     singleRun: true
-  }, function() {
+  }, function (err) {
+    if (err === 0) {
       done();
+    } else {
+      // if karma server failed to start raise error
+      done(new gutil.PluginError('karma', {
+        message: 'Karma Tests failed'
+      }));
+    }
   }).start();
 });
 
 gulp.task('dev:fonts', function () {
-  var fileList = [
+  let fileList = [
     'bower_components/bootstrap/dist/fonts/*',
     'bower_components/fontawesome/fonts/*'
   ];
 
   return gulp.src(fileList)
-    .pipe(gulp.dest('.tmp/fonts'));
+    .pipe(gulp.dest('.tmp/fonts')); // returns a stream making it async
 });
 
 gulp.task('dev:styles', function () {
-  var baseContent = '@import "bower_components/bootstrap/less/bootstrap.less";@import "bower_components/bootswatch/$theme$/variables.less";@import "bower_components/bootswatch/$theme$/bootswatch.less";@import "bower_components/bootstrap/less/utilities.less";';
-  var isBootswatchFile = function (file) {
+  let baseContent = '@import "bower_components/bootstrap/less/bootstrap.less";@import "bower_components/bootswatch/$theme$/variables.less";@import "bower_components/bootswatch/$theme$/bootswatch.less";@import "bower_components/bootstrap/less/utilities.less";';
+  let isBootswatchFile = function (file) {
 
-    var suffix = 'bootswatch.less';
+    let suffix = 'bootswatch.less';
     return file.path.indexOf(suffix, file.path.length - suffix.length) !== -1;
   };
 
-  var isBootstrapFile = function (file) {
-    var suffix = 'bootstrap-',
+  let isBootstrapFile = function (file) {
+    let suffix = 'bootstrap-',
       fileName = path.basename(file.path);
 
     return fileName.indexOf(suffix) === 0;
   };
 
-  var fileList = [
+  let fileList = [
     'bower_components/bootswatch/sandstone/bootswatch.less',
     'bower_components/fontawesome/css/font-awesome.css',
-    'bower_components/angular-spinkit/src/angular-spinkit.css',
     'bower_components/angular-chart.js/dist/angular-chart.css',
     'bower_components/angular-loading-bar/src/loading-bar.css',
     'bower_components/angular-ui-switch/angular-ui-switch.css',
@@ -87,20 +111,18 @@ gulp.task('dev:styles', function () {
 
   return gulp.src(fileList)
     .pipe(gulpif(isBootswatchFile, foreach(function (stream, file) {
-      var themeName = path.basename(path.dirname(file.path)),
-        content = replaceAll(baseContent, '$theme$', themeName),
-        file2 = string_src('bootstrap-' +  themeName + '.less', content);
-
-      return file2;
+      let themeName = path.basename(path.dirname(file.path)),
+        content = replaceAll(baseContent, '$theme$', themeName);
+      return stringSrc('bootstrap-' + themeName + '.less', content);
     })))
     .pipe(less())
     .pipe(gulpif(isBootstrapFile, foreach(function (stream, file) {
-      var fileName = path.basename(file.path),
+      let fileName = path.basename(file.path),
         themeName = fileName.substring(fileName.indexOf('-') + 1, fileName.indexOf('.'));
 
       // http://stackoverflow.com/questions/21719833/gulp-how-to-add-src-files-in-the-middle-of-a-pipe
       // https://github.com/gulpjs/gulp/blob/master/docs/recipes/using-multiple-sources-in-one-task.md
-      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css']))
+      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css'], {allowEmpty: true}))
         .pipe(concat('style-' + themeName + '.css'));
     })))
     .pipe(plumber())
@@ -111,24 +133,6 @@ gulp.task('dev:styles', function () {
     .pipe(size());
 });
 
-// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
-function escapeRegExp(string) {
-  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
-}
-
-function replaceAll(string, find, replace) {
-  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
-}
-
-function string_src(filename, string) {
-  var src = require('stream').Readable({ objectMode: true });
-  src._read = function () {
-    this.push(new gutil.File({ cwd: '', base: '', path: filename, contents: new Buffer(string) }));
-    this.push(null);
-  };
-  return src;
-}
-
 gulp.task('dev:scripts', function () {
   return gulp.src(['lemur/static/app/angular/**/*.js'])
     .pipe(jshint())
@@ -162,7 +166,7 @@ function injectHtml(isDev) {
     }))
     .pipe(
       gulpif(!isDev,
-      inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js'), {
+        inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js', {allowEmpty: true}), {
           starttag: '<!-- inject:ngviews -->',
           addRootSlash: false
         })
@@ -170,13 +174,9 @@ function injectHtml(isDev) {
     ).pipe(gulp.dest('.tmp/'));
 }
 
-gulp.task('dev:inject', ['dev:styles', 'dev:scripts'], function () {
+gulp.task('dev:inject', gulp.series(['dev:styles', 'dev:scripts'], function () {
   return injectHtml(true);
-});
-
-gulp.task('build:inject', ['dev:styles', 'dev:scripts', 'build:ngviews'], function () {
-  return injectHtml(false);
-});
+}));
 
 gulp.task('build:ngviews', function () {
   return gulp.src(['lemur/static/app/angular/**/*.html'])
@@ -189,9 +189,13 @@ gulp.task('build:ngviews', function () {
     .pipe(size());
 });
 
-gulp.task('build:html', ['dev:styles', 'dev:scripts', 'build:ngviews', 'build:inject'], function () {
-  var jsFilter = filter(['**/*.js'], {'restore': true});
-  var cssFilter = filter(['**/*.css'], {'restore': true});
+gulp.task('build:inject', gulp.series(['dev:styles', 'dev:scripts', 'build:ngviews'], function () {
+  return injectHtml(false);
+}));
+
+gulp.task('build:html', gulp.series(['build:inject'], function () {
+  let jsFilter = filter(['**/*.js'], {'restore': true});
+  let cssFilter = filter(['**/*.css'], {'restore': true});
 
   return gulp.src('.tmp/index.html')
     .pipe(jsFilter)
@@ -203,12 +207,12 @@ gulp.task('build:html', ['dev:styles', 'dev:scripts', 'build:ngviews', 'build:in
     .pipe(useref())
     .pipe(gulp.dest('lemur/static/dist'))
     .pipe(size());
-});
+}));
 
-gulp.task('build:fonts', ['dev:fonts'], function () {
+gulp.task('build:fonts', gulp.series(['dev:fonts'], function () {
   return gulp.src('.tmp/fonts/**/*')
     .pipe(gulp.dest('lemur/static/dist/fonts'));
-});
+}));
 
 gulp.task('build:images', function () {
   return gulp.src('lemur/static/app/images/**/*')
@@ -230,36 +234,28 @@ gulp.task('package:strip', function () {
     .pipe(size());
 });
 
-gulp.task('addUrlContextPath',['addUrlContextPath:revreplace'], function(){
-  var urlContextPathExists = argv.urlContextPath ? true : false;
-  ['lemur/static/dist/scripts/main*.js',
-  'lemur/static/dist/angular/**/*.html']
-  .forEach(function(file){
-    return gulp.src(file)
-      .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
-      .pipe(gulpif(urlContextPathExists, replace('angular/', argv.urlContextPath + '/angular/')))
-      .pipe(gulp.dest(function(file){
-        return file.base;
-      }))
-  })
-});
-
 gulp.task('addUrlContextPath:revision', function () {
   return gulp.src(['lemur/static/dist/**/*.css', 'lemur/static/dist/**/*.js'])
     .pipe(rev())
     .pipe(gulp.dest('lemur/static/dist'))
     .pipe(rev.manifest())
-    .pipe(gulp.dest('lemur/static/dist'))
-})
-
-gulp.task('addUrlContextPath:revreplace', ['addUrlContextPath:revision'], function(){
-  var manifest = gulp.src("lemur/static/dist/rev-manifest.json");
-  var urlContextPathExists = argv.urlContextPath ? true : false;
-  return gulp.src( "lemur/static/dist/index.html")
-    .pipe(gulpif(urlContextPathExists, revReplace({prefix: argv.urlContextPath + '/', manifest: manifest}, revReplace({manifest: manifest}))))
     .pipe(gulp.dest('lemur/static/dist'));
-})
+});
 
+gulp.task('addUrlContextPath:revreplace', gulp.series(['addUrlContextPath:revision'], function () {
+  return gulp.src('lemur/static/dist/index.html')
+    .pipe(gulp.dest('lemur/static/dist'));
+}));
 
-gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']);
-gulp.task('package', ['addUrlContextPath', 'package:strip']);
+gulp.task('addUrlContextPath', gulp.series(['addUrlContextPath:revreplace'], function () {
+  let urlContextPathExists = !!argv.urlContextPath;
+  return gulp.src(['lemur/static/dist/scripts/main*.js', 'lemur/static/dist/angular/**/*.html'])
+    .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
+    .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/')))
+    .pipe(gulp.dest(function (file) {
+      return file.base;
+    }));
+}));
+
+gulp.task('build', gulp.series(['build:images', 'build:fonts', 'build:html', 'build:extras']));
+gulp.task('package', gulp.series(['addUrlContextPath', 'package:strip']));

gulp/karma.conf.js

@@ -1,10 +1,12 @@
 // Contents of: config/karma.conf.js
+'use strict';
+
 module.exports = function (config) {
   config.set({
     basePath: '../',
 
     // Fix for "JASMINE is not supported anymore" warning
-    frameworks : ["jasmine"],
+    frameworks: ['jasmine'],
 
     files: [
       'app/lib/angular/angular.js',
@@ -16,12 +18,20 @@ module.exports = function (config) {
 
     autoWatch: true,
 
-    browsers : ['Chrome'],
+    browsers: [process.env.TRAVIS ? 'Chrome_travis_ci' : 'Chrome'],
+    customLaunchers: {
+      'Chrome_travis_ci': {
+        base: 'Chrome',
+        flags: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu',],
+      },
+    },
 
     junitReporter: {
       outputFile: 'test_out/unit.xml',
       suite: 'unit'
       //...
-    }
+    },
+
+    failOnEmptyTestSuite: false,
   });
 };

gulp/server.js

@@ -1,14 +1,15 @@
 'use strict';
 
 var gulp = require('gulp');
+const watch = require('./watch')
 
 var browserSync = require('browser-sync');
 var httpProxy = require('http-proxy');
 
 /* This configuration allow you to configure browser sync to proxy your backend */
-/*
- var proxyTarget = 'http://localhost/context/'; // The location of your backend
- var proxyApiPrefix = 'api'; // The element in the URL which differentiate between API request and static file request
+
+ var proxyTarget = 'http://localhost:8000/'; // The location of your backend
+ var proxyApiPrefix = '/api/'; // The element in the URL which differentiate between API request and static file request
  var proxy = httpProxy.createProxyServer({
      target: proxyTarget
  });
@@ -19,7 +20,6 @@ var httpProxy = require('http-proxy');
          next();
      }
  }
- */
 
 function browserSyncInit(baseDir, files, browser) {
   browser = browser === undefined ? 'default' : browser;
@@ -27,6 +27,7 @@ function browserSyncInit(baseDir, files, browser) {
   browserSync.instance = browserSync.init(files, {
     startPath: '/index.html',
       server: {
+          middleware: [proxyMiddleware],
           baseDir: baseDir,
           routes: {
               '/bower_components': './bower_components'
@@ -38,7 +39,7 @@ function browserSyncInit(baseDir, files, browser) {
 
 }
 
-gulp.task('serve', ['watch'], function () {
+gulp.task('serve', gulp.series(['watch'], function (done) {
   browserSyncInit([
     '.tmp',
     'lemur/static/app'
@@ -51,9 +52,12 @@ gulp.task('serve', ['watch'], function () {
     'lemur/static/app/angular/**/*',
     'lemur/static/app/index.html'
   ]);
-});
+
+  done();
+}));
 
 
-gulp.task('serve:dist', ['build'], function () {
+gulp.task('serve:dist', gulp.series(['build'], function (done) {
   browserSyncInit('lemur/static/dist');
-});
+  done();
+}));

gulp/watch.js

@@ -3,10 +3,13 @@
 var gulp = require('gulp');
 
 
-gulp.task('watch', ['dev:styles', 'dev:scripts', 'dev:inject', 'dev:fonts'] ,function () {
-  gulp.watch('app/styles/**/*.less', ['dev:styles']);
-  gulp.watch('app/styles/**/*.css', ['dev:styles']);
-  gulp.watch('app/**/*.js', ['dev:scripts']);
-  gulp.watch('app/images/**/*', ['build:images']);
-  gulp.watch('bower.json', ['dev:inject']);
-});
+const watch = gulp.task('watch', gulp.series(['dev:inject', 'dev:fonts'] ,function (done) {
+  gulp.watch('app/styles/**/*.less', gulp.series('dev:styles'));
+  gulp.watch('app/styles/**/*.css', gulp.series('dev:styles'));
+  gulp.watch('app/**/*.js', gulp.series('dev:scripts'));
+  gulp.watch('app/images/**/*', gulp.series('build:images'));
+  gulp.watch('bower.json', gulp.series('dev:inject'));
+  done();
+}));
+
+module.exports = {watch:watch}

lemur/__about__.py

@@ -1,15 +1,21 @@
 from __future__ import absolute_import, division, print_function
 
 __all__ = [
-    "__title__", "__summary__", "__uri__", "__version__", "__author__",
-    "__email__", "__license__", "__copyright__",
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
 ]
 
 __title__ = "lemur"
-__summary__ = ("Certificate management and orchestration service")
+__summary__ = "Certificate management and orchestration service"
 __uri__ = "https://github.com/Netflix/lemur"
 
-__version__ = "0.7.0"
+__version__ = "develop"
 
 __author__ = "The Lemur developers"
 __email__ = "security@netflix.com"

lemur/__init__.py

@@ -5,7 +5,8 @@
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-
+.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
+.. moduleauthor:: Hossein Shafagh <hshafagh@netflix.com>
 
 """
 import time
@@ -32,14 +33,26 @@ from lemur.pending_certificates.views import mod as pending_certificates_bp
 from lemur.dns_providers.views import mod as dns_providers_bp
 
 from lemur.__about__ import (
-    __author__, __copyright__, __email__, __license__, __summary__, __title__,
-    __uri__, __version__
+    __author__,
+    __copyright__,
+    __email__,
+    __license__,
+    __summary__,
+    __title__,
+    __uri__,
+    __version__,
 )
 
 
 __all__ = [
-    "__title__", "__summary__", "__uri__", "__version__", "__author__",
-    "__email__", "__license__", "__copyright__",
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
 ]
 
 LEMUR_BLUEPRINTS = (
@@ -62,8 +75,10 @@ LEMUR_BLUEPRINTS = (
 )
 
 
-def create_app(config=None):
-    app = factory.create_app(app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config)
+def create_app(config_path=None):
+    app = factory.create_app(
+        app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config_path
+    )
     configure_hook(app)
     return app
 
@@ -93,7 +108,7 @@ def configure_hook(app):
     @app.after_request
     def after_request(response):
         # Return early if we don't have the start time
-        if not hasattr(g, 'request_start_time'):
+        if not hasattr(g, "request_start_time"):
             return response
 
         # Get elapsed time in milliseconds
@@ -102,12 +117,12 @@ def configure_hook(app):
 
         # Collect request/response tags
         tags = {
-            'endpoint': request.endpoint,
-            'request_method': request.method.lower(),
-            'status_code': response.status_code
+            "endpoint": request.endpoint,
+            "request_method": request.method.lower(),
+            "status_code": response.status_code,
         }
 
         # Record our response time metric
-        metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags)
-        metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
+        metrics.send("response_time", "TIMER", elapsed, metric_tags=tags)
+        metrics.send("status_code_{}".format(response.status_code), "counter", 1)
         return response

lemur/acme_providers/cli.py

@@ -0,0 +1,191 @@
+import time
+import json
+import arrow
+
+from flask_script import Manager
+from flask import current_app
+
+from lemur.extensions import sentry
+from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.plugins import plugins
+from lemur.plugins.lemur_acme.plugin import AcmeHandler
+from lemur.plugins.lemur_aws import s3
+
+manager = Manager(
+    usage="Handles all ACME related tasks"
+)
+
+
+@manager.option(
+    "-d",
+    "--domain",
+    dest="domain",
+    required=True,
+    help="Name of the Domain to store to (ex. \"_acme-chall.test.com\".",
+)
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    required=True,
+    help="Value of the Token to store in DNS as content.",
+)
+def dnstest(domain, token):
+    """
+    Create, verify, and delete DNS TXT records using an autodetected provider.
+    """
+    print("[+] Starting ACME Tests.")
+    change_id = (domain, token)
+
+    acme_handler = AcmeHandler()
+    acme_handler.autodetect_dns_providers(domain)
+    if not acme_handler.dns_providers_for_domain[domain]:
+        raise Exception(f"No DNS providers found for domain: {format(domain)}.")
+
+    # Create TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        print(f"[+] Creating TXT Record in `{dns_provider.name}` provider")
+        change_id = dns_provider_plugin.create_txt_record(domain, token, account_number)
+
+    print("[+] Verifying TXT Record has propagated to DNS.")
+    print("[+] This step could take a while...")
+    time.sleep(10)
+
+    # Verify TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        try:
+            dns_provider_plugin.wait_for_dns_change(change_id, account_number)
+            print(f"[+] Verified TXT Record in `{dns_provider.name}` provider")
+        except Exception:
+            sentry.captureException()
+            current_app.logger.debug(
+                f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: "
+                f"{account_number}",
+                exc_info=True,
+            )
+            print(f"[+] Unable to Verify TXT Record in `{dns_provider.name}` provider")
+
+    time.sleep(10)
+
+    # Delete TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        # TODO(csine@: Add Exception Handling
+        dns_provider_plugin.delete_txt_record(change_id, account_number, domain, token)
+        print(f"[+] Deleted TXT Record in `{dns_provider.name}` provider")
+
+    status = SUCCESS_METRIC_STATUS
+    print("[+] Done with ACME Tests.")
+
+
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="Value of the Token",
+)
+@manager.option(
+    "-n",
+    "--token_name",
+    dest="token_name",
+    default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="path",
+)
+@manager.option(
+    "-p",
+    "--prefix",
+    dest="prefix",
+    default="test/",
+    required=False,
+    help="S3 bucket prefix",
+)
+@manager.option(
+    "-a",
+    "--account_number",
+    dest="account_number",
+    required=True,
+    help="AWS Account",
+)
+@manager.option(
+    "-b",
+    "--bucket_name",
+    dest="bucket_name",
+    required=True,
+    help="Bucket Name",
+)
+def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
+    """
+    This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
+    It mainly serves for testing purposes.
+    :param token:
+    :param token_name:
+    :param prefix:
+    :param account_number:
+    :param bucket_name:
+    :return:
+    """
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket_name,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account_number,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    p = plugins.get("aws-s3")
+    p.upload_acme_token(token_name, token, additional_options)
+
+    if not prefix.endswith("/"):
+        prefix + "/"
+
+    token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
+    assert(token_res == token)
+    s3.delete(bucket_name, prefix + token_name, account_number=account_number)

lemur/api_keys/cli.py

@@ -14,23 +14,32 @@ from datetime import datetime
 manager = Manager(usage="Handles all api key related tasks.")
 
 
-@manager.option('-u', '--user-id', dest='uid', help='The User ID this access key belongs too.')
-@manager.option('-n', '--name', dest='name', help='The name of this API Key.')
-@manager.option('-t', '--ttl', dest='ttl', help='The TTL of this API Key. -1 for forever.')
+@manager.option(
+    "-u", "--user-id", dest="uid", help="The User ID this access key belongs too."
+)
+@manager.option("-n", "--name", dest="name", help="The name of this API Key.")
+@manager.option(
+    "-t", "--ttl", dest="ttl", help="The TTL of this API Key. -1 for forever."
+)
 def create(uid, name, ttl):
     """
     Create a new api key for a user.
     :return:
     """
     print("[+] Creating a new api key.")
-    key = api_key_service.create(user_id=uid, name=name,
-        ttl=ttl, issued_at=int(datetime.utcnow().timestamp()), revoked=False)
+    key = api_key_service.create(
+        user_id=uid,
+        name=name,
+        ttl=ttl,
+        issued_at=int(datetime.utcnow().timestamp()),
+        revoked=False,
+    )
     print("[+] Successfully created a new api key. Generating a JWT...")
     jwt = create_token(uid, key.id, key.ttl)
     print("[+] Your JWT is: {jwt}".format(jwt=jwt))
 
 
-@manager.option('-a', '--api-key-id', dest='aid', help='The API Key ID to revoke.')
+@manager.option("-a", "--api-key-id", dest="aid", help="The API Key ID to revoke.")
 def revoke(aid):
     """
     Revokes an api key for a user.

lemur/api_keys/models.py

@@ -12,14 +12,19 @@ from lemur.database import db
 
 
 class ApiKey(db.Model):
-    __tablename__ = 'api_keys'
+    __tablename__ = "api_keys"
     id = Column(Integer, primary_key=True)
     name = Column(String)
-    user_id = Column(Integer, ForeignKey('users.id'))
+    user_id = Column(Integer, ForeignKey("users.id"))
     ttl = Column(BigInteger)
     issued_at = Column(BigInteger)
     revoked = Column(Boolean)
 
     def __repr__(self):
         return "ApiKey(name={name}, user_id={user_id}, ttl={ttl}, issued_at={iat}, revoked={revoked})".format(
-            user_id=self.user_id, name=self.name, ttl=self.ttl, iat=self.issued_at, revoked=self.revoked)
+            user_id=self.user_id,
+            name=self.name,
+            ttl=self.ttl,
+            iat=self.issued_at,
+            revoked=self.revoked,
+        )

lemur/api_keys/schemas.py

@@ -13,12 +13,18 @@ from lemur.users.schemas import UserNestedOutputSchema, UserInputSchema
 
 
 def current_user_id():
-    return {'id': g.current_user.id, 'email': g.current_user.email, 'username': g.current_user.username}
+    return {
+        "id": g.current_user.id,
+        "email": g.current_user.email,
+        "username": g.current_user.username,
+    }
 
 
 class ApiKeyInputSchema(LemurInputSchema):
     name = fields.String(required=False)
-    user = fields.Nested(UserInputSchema, missing=current_user_id, default=current_user_id)
+    user = fields.Nested(
+        UserInputSchema, missing=current_user_id, default=current_user_id
+    )
     ttl = fields.Integer()
 
 

lemur/api_keys/service.py

@@ -7,6 +7,7 @@
 """
 from lemur import database
 from lemur.api_keys.models import ApiKey
+from lemur.logs import service as log_service
 
 
 def get(aid):
@@ -24,6 +25,7 @@ def delete(access_key):
     :param access_key:
     :return:
     """
+    log_service.audit_log("delete_api_key", access_key.name, "Deleting the API key")
     database.delete(access_key)
 
 
@@ -34,8 +36,9 @@ def revoke(aid):
     :return:
     """
     api_key = get(aid)
-    setattr(api_key, 'revoked', False)
+    setattr(api_key, "revoked", True)
 
+    log_service.audit_log("revoke_api_key", api_key.name, "Revoking API key")
     return database.update(api_key)
 
 
@@ -55,6 +58,9 @@ def create(**kwargs):
     :return:
     """
     api_key = ApiKey(**kwargs)
+    # this logs only metadata about the api key
+    log_service.audit_log("create_api_key", api_key.name, f"Creating the API key {api_key}")
+
     database.create(api_key)
     return api_key
 
@@ -69,6 +75,7 @@ def update(api_key, **kwargs):
     for key, value in kwargs.items():
         setattr(api_key, key, value)
 
+    log_service.audit_log("update_api_key", api_key.name, f"Update summary - {kwargs}")
     return database.update(api_key)
 
 
@@ -80,10 +87,10 @@ def render(args):
     :return:
     """
     query = database.session_query(ApiKey)
-    user_id = args.pop('user_id', None)
-    aid = args.pop('id', None)
-    has_permission = args.pop('has_permission', False)
-    requesting_user_id = args.pop('requesting_user_id')
+    user_id = args.pop("user_id", None)
+    aid = args.pop("id", None)
+    has_permission = args.pop("has_permission", False)
+    requesting_user_id = args.pop("requesting_user_id")
 
     if user_id:
         query = query.filter(ApiKey.user_id == user_id)

lemur/api_keys/views.py

@@ -19,10 +19,16 @@ from lemur.auth.permissions import ApiKeyCreatorPermission
 from lemur.common.schema import validate_schema
 from lemur.common.utils import paginated_parser
 
-from lemur.api_keys.schemas import api_key_input_schema, api_key_revoke_schema, api_key_output_schema, \
-    api_keys_output_schema, api_key_described_output_schema, user_api_key_input_schema
+from lemur.api_keys.schemas import (
+    api_key_input_schema,
+    api_key_revoke_schema,
+    api_key_output_schema,
+    api_keys_output_schema,
+    api_key_described_output_schema,
+    user_api_key_input_schema,
+)
 
-mod = Blueprint('api_keys', __name__)
+mod = Blueprint("api_keys", __name__)
 api = Api(mod)
 
 
@@ -81,8 +87,8 @@ class ApiKeyList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['has_permission'] = ApiKeyCreatorPermission().can()
-        args['requesting_user_id'] = g.current_user.id
+        args["has_permission"] = ApiKeyCreatorPermission().can()
+        args["requesting_user_id"] = g.current_user.id
         return service.render(args)
 
     @validate_schema(api_key_input_schema, api_key_output_schema)
@@ -99,6 +105,7 @@ class ApiKeyList(AuthenticatedResource):
               POST /keys HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "name": "my custom name",
@@ -124,12 +131,26 @@ class ApiKeyList(AuthenticatedResource):
            :statuscode 403: unauthenticated
         """
         if not ApiKeyCreatorPermission().can():
-            if data['user']['id'] != g.current_user.id:
-                return dict(message="You are not authorized to create tokens for: {0}".format(data['user']['username'])), 403
+            if data["user"]["id"] != g.current_user.id:
+                return (
+                    dict(
+                        message="You are not authorized to create tokens for: {0}".format(
+                            data["user"]["username"]
+                        )
+                    ),
+                    403,
+                )
 
-        access_token = service.create(name=data['name'], user_id=data['user']['id'], ttl=data['ttl'],
-                                      revoked=False, issued_at=int(datetime.utcnow().timestamp()))
-        return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
+        access_token = service.create(
+            name=data["name"],
+            user_id=data["user"]["id"],
+            ttl=data["ttl"],
+            revoked=False,
+            issued_at=int(datetime.utcnow().timestamp()),
+        )
+        return dict(
+            jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
+        )
 
 
 class ApiKeyUserList(AuthenticatedResource):
@@ -186,9 +207,9 @@ class ApiKeyUserList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['has_permission'] = ApiKeyCreatorPermission().can()
-        args['requesting_user_id'] = g.current_user.id
-        args['user_id'] = user_id
+        args["has_permission"] = ApiKeyCreatorPermission().can()
+        args["requesting_user_id"] = g.current_user.id
+        args["user_id"] = user_id
         return service.render(args)
 
     @validate_schema(user_api_key_input_schema, api_key_output_schema)
@@ -205,6 +226,7 @@ class ApiKeyUserList(AuthenticatedResource):
               POST /users/1/keys HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "name": "my custom name"
@@ -230,11 +252,25 @@ class ApiKeyUserList(AuthenticatedResource):
         """
         if not ApiKeyCreatorPermission().can():
             if user_id != g.current_user.id:
-                return dict(message="You are not authorized to create tokens for: {0}".format(user_id)), 403
+                return (
+                    dict(
+                        message="You are not authorized to create tokens for: {0}".format(
+                            user_id
+                        )
+                    ),
+                    403,
+                )
 
-        access_token = service.create(name=data['name'], user_id=user_id, ttl=data['ttl'],
-                                      revoked=False, issued_at=int(datetime.utcnow().timestamp()))
-        return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
+        access_token = service.create(
+            name=data["name"],
+            user_id=user_id,
+            ttl=data["ttl"],
+            revoked=False,
+            issued_at=int(datetime.utcnow().timestamp()),
+        )
+        return dict(
+            jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
+        )
 
 
 class ApiKeys(AuthenticatedResource):
@@ -298,6 +334,7 @@ class ApiKeys(AuthenticatedResource):
               PUT /keys/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                   "name": "new_name",
@@ -329,7 +366,9 @@ class ApiKeys(AuthenticatedResource):
             if not ApiKeyCreatorPermission().can():
                 return dict(message="You are not authorized to update this token!"), 403
 
-        service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
+        service.update(
+            access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
+        )
         return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
 
     def delete(self, aid):
@@ -371,7 +410,7 @@ class ApiKeys(AuthenticatedResource):
                 return dict(message="You are not authorized to delete this token!"), 403
 
         service.delete(access_key)
-        return {'result': True}
+        return {"result": True}
 
 
 class UserApiKeys(AuthenticatedResource):
@@ -438,6 +477,7 @@ class UserApiKeys(AuthenticatedResource):
               PUT /users/1/keys/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                   "name": "new_name",
@@ -472,7 +512,9 @@ class UserApiKeys(AuthenticatedResource):
         if access_key.user_id != uid:
             return dict(message="You are not authorized to update this token!"), 403
 
-        service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
+        service.update(
+            access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
+        )
         return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
 
     def delete(self, uid, aid):
@@ -517,7 +559,7 @@ class UserApiKeys(AuthenticatedResource):
             return dict(message="You are not authorized to delete this token!"), 403
 
         service.delete(access_key)
-        return {'result': True}
+        return {"result": True}
 
 
 class ApiKeysDescribed(AuthenticatedResource):
@@ -572,8 +614,12 @@ class ApiKeysDescribed(AuthenticatedResource):
         return access_key
 
 
-api.add_resource(ApiKeyList, '/keys', endpoint='api_keys')
-api.add_resource(ApiKeys, '/keys/<int:aid>', endpoint='api_key')
-api.add_resource(ApiKeysDescribed, '/keys/<int:aid>/described', endpoint='api_key_described')
-api.add_resource(ApiKeyUserList, '/users/<int:user_id>/keys', endpoint='user_api_keys')
-api.add_resource(UserApiKeys, '/users/<int:uid>/keys/<int:aid>', endpoint='user_api_key')
+api.add_resource(ApiKeyList, "/keys", endpoint="api_keys")
+api.add_resource(ApiKeys, "/keys/<int:aid>", endpoint="api_key")
+api.add_resource(
+    ApiKeysDescribed, "/keys/<int:aid>/described", endpoint="api_key_described"
+)
+api.add_resource(ApiKeyUserList, "/users/<int:user_id>/keys", endpoint="user_api_keys")
+api.add_resource(
+    UserApiKeys, "/users/<int:uid>/keys/<int:aid>", endpoint="user_api_key"
+)

lemur/auth/ldap.py

@@ -14,34 +14,41 @@ from lemur.roles import service as role_service
 from lemur.common.utils import validate_conf, get_psuedo_random_string
 
 
-class LdapPrincipal():
+class LdapPrincipal:
     """
     Provides methods for authenticating against an LDAP server.
     """
+
     def __init__(self, args):
         self._ldap_validate_conf()
         # setup ldap config
-        if not args['username']:
+        if not args["username"]:
             raise Exception("missing ldap username")
-        if not args['password']:
+        if not args["password"]:
             self.error_message = "missing ldap password"
             raise Exception("missing ldap password")
-        self.ldap_principal = args['username']
+        self.ldap_principal = args["username"]
         self.ldap_email_domain = current_app.config.get("LDAP_EMAIL_DOMAIN", None)
-        if '@' not in self.ldap_principal:
-            self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
-        self.ldap_username = args['username']
-        if '@' in self.ldap_username:
-            self.ldap_username = args['username'].split("@")[0]
-        self.ldap_password = args['password']
-        self.ldap_server = current_app.config.get('LDAP_BIND_URI', None)
+        if "@" not in self.ldap_principal:
+            self.ldap_principal = "%s@%s" % (
+                self.ldap_principal,
+                self.ldap_email_domain,
+            )
+        self.ldap_username = args["username"]
+        if "@" in self.ldap_username:
+            self.ldap_username = args["username"].split("@")[0]
+        self.ldap_password = args["password"]
+        self.ldap_server = current_app.config.get("LDAP_BIND_URI", None)
         self.ldap_base_dn = current_app.config.get("LDAP_BASE_DN", None)
         self.ldap_use_tls = current_app.config.get("LDAP_USE_TLS", False)
         self.ldap_cacert_file = current_app.config.get("LDAP_CACERT_FILE", None)
         self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None)
         self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None)
         self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None)
-        self.ldap_attrs = ['memberOf']
+        self.ldap_is_active_directory = current_app.config.get(
+            "LDAP_IS_ACTIVE_DIRECTORY", False
+        )
+        self.ldap_attrs = ["memberOf"]
         self.ldap_client = None
         self.ldap_groups = None
 
@@ -59,8 +66,8 @@ class LdapPrincipal():
                 get_psuedo_random_string(),
                 self.ldap_principal,
                 True,
-                '',     # thumbnailPhotoUrl
-                list(roles)
+                "",  # thumbnailPhotoUrl
+                list(roles),
             )
         else:
             # we add 'lemur' specific roles, so they do not get marked as removed
@@ -75,7 +82,7 @@ class LdapPrincipal():
                 self.ldap_principal,
                 user.active,
                 user.profile_picture,
-                list(roles)
+                list(roles),
             )
         return user
 
@@ -98,15 +105,18 @@ class LdapPrincipal():
             role = role_service.get_by_name(self.ldap_default_role)
             if role:
                 if not role.third_party:
-                    role = role.set_third_party(role.id, third_party_status=True)
+                    role = role_service.set_third_party(role.id, third_party_status=True)
                 roles.add(role)
 
         # update their 'roles'
         role = role_service.get_by_name(self.ldap_principal)
         if not role:
-            description = "auto generated role based on owner: {0}".format(self.ldap_principal)
-            role = role_service.create(self.ldap_principal, description=description,
-                third_party=True)
+            description = "auto generated role based on owner: {0}".format(
+                self.ldap_principal
+            )
+            role = role_service.create(
+                self.ldap_principal, description=description, third_party=True
+            )
         if not role.third_party:
             role = role_service.set_third_party(role.id, third_party_status=True)
         roles.add(role)
@@ -117,9 +127,15 @@ class LdapPrincipal():
             role = role_service.get_by_name(role_name)
             if role:
                 if ldap_group_name in self.ldap_groups:
-                    current_app.logger.debug("assigning role {0} to ldap user {1}".format(self.ldap_principal, role))
+                    current_app.logger.debug(
+                        "assigning role {0} to ldap user {1}".format(
+                            self.ldap_principal, role
+                        )
+                    )
                     if not role.third_party:
-                        role = role_service.set_third_party(role.id, third_party_status=True)
+                        role = role_service.set_third_party(
+                            role.id, third_party_status=True
+                        )
                     roles.add(role)
         return roles
 
@@ -131,7 +147,7 @@ class LdapPrincipal():
         self._bind()
         roles = self._authorize()
         if not roles:
-            raise Exception('ldap authorization failed')
+            raise Exception("ldap authorization failed")
         return self._update_user(roles)
 
     def _bind(self):
@@ -140,9 +156,12 @@ class LdapPrincipal():
         list groups for a user.
         raise an exception on error.
         """
-        if '@' not in self.ldap_principal:
-            self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
-        ldap_filter = 'userPrincipalName=%s' % self.ldap_principal
+        if "@" not in self.ldap_principal:
+            self.ldap_principal = "%s@%s" % (
+                self.ldap_principal,
+                self.ldap_email_domain,
+            )
+        ldap_filter = "userPrincipalName=%s" % self.ldap_principal
 
         # query ldap for auth
         try:
@@ -158,30 +177,54 @@ class LdapPrincipal():
                 self.ldap_client.set_option(ldap.OPT_X_TLS_DEMAND, True)
                 self.ldap_client.set_option(ldap.OPT_DEBUG_LEVEL, 255)
             if self.ldap_cacert_file:
-                self.ldap_client.set_option(ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file)
+                self.ldap_client.set_option(
+                    ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file
+                )
             self.ldap_client.simple_bind_s(self.ldap_principal, self.ldap_password)
         except ldap.INVALID_CREDENTIALS:
             self.ldap_client.unbind()
-            raise Exception('The supplied ldap credentials are invalid')
+            raise Exception("The supplied ldap credentials are invalid")
         except ldap.SERVER_DOWN:
-            raise Exception('ldap server unavailable')
+            raise Exception("ldap server unavailable")
         except ldap.LDAPError as e:
             raise Exception("ldap error: {0}".format(e))
 
-        lgroups = self.ldap_client.search_s(self.ldap_base_dn,
-                        ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf']
+        if self.ldap_is_active_directory:
+            # Lookup user DN, needed to search for group membership
+            userdn = self.ldap_client.search_s(
+                self.ldap_base_dn,
+                ldap.SCOPE_SUBTREE,
+                ldap_filter,
+                ["distinguishedName"],
+            )[0][1]["distinguishedName"][0]
+            userdn = userdn.decode("utf-8")
+            # Search all groups that have the userDN as a member
+            groupfilter = "(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))".format(
+                userdn
+            )
+            lgroups = self.ldap_client.search_s(
+                self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ["cn"]
+            )
+
+            # Create a list of group CN's from the result
+            self.ldap_groups = []
+            for group in lgroups:
+                (dn, values) = group
+                if type(values) == dict:
+                    self.ldap_groups.append(values["cn"][0].decode("utf-8"))
+        else:
+            lgroups = self.ldap_client.search_s(
+                self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs
+            )[0][1]["memberOf"]
             # lgroups is a list of utf-8 encoded strings
             # convert to a single string of groups to allow matching
-        self.ldap_groups = b''.join(lgroups).decode('ascii')
+            self.ldap_groups = b"".join(lgroups).decode("ascii")
+
         self.ldap_client.unbind()
 
     def _ldap_validate_conf(self):
         """
         Confirms required ldap config settings exist.
         """
-        required_vars = [
-            'LDAP_BIND_URI',
-            'LDAP_BASE_DN',
-            'LDAP_EMAIL_DOMAIN',
-        ]
+        required_vars = ["LDAP_BIND_URI", "LDAP_BASE_DN", "LDAP_EMAIL_DOMAIN"]
         validate_conf(current_app, required_vars)

lemur/auth/permissions.py

@@ -9,55 +9,66 @@
 from functools import partial
 from collections import namedtuple
 
+from flask import current_app
 from flask_principal import Permission, RoleNeed
 
 # Permissions
-operator_permission = Permission(RoleNeed('operator'))
-admin_permission = Permission(RoleNeed('admin'))
+operator_permission = Permission(RoleNeed("operator"))
+admin_permission = Permission(RoleNeed("admin"))
 
-CertificateOwner = namedtuple('certificate', ['method', 'value'])
-CertificateOwnerNeed = partial(CertificateOwner, 'role')
+CertificateOwner = namedtuple("certificate", ["method", "value"])
+CertificateOwnerNeed = partial(CertificateOwner, "role")
 
 
 class SensitiveDomainPermission(Permission):
     def __init__(self):
-        super(SensitiveDomainPermission, self).__init__(RoleNeed('admin'))
+        needs = [RoleNeed("admin")]
+        sensitive_domain_roles = current_app.config.get("SENSITIVE_DOMAIN_ROLES", [])
+
+        if sensitive_domain_roles:
+            for role in sensitive_domain_roles:
+                needs.append(RoleNeed(role))
+
+        super(SensitiveDomainPermission, self).__init__(*needs)
 
 
 class CertificatePermission(Permission):
     def __init__(self, owner, roles):
-        needs = [RoleNeed('admin'), RoleNeed(owner), RoleNeed('creator')]
+        needs = [RoleNeed("admin"), RoleNeed(owner), RoleNeed("creator")]
         for r in roles:
             needs.append(CertificateOwnerNeed(str(r)))
+            # Backwards compatibility with mixed-case role names
+            if str(r) != str(r).lower():
+                needs.append(CertificateOwnerNeed(str(r).lower()))
 
         super(CertificatePermission, self).__init__(*needs)
 
 
 class ApiKeyCreatorPermission(Permission):
     def __init__(self):
-        super(ApiKeyCreatorPermission, self).__init__(RoleNeed('admin'))
+        super(ApiKeyCreatorPermission, self).__init__(RoleNeed("admin"))
 
 
-RoleMember = namedtuple('role', ['method', 'value'])
-RoleMemberNeed = partial(RoleMember, 'member')
+RoleMember = namedtuple("role", ["method", "value"])
+RoleMemberNeed = partial(RoleMember, "member")
 
 
 class RoleMemberPermission(Permission):
     def __init__(self, role_id):
-        needs = [RoleNeed('admin'), RoleMemberNeed(role_id)]
+        needs = [RoleNeed("admin"), RoleMemberNeed(role_id)]
         super(RoleMemberPermission, self).__init__(*needs)
 
 
-AuthorityCreator = namedtuple('authority', ['method', 'value'])
-AuthorityCreatorNeed = partial(AuthorityCreator, 'authorityUse')
+AuthorityCreator = namedtuple("authority", ["method", "value"])
+AuthorityCreatorNeed = partial(AuthorityCreator, "authorityUse")
 
-AuthorityOwner = namedtuple('authority', ['method', 'value'])
-AuthorityOwnerNeed = partial(AuthorityOwner, 'role')
+AuthorityOwner = namedtuple("authority", ["method", "value"])
+AuthorityOwnerNeed = partial(AuthorityOwner, "role")
 
 
 class AuthorityPermission(Permission):
     def __init__(self, authority_id, roles):
-        needs = [RoleNeed('admin'), AuthorityCreatorNeed(str(authority_id))]
+        needs = [RoleNeed("admin"), AuthorityCreatorNeed(str(authority_id))]
         for r in roles:
             needs.append(AuthorityOwnerNeed(str(r)))
 

lemur/auth/service.py

@@ -39,13 +39,13 @@ def get_rsa_public_key(n, e):
     :param e:
     :return: a RSA Public Key in PEM format
     """
-    n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, 'utf-8'))), 16)
-    e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, 'utf-8'))), 16)
+    n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, "utf-8"))), 16)
+    e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, "utf-8"))), 16)
 
     pub = RSAPublicNumbers(e, n).public_key(default_backend())
     return pub.public_bytes(
         encoding=serialization.Encoding.PEM,
-        format=serialization.PublicFormat.SubjectPublicKeyInfo
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
     )
 
 
@@ -57,28 +57,27 @@ def create_token(user, aid=None, ttl=None):
     :param user:
     :return:
     """
-    expiration_delta = timedelta(days=int(current_app.config.get('LEMUR_TOKEN_EXPIRATION', 1)))
-    payload = {
-        'iat': datetime.utcnow(),
-        'exp': datetime.utcnow() + expiration_delta
-    }
+    expiration_delta = timedelta(
+        days=int(current_app.config.get("LEMUR_TOKEN_EXPIRATION", 1))
+    )
+    payload = {"iat": datetime.utcnow(), "exp": datetime.utcnow() + expiration_delta}
 
     # Handle Just a User ID & User Object.
     if isinstance(user, int):
-        payload['sub'] = user
+        payload["sub"] = user
     else:
-        payload['sub'] = user.id
+        payload["sub"] = user.id
     if aid is not None:
-        payload['aid'] = aid
+        payload["aid"] = aid
     # Custom TTLs are only supported on Access Keys.
     if ttl is not None and aid is not None:
         # Tokens that are forever until revoked.
         if ttl == -1:
-            del payload['exp']
+            del payload["exp"]
         else:
-            payload['exp'] = ttl
-    token = jwt.encode(payload, current_app.config['LEMUR_TOKEN_SECRET'])
-    return token.decode('unicode_escape')
+            payload["exp"] = datetime.utcnow() + timedelta(days=ttl)
+    token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"])
+    return token
 
 
 def login_required(f):
@@ -88,49 +87,54 @@ def login_required(f):
     :param f:
     :return:
     """
+
     @wraps(f)
     def decorated_function(*args, **kwargs):
-        if not request.headers.get('Authorization'):
-            response = jsonify(message='Missing authorization header')
+        if not request.headers.get("Authorization"):
+            response = jsonify(message="Missing authorization header")
             response.status_code = 401
             return response
 
         try:
-            token = request.headers.get('Authorization').split()[1]
+            token = request.headers.get("Authorization").split()[1]
         except Exception as e:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
 
         try:
-            payload = jwt.decode(token, current_app.config['LEMUR_TOKEN_SECRET'])
+            header_data = fetch_token_header(token)
+            payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"], algorithms=[header_data["alg"]])
         except jwt.DecodeError:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
         except jwt.ExpiredSignatureError:
-            return dict(message='Token has expired'), 403
+            return dict(message="Token has expired"), 403
         except jwt.InvalidTokenError:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
 
-        if 'aid' in payload:
-            access_key = api_key_service.get(payload['aid'])
+        if "aid" in payload:
+            access_key = api_key_service.get(payload["aid"])
             if access_key.revoked:
-                return dict(message='Token has been revoked'), 403
+                return dict(message="Token has been revoked"), 403
             if access_key.ttl != -1:
                 current_time = datetime.utcnow()
-                expired_time = datetime.fromtimestamp(access_key.issued_at + access_key.ttl)
+                # API key uses days
+                expired_time = datetime.fromtimestamp(access_key.issued_at) + timedelta(days=access_key.ttl)
                 if current_time >= expired_time:
-                    return dict(message='Token has expired'), 403
+                    return dict(message="Token has expired"), 403
 
-        user = user_service.get(payload['sub'])
+        user = user_service.get(payload["sub"])
 
         if not user.active:
-            return dict(message='User is not currently active'), 403
+            return dict(message="User is not currently active"), 403
 
         g.current_user = user
 
         if not g.current_user:
-            return dict(message='You are not logged in'), 403
+            return dict(message="You are not logged in"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(g.current_user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(g.current_user.id)
+        )
 
         return f(*args, **kwargs)
 
@@ -144,18 +148,18 @@ def fetch_token_header(token):
     :param token:
     :return: :raise jwt.DecodeError:
     """
-    token = token.encode('utf-8')
+    token = token.encode("utf-8")
     try:
-        signing_input, crypto_segment = token.rsplit(b'.', 1)
-        header_segment, payload_segment = signing_input.split(b'.', 1)
+        signing_input, crypto_segment = token.rsplit(b".", 1)
+        header_segment, payload_segment = signing_input.split(b".", 1)
     except ValueError:
-        raise jwt.DecodeError('Not enough segments')
+        raise jwt.DecodeError("Not enough segments")
 
     try:
-        return json.loads(jwt.utils.base64url_decode(header_segment).decode('utf-8'))
+        return json.loads(jwt.utils.base64url_decode(header_segment).decode("utf-8"))
     except TypeError as e:
         current_app.logger.exception(e)
-        raise jwt.DecodeError('Invalid header padding')
+        raise jwt.DecodeError("Invalid header padding")
 
 
 @identity_loaded.connect
@@ -174,13 +178,13 @@ def on_identity_loaded(sender, identity):
     identity.provides.add(UserNeed(identity.id))
 
     # identity with the roles that the user provides
-    if hasattr(user, 'roles'):
+    if hasattr(user, "roles"):
         for role in user.roles:
             identity.provides.add(RoleNeed(role.name))
             identity.provides.add(RoleMemberNeed(role.id))
 
     # apply ownership for authorities
-    if hasattr(user, 'authorities'):
+    if hasattr(user, "authorities"):
         for authority in user.authorities:
             identity.provides.add(AuthorityCreatorNeed(authority.id))
 
@@ -191,6 +195,7 @@ class AuthenticatedResource(Resource):
     """
     Inherited by all resources that need to be protected by authentication.
     """
+
     method_decorators = [login_required]
 
     def __init__(self):

lemur/auth/views.py

@@ -5,6 +5,8 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import json
+
 import jwt
 import base64
 import requests
@@ -20,15 +22,18 @@ from lemur.common.utils import get_psuedo_random_string
 
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
+from lemur.logs import service as log_service
 from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key
-import lemur.auth.ldap as ldap
+from lemur.auth import ldap
+from lemur.plugins.base import plugins
 
-
-mod = Blueprint('auth', __name__)
+mod = Blueprint("auth", __name__)
 api = Api(mod)
 
 
-def exchange_for_access_token(code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True):
+def exchange_for_access_token(
+    code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True
+):
     """
     Exchanges authorization code for access token.
 
@@ -43,28 +48,32 @@ def exchange_for_access_token(code, redirect_uri, client_id, secret, access_toke
     """
     # take the information we have received from the provider to create a new request
     params = {
-        'grant_type': 'authorization_code',
-        'scope': 'openid email profile address',
-        'code': code,
-        'redirect_uri': redirect_uri,
-        'client_id': client_id
+        "grant_type": "authorization_code",
+        "scope": "openid email profile address",
+        "code": code,
+        "redirect_uri": redirect_uri,
+        "client_id": client_id,
     }
 
     # the secret and cliendId will be given to you when you signup for the provider
-    token = '{0}:{1}'.format(client_id, secret)
+    token = "{0}:{1}".format(client_id, secret)
 
-    basic = base64.b64encode(bytes(token, 'utf-8'))
+    basic = base64.b64encode(bytes(token, "utf-8"))
     headers = {
-        'Content-Type': 'application/x-www-form-urlencoded',
-        'authorization': 'basic {0}'.format(basic.decode('utf-8'))
+        "Content-Type": "application/x-www-form-urlencoded",
+        "authorization": "basic {0}".format(basic.decode("utf-8")),
     }
 
     # exchange authorization code for access token.
-    r = requests.post(access_token_url, headers=headers, params=params, verify=verify_cert)
+    r = requests.post(
+        access_token_url, headers=headers, params=params, verify=verify_cert
+    )
     if r.status_code == 400:
-        r = requests.post(access_token_url, headers=headers, data=params, verify=verify_cert)
-    id_token = r.json()['id_token']
-    access_token = r.json()['access_token']
+        r = requests.post(
+            access_token_url, headers=headers, data=params, verify=verify_cert
+        )
+    id_token = r.json()["id_token"]
+    access_token = r.json()["access_token"]
 
     return id_token, access_token
 
@@ -83,23 +92,25 @@ def validate_id_token(id_token, client_id, jwks_url):
 
     # retrieve the key material as specified by the token header
     r = requests.get(jwks_url)
-    for key in r.json()['keys']:
-        if key['kid'] == header_data['kid']:
-            secret = get_rsa_public_key(key['n'], key['e'])
-            algo = header_data['alg']
+    for key in r.json()["keys"]:
+        if key["kid"] == header_data["kid"]:
+            secret = get_rsa_public_key(key["n"], key["e"])
+            algo = header_data["alg"]
             break
     else:
-        return dict(message='Key not found'), 401
+        return dict(message="Key not found"), 401
 
     # validate your token based on the key it was signed with
     try:
-        jwt.decode(id_token, secret.decode('utf-8'), algorithms=[algo], audience=client_id)
+        jwt.decode(
+            id_token, secret.decode("utf-8"), algorithms=[algo], audience=client_id
+        )
     except jwt.DecodeError:
-        return dict(message='Token is invalid'), 401
+        return dict(message="Token is invalid"), 401
     except jwt.ExpiredSignatureError:
-        return dict(message='Token has expired'), 401
+        return dict(message="Token has expired"), 401
     except jwt.InvalidTokenError:
-        return dict(message='Token is invalid'), 401
+        return dict(message="Token is invalid"), 401
 
 
 def retrieve_user(user_api_url, access_token):
@@ -110,16 +121,66 @@ def retrieve_user(user_api_url, access_token):
     :param access_token:
     :return:
     """
-    user_params = dict(access_token=access_token, schema='profile')
+    user_params = dict(access_token=access_token, schema="profile")
+
+    headers = {}
+
+    if current_app.config.get("PING_INCLUDE_BEARER_TOKEN"):
+        headers = {"Authorization": f"Bearer {access_token}"}
 
     # retrieve information about the current user.
-    r = requests.get(user_api_url, params=user_params)
+    r = requests.get(user_api_url, params=user_params, headers=headers)
+    # Some IDPs, like "Keycloak", require a POST instead of a GET
+    if r.status_code == 400:
+        r = requests.post(user_api_url, data=user_params, headers=headers)
+
     profile = r.json()
 
-    user = user_service.get_by_email(profile['email'])
+    user = user_service.get_by_email(profile["email"])
     return user, profile
 
 
+def retrieve_user_memberships(user_api_url, user_membership_api_url, access_token):
+    user, profile = retrieve_user(user_api_url, access_token)
+
+    if user_membership_api_url is None:
+        return user, profile
+    """
+    Potentially, below code can be made more generic i.e., plugin driven. Unaware of the usage of this
+    code across the community, current implementation is config driven. Without user_membership_api_url
+    configured, it is backward compatible.
+    """
+    tls_provider = plugins.get(current_app.config.get("PING_USER_MEMBERSHIP_TLS_PROVIDER"))
+
+    # put user id in url
+    user_membership_api_url = user_membership_api_url.replace("%user_id%", profile["userId"])
+
+    session = tls_provider.session(current_app.config.get("PING_USER_MEMBERSHIP_SERVICE"))
+    headers = {"Content-Type": "application/json"}
+    data = {"relation": "DIRECT_ONLY", "groupFilter": {"type": "GOOGLE"}, "size": 500}
+    user_membership = {"email": profile["email"],
+                       "thumbnailPhotoUrl": profile["thumbnailPhotoUrl"],
+                       "googleGroups": []}
+    while True:
+        # retrieve information about the current user memberships
+        r = session.post(user_membership_api_url, data=json.dumps(data), headers=headers)
+
+        if r.status_code == 200:
+            response = r.json()
+            membership_details = response["data"]
+            for membership in membership_details:
+                user_membership["googleGroups"].append(membership["membership"]["name"])
+
+            if "nextPageToken" in response and response["nextPageToken"]:
+                data["nextPageToken"] = response["nextPageToken"]
+            else:
+                break
+        else:
+            current_app.logger.error(f"Response Code:{r.status_code} {r.text}")
+            break
+    return user, user_membership
+
+
 def create_user_roles(profile):
     """Creates new roles based on profile information.
 
@@ -129,28 +190,44 @@ def create_user_roles(profile):
     roles = []
 
     # update their google 'roles'
-    for group in profile['googleGroups']:
+    if "googleGroups" in profile:
+        for group in profile["googleGroups"]:
             role = role_service.get_by_name(group)
             if not role:
-            role = role_service.create(group, description='This is a google group based role created by Lemur', third_party=True)
-        if not role.third_party:
+                role = role_service.create(
+                    group,
+                    description="This is a google group based role created by Lemur",
+                    third_party=True,
+                )
+            if (group != 'admin') and (not role.third_party):
                 role = role_service.set_third_party(role.id, third_party_status=True)
             roles.append(role)
+    else:
+        current_app.logger.warning(
+            "'googleGroups' not sent by identity provider, no specific roles will assigned to the user."
+        )
 
-    role = role_service.get_by_name(profile['email'])
+    role = role_service.get_by_name(profile["email"])
 
     if not role:
-        role = role_service.create(profile['email'], description='This is a user specific role', third_party=True)
+        role = role_service.create(
+            profile["email"],
+            description="This is a user specific role",
+            third_party=True,
+        )
     if not role.third_party:
         role = role_service.set_third_party(role.id, third_party_status=True)
 
     roles.append(role)
 
     # every user is an operator (tied to a default role)
-    if current_app.config.get('LEMUR_DEFAULT_ROLE'):
-        default = role_service.get_by_name(current_app.config['LEMUR_DEFAULT_ROLE'])
+    if current_app.config.get("LEMUR_DEFAULT_ROLE"):
+        default = role_service.get_by_name(current_app.config["LEMUR_DEFAULT_ROLE"])
         if not default:
-            default = role_service.create(current_app.config['LEMUR_DEFAULT_ROLE'], description='This is the default Lemur role.')
+            default = role_service.create(
+                current_app.config["LEMUR_DEFAULT_ROLE"],
+                description="This is the default Lemur role.",
+            )
         if not default.third_party:
             role_service.set_third_party(default.id, third_party_status=True)
         roles.append(default)
@@ -165,32 +242,37 @@ def update_user(user, profile, roles):
     :param profile:
     :param roles:
     """
-
     # if we get an sso user create them an account
     if not user:
         user = user_service.create(
-            profile['email'],
+            profile["email"],
             get_psuedo_random_string(),
-            profile['email'],
+            profile["email"],
             True,
-            profile.get('thumbnailPhotoUrl'),
-            roles
+            profile.get("thumbnailPhotoUrl"),
+            roles,
         )
 
     else:
         # we add 'lemur' specific roles, so they do not get marked as removed
+        removed_roles = []
         for ur in user.roles:
             if not ur.third_party:
                 roles.append(ur)
+            elif ur not in roles:
+                # This is a role assigned in lemur, but not returned by sso during current login
+                removed_roles.append(ur.name)
 
+        if removed_roles:
+            log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
         # update any changes to the user
         user_service.update(
             user.id,
-            profile['email'],
-            profile['email'],
+            profile["email"],
+            profile["email"],
             True,
-            profile.get('thumbnailPhotoUrl'),  # profile isn't google+ enabled
-            roles
+            profile.get("thumbnailPhotoUrl"),  # profile isn't google+ enabled
+            roles,
         )
 
 
@@ -211,6 +293,7 @@ class Login(Resource):
     on your uses cases but. It is important to not that there is currently no build in method to revoke a users token \
     and force re-authentication.
     """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Login, self).__init__()
@@ -228,6 +311,7 @@ class Login(Resource):
               POST /auth/login HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "username": "test",
@@ -251,23 +335,26 @@ class Login(Resource):
            :statuscode 401: invalid credentials
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('username', type=str, required=True, location='json')
-        self.reqparse.add_argument('password', type=str, required=True, location='json')
+        self.reqparse.add_argument("username", type=str, required=True, location="json")
+        self.reqparse.add_argument("password", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
-        if '@' in args['username']:
-            user = user_service.get_by_email(args['username'])
+        if "@" in args["username"]:
+            user = user_service.get_by_email(args["username"])
         else:
-            user = user_service.get_by_username(args['username'])
+            user = user_service.get_by_username(args["username"])
 
         # default to local authentication
-        if user and user.check_password(args['password']) and user.active:
+        if user and user.check_password(args["password"]) and user.active:
             # Tell Flask-Principal the identity changed
-            identity_changed.send(current_app._get_current_object(),
-                                  identity=Identity(user.id))
+            identity_changed.send(
+                current_app._get_current_object(), identity=Identity(user.id)
+            )
 
-            metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+            )
             return dict(token=create_token(user))
 
         # try ldap login
@@ -277,19 +364,29 @@ class Login(Resource):
                 user = ldap_principal.authenticate()
                 if user and user.active:
                     # Tell Flask-Principal the identity changed
-                    identity_changed.send(current_app._get_current_object(),
-                                  identity=Identity(user.id))
-                    metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+                    identity_changed.send(
+                        current_app._get_current_object(), identity=Identity(user.id)
+                    )
+                    metrics.send(
+                        "login",
+                        "counter",
+                        1,
+                        metric_tags={"status": SUCCESS_METRIC_STATUS},
+                    )
                     return dict(token=create_token(user))
             except Exception as e:
                 current_app.logger.error("ldap error: {0}".format(e))
-                    ldap_message = 'ldap error: %s' % e
-                    metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+                ldap_message = "ldap error: %s" % e
+                metrics.send(
+                    "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+                )
                 return dict(message=ldap_message), 403
 
         # if not valid user - no certificates for you
-        metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-        return dict(message='The supplied credentials are invalid'), 403
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+        )
+        return dict(message="The supplied credentials are invalid"), 403
 
 
 class Ping(Resource):
@@ -302,49 +399,63 @@ class Ping(Resource):
     provider uses for its callbacks.
     2. Add or change the Lemur AngularJS Configuration to point to your new provider
     """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Ping, self).__init__()
 
     def get(self):
-        return 'Redirecting...'
+        return "Redirecting..."
 
     def post(self):
-        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
-        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
-        self.reqparse.add_argument('code', type=str, required=True, location='json')
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
         # you can either discover these dynamically or simply configure them
-        access_token_url = current_app.config.get('PING_ACCESS_TOKEN_URL')
-        user_api_url = current_app.config.get('PING_USER_API_URL')
+        access_token_url = current_app.config.get("PING_ACCESS_TOKEN_URL")
 
-        secret = current_app.config.get('PING_SECRET')
+        secret = current_app.config.get("PING_SECRET")
 
         id_token, access_token = exchange_for_access_token(
-            args['code'],
-            args['redirectUri'],
-            args['clientId'],
+            args["code"],
+            args["redirectUri"],
+            args["clientId"],
             secret,
-            access_token_url=access_token_url
+            access_token_url=access_token_url,
         )
 
-        jwks_url = current_app.config.get('PING_JWKS_URL')
-        validate_id_token(id_token, args['clientId'], jwks_url)
+        jwks_url = current_app.config.get("PING_JWKS_URL")
+        error_code = validate_id_token(id_token, args["clientId"], jwks_url)
+        if error_code:
+            return error_code
 
-        user, profile = retrieve_user(user_api_url, access_token)
+        user, profile = retrieve_user_memberships(
+            current_app.config.get("PING_USER_API_URL"),
+            current_app.config.get("PING_USER_MEMBERSHIP_URL"),
+            access_token
+        )
         roles = create_user_roles(profile)
         update_user(user, profile, roles)
 
         if not user or not user.active:
-            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-            return dict(message='The supplied credentials are invalid'), 403
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(user.id)
+        )
 
-        metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+        )
         return dict(token=create_token(user))
 
 
@@ -354,46 +465,56 @@ class OAuth2(Resource):
         super(OAuth2, self).__init__()
 
     def get(self):
-        return 'Redirecting...'
+        return "Redirecting..."
 
     def post(self):
-        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
-        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
-        self.reqparse.add_argument('code', type=str, required=True, location='json')
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
         # you can either discover these dynamically or simply configure them
-        access_token_url = current_app.config.get('OAUTH2_ACCESS_TOKEN_URL')
-        user_api_url = current_app.config.get('OAUTH2_USER_API_URL')
-        verify_cert = current_app.config.get('OAUTH2_VERIFY_CERT')
+        access_token_url = current_app.config.get("OAUTH2_ACCESS_TOKEN_URL")
+        user_api_url = current_app.config.get("OAUTH2_USER_API_URL")
+        verify_cert = current_app.config.get("OAUTH2_VERIFY_CERT")
 
-        secret = current_app.config.get('OAUTH2_SECRET')
+        secret = current_app.config.get("OAUTH2_SECRET")
 
         id_token, access_token = exchange_for_access_token(
-            args['code'],
-            args['redirectUri'],
-            args['clientId'],
+            args["code"],
+            args["redirectUri"],
+            args["clientId"],
             secret,
             access_token_url=access_token_url,
-            verify_cert=verify_cert
+            verify_cert=verify_cert,
         )
 
-        jwks_url = current_app.config.get('PING_JWKS_URL')
-        validate_id_token(id_token, args['clientId'], jwks_url)
+        jwks_url = current_app.config.get("OAUTH2_JWKS_URL")
+        error_code = validate_id_token(id_token, args["clientId"], jwks_url)
+        if error_code:
+            return error_code
 
         user, profile = retrieve_user(user_api_url, access_token)
         roles = create_user_roles(profile)
         update_user(user, profile, roles)
 
         if not user.active:
-            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-            return dict(message='The supplied credentials are invalid'), 403
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(user.id)
+        )
 
-        metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+        )
 
         return dict(token=create_token(user))
 
@@ -404,44 +525,52 @@ class Google(Resource):
         super(Google, self).__init__()
 
     def post(self):
-        access_token_url = 'https://accounts.google.com/o/oauth2/token'
-        people_api_url = 'https://www.googleapis.com/plus/v1/people/me/openIdConnect'
+        access_token_url = "https://accounts.google.com/o/oauth2/token"
+        people_api_url = "https://www.googleapis.com/plus/v1/people/me/openIdConnect"
 
-        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
-        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
-        self.reqparse.add_argument('code', type=str, required=True, location='json')
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
         # Step 1. Exchange authorization code for access token
         payload = {
-            'client_id': args['clientId'],
-            'grant_type': 'authorization_code',
-            'redirect_uri': args['redirectUri'],
-            'code': args['code'],
-            'client_secret': current_app.config.get('GOOGLE_SECRET')
+            "client_id": args["clientId"],
+            "grant_type": "authorization_code",
+            "redirect_uri": args["redirectUri"],
+            "code": args["code"],
+            "client_secret": current_app.config.get("GOOGLE_SECRET"),
         }
 
         r = requests.post(access_token_url, data=payload)
         token = r.json()
 
         # Step 2. Retrieve information about the current user
-        headers = {'Authorization': 'Bearer {0}'.format(token['access_token'])}
+        headers = {"Authorization": "Bearer {0}".format(token["access_token"])}
 
         r = requests.get(people_api_url, headers=headers)
         profile = r.json()
 
-        user = user_service.get_by_email(profile['email'])
+        user = user_service.get_by_email(profile["email"])
 
         if not (user and user.active):
-            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-            return dict(message='The supplied credentials are invalid.'), 403
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid."), 403
 
         if user:
-            metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+            )
             return dict(token=create_token(user))
 
-        metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+        )
 
 
 class Providers(Resource):
@@ -452,47 +581,57 @@ class Providers(Resource):
             provider = provider.lower()
 
             if provider == "google":
-                active_providers.append({
-                    'name': 'google',
-                    'clientId': current_app.config.get("GOOGLE_CLIENT_ID"),
-                    'url': api.url_for(Google)
-                })
+                active_providers.append(
+                    {
+                        "name": "google",
+                        "clientId": current_app.config.get("GOOGLE_CLIENT_ID"),
+                        "url": api.url_for(Google),
+                    }
+                )
 
             elif provider == "ping":
-                active_providers.append({
-                    'name': current_app.config.get("PING_NAME"),
-                    'url': current_app.config.get('PING_REDIRECT_URI'),
-                    'redirectUri': current_app.config.get("PING_REDIRECT_URI"),
-                    'clientId': current_app.config.get("PING_CLIENT_ID"),
-                    'responseType': 'code',
-                    'scope': ['openid', 'email', 'profile', 'address'],
-                    'scopeDelimiter': ' ',
-                    'authorizationEndpoint': current_app.config.get("PING_AUTH_ENDPOINT"),
-                    'requiredUrlParams': ['scope'],
-                    'type': '2.0'
-                })
+                active_providers.append(
+                    {
+                        "name": current_app.config.get("PING_NAME"),
+                        "url": current_app.config.get("PING_REDIRECT_URI"),
+                        "redirectUri": current_app.config.get("PING_REDIRECT_URI"),
+                        "clientId": current_app.config.get("PING_CLIENT_ID"),
+                        "responseType": "code",
+                        "scope": ["openid", "email", "profile", "address"],
+                        "scopeDelimiter": " ",
+                        "authorizationEndpoint": current_app.config.get(
+                            "PING_AUTH_ENDPOINT"
+                        ),
+                        "requiredUrlParams": ["scope"],
+                        "type": "2.0",
+                    }
+                )
 
             elif provider == "oauth2":
-                active_providers.append({
-                    'name': current_app.config.get("OAUTH2_NAME"),
-                    'url': current_app.config.get('OAUTH2_REDIRECT_URI'),
-                    'redirectUri': current_app.config.get("OAUTH2_REDIRECT_URI"),
-                    'clientId': current_app.config.get("OAUTH2_CLIENT_ID"),
-                    'responseType': 'code',
-                    'scope': ['openid', 'email', 'profile', 'groups'],
-                    'scopeDelimiter': ' ',
-                    'authorizationEndpoint': current_app.config.get("OAUTH2_AUTH_ENDPOINT"),
-                    'requiredUrlParams': ['scope', 'state', 'nonce'],
-                    'state': 'STATE',
-                    'nonce': get_psuedo_random_string(),
-                    'type': '2.0'
-                })
+                active_providers.append(
+                    {
+                        "name": current_app.config.get("OAUTH2_NAME"),
+                        "url": current_app.config.get("OAUTH2_REDIRECT_URI"),
+                        "redirectUri": current_app.config.get("OAUTH2_REDIRECT_URI"),
+                        "clientId": current_app.config.get("OAUTH2_CLIENT_ID"),
+                        "responseType": "code",
+                        "scope": ["openid", "email", "profile", "groups"],
+                        "scopeDelimiter": " ",
+                        "authorizationEndpoint": current_app.config.get(
+                            "OAUTH2_AUTH_ENDPOINT"
+                        ),
+                        "requiredUrlParams": ["scope", "state", "nonce"],
+                        "state": "STATE",
+                        "nonce": get_psuedo_random_string(),
+                        "type": "2.0",
+                    }
+                )
 
         return active_providers
 
 
-api.add_resource(Login, '/auth/login', endpoint='login')
-api.add_resource(Ping, '/auth/ping', endpoint='ping')
-api.add_resource(Google, '/auth/google', endpoint='google')
-api.add_resource(OAuth2, '/auth/oauth2', endpoint='oauth2')
-api.add_resource(Providers, '/auth/providers', endpoint='providers')
+api.add_resource(Login, "/auth/login", endpoint="login")
+api.add_resource(Ping, "/auth/ping", endpoint="ping")
+api.add_resource(Google, "/auth/google", endpoint="google")
+api.add_resource(OAuth2, "/auth/oauth2", endpoint="oauth2")
+api.add_resource(Providers, "/auth/providers", endpoint="providers")

lemur/authorities/models.py

@@ -6,8 +6,21 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import json
+
+from flask import current_app
 from sqlalchemy.orm import relationship
-from sqlalchemy import Column, Integer, String, Text, func, ForeignKey, DateTime, PassiveDefault, Boolean
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    Text,
+    func,
+    ForeignKey,
+    DateTime,
+    DefaultClause,
+    Boolean,
+)
 from sqlalchemy.dialects.postgresql import JSON
 
 from lemur.database import db
@@ -16,7 +29,7 @@ from lemur.models import roles_authorities
 
 
 class Authority(db.Model):
-    __tablename__ = 'authorities'
+    __tablename__ = "authorities"
     id = Column(Integer, primary_key=True)
     owner = Column(String(128), nullable=False)
     name = Column(String(128), unique=True)
@@ -26,27 +39,79 @@ class Authority(db.Model):
     plugin_name = Column(String(64))
     description = Column(Text)
     options = Column(JSON)
-    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
-    roles = relationship('Role', secondary=roles_authorities, passive_deletes=True, backref=db.backref('authority'), lazy='dynamic')
-    user_id = Column(Integer, ForeignKey('users.id'))
-    authority_certificate = relationship("Certificate", backref='root_authority', uselist=False, foreign_keys='Certificate.root_authority_id')
-    certificates = relationship("Certificate", backref='authority', foreign_keys='Certificate.authority_id')
+    date_created = Column(DateTime, DefaultClause(func.now()), nullable=False)
+    roles = relationship(
+        "Role",
+        secondary=roles_authorities,
+        passive_deletes=True,
+        backref=db.backref("authority"),
+        lazy="dynamic",
+    )
+    user_id = Column(Integer, ForeignKey("users.id"))
+    authority_certificate = relationship(
+        "Certificate",
+        backref="root_authority",
+        uselist=False,
+        foreign_keys="Certificate.root_authority_id",
+    )
+    certificates = relationship(
+        "Certificate", backref="authority", foreign_keys="Certificate.authority_id"
+    )
 
-    authority_pending_certificate = relationship("PendingCertificate", backref='root_authority', uselist=False, foreign_keys='PendingCertificate.root_authority_id')
-    pending_certificates = relationship('PendingCertificate', backref='authority', foreign_keys='PendingCertificate.authority_id')
+    authority_pending_certificate = relationship(
+        "PendingCertificate",
+        backref="root_authority",
+        uselist=False,
+        foreign_keys="PendingCertificate.root_authority_id",
+    )
+    pending_certificates = relationship(
+        "PendingCertificate",
+        backref="authority",
+        foreign_keys="PendingCertificate.authority_id",
+    )
 
     def __init__(self, **kwargs):
-        self.owner = kwargs['owner']
-        self.roles = kwargs.get('roles', [])
-        self.name = kwargs.get('name')
-        self.description = kwargs.get('description')
-        self.authority_certificate = kwargs['authority_certificate']
-        self.plugin_name = kwargs['plugin']['slug']
-        self.options = kwargs.get('options')
+        self.owner = kwargs["owner"]
+        self.roles = kwargs.get("roles", [])
+        self.name = kwargs.get("name")
+        self.description = kwargs.get("description")
+        self.authority_certificate = kwargs["authority_certificate"]
+        self.plugin_name = kwargs["plugin"]["slug"]
+        self.options = kwargs.get("options")
 
     @property
     def plugin(self):
         return plugins.get(self.plugin_name)
 
+    @property
+    def is_cab_compliant(self):
+        """
+        Parse the options to find whether authority is CAB Forum Compliant,
+        i.e., adhering to the CA/Browser Forum Baseline Requirements.
+        Returns None if option is not available
+        """
+        if not self.options:
+            return None
+
+        options_array = json.loads(self.options)
+        if isinstance(options_array, list):
+            for option in options_array:
+                if "name" in option and option["name"] == 'cab_compliant':
+                    return option["value"]
+
+        return None
+
+    @property
+    def max_issuance_days(self):
+        if self.is_cab_compliant:
+            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
+
+    @property
+    def default_validity_days(self):
+        if self.is_cab_compliant:
+            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
+
+        return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365)  # 1 year default
+
     def __repr__(self):
         return "Authority(name={name})".format(name=self.name)

lemur/authorities/schemas.py

@@ -11,12 +11,19 @@ from marshmallow import fields, validates_schema, pre_load
 from marshmallow import validate
 from marshmallow.exceptions import ValidationError
 
-from lemur.schemas import PluginInputSchema, PluginOutputSchema, ExtensionSchema, AssociatedAuthoritySchema, AssociatedRoleSchema
+from lemur.schemas import (
+    PluginInputSchema,
+    PluginOutputSchema,
+    ExtensionSchema,
+    AssociatedAuthoritySchema,
+    AssociatedRoleSchema,
+)
 from lemur.users.schemas import UserNestedOutputSchema
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.common import validators, missing
 
 from lemur.common.fields import ArrowDateTime
+from lemur.constants import CERTIFICATE_KEY_TYPES
 
 
 class AuthorityInputSchema(LemurInputSchema):
@@ -30,21 +37,37 @@ class AuthorityInputSchema(LemurInputSchema):
     validity_years = fields.Integer()
 
     # certificate body fields
-    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
-    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
-    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
-    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
-    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+    organizational_unit = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATIONAL_UNIT")
+    )
+    organization = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATION")
+    )
+    location = fields.String()
+    country = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_COUNTRY")
+    )
+    state = fields.String(missing=lambda: current_app.config.get("LEMUR_DEFAULT_STATE"))
+    # Creating a String field instead of Email to allow empty value
+    email = fields.String()
 
     plugin = fields.Nested(PluginInputSchema)
 
     # signing related options
-    type = fields.String(validate=validate.OneOf(['root', 'subca']), missing='root')
+    type = fields.String(validate=validate.OneOf(["root", "subca"]), missing="root")
     parent = fields.Nested(AssociatedAuthoritySchema)
-    signing_algorithm = fields.String(validate=validate.OneOf(['sha256WithRSA', 'sha1WithRSA']), missing='sha256WithRSA')
-    key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')
+    signing_algorithm = fields.String(
+        validate=validate.OneOf(["sha256WithRSA", "sha1WithRSA",
+                                 "sha256WithECDSA", "SHA384withECDSA", "SHA512withECDSA"]),
+        missing="sha256WithRSA",
+    )
+    key_type = fields.String(
+        validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="RSA2048"
+    )
     key_name = fields.String()
-    sensitivity = fields.String(validate=validate.OneOf(['medium', 'high']), missing='medium')
+    sensitivity = fields.String(
+        validate=validate.OneOf(["medium", "high"]), missing="medium"
+    )
     serial_number = fields.Integer()
     first_serial = fields.Integer(missing=1)
 
@@ -58,9 +81,11 @@ class AuthorityInputSchema(LemurInputSchema):
 
     @validates_schema
     def validate_subca(self, data):
-        if data['type'] == 'subca':
-            if not data.get('parent'):
-                raise ValidationError("If generating a subca, parent 'authority' must be specified.")
+        if data["type"] == "subca":
+            if not data.get("parent"):
+                raise ValidationError(
+                    "If generating a subca, parent 'authority' must be specified."
+                )
 
     @pre_load
     def ensure_dates(self, data):
@@ -100,6 +125,8 @@ class AuthorityOutputSchema(LemurOutputSchema):
     active = fields.Boolean()
     options = fields.Dict()
     roles = fields.List(fields.Nested(AssociatedRoleSchema))
+    max_issuance_days = fields.Integer()
+    default_validity_days = fields.Integer()
     authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema)
 
 
@@ -111,6 +138,10 @@ class AuthorityNestedOutputSchema(LemurOutputSchema):
     owner = fields.Email()
     plugin = fields.Nested(PluginOutputSchema)
     active = fields.Boolean()
+    authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["not_after", "not_before"])
+    is_cab_compliant = fields.Boolean()
+    max_issuance_days = fields.Integer()
+    default_validity_days = fields.Integer()
 
 
 authority_update_schema = AuthorityUpdateSchema()

lemur/authorities/service.py

@@ -15,6 +15,7 @@ from lemur import database
 from lemur.common.utils import truthiness
 from lemur.extensions import metrics
 from lemur.authorities.models import Authority
+from lemur.certificates.models import Certificate
 from lemur.roles import service as role_service
 
 from lemur.certificates.service import upload
@@ -38,11 +39,27 @@ def update(authority_id, description, owner, active, roles):
     return database.update(authority)
 
 
+def update_options(authority_id, options):
+    """
+    Update an authority with new options.
+
+    :param authority_id:
+    :param options: the new options to be saved into the authority
+    :return:
+    """
+
+    authority = get(authority_id)
+
+    authority.options = options
+
+    return database.update(authority)
+
+
 def mint(**kwargs):
     """
     Creates the authority based on the plugin provided.
     """
-    issuer = kwargs['plugin']['plugin_object']
+    issuer = kwargs["plugin"]["plugin_object"]
     values = issuer.create_authority(kwargs)
 
     # support older plugins
@@ -52,7 +69,12 @@ def mint(**kwargs):
     elif len(values) == 4:
         body, private_key, chain, roles = values
 
-    roles = create_authority_roles(roles, kwargs['owner'], kwargs['plugin']['plugin_object'].title, kwargs['creator'])
+    roles = create_authority_roles(
+        roles,
+        kwargs["owner"],
+        kwargs["plugin"]["plugin_object"].title,
+        kwargs["creator"],
+    )
     return body, private_key, chain, roles
 
 
@@ -65,16 +87,17 @@ def create_authority_roles(roles, owner, plugin_title, creator):
     """
     role_objs = []
     for r in roles:
-        role = role_service.get_by_name(r['name'])
+        role = role_service.get_by_name(r["name"])
         if not role:
             role = role_service.create(
-                r['name'],
-                password=r['password'],
+                r["name"],
+                password=r["password"],
                 description="Auto generated role for {0}".format(plugin_title),
-                username=r['username'])
+                username=r["username"],
+            )
 
         # the user creating the authority should be able to administer it
-        if role.username == 'admin':
+        if role.username == "admin":
             creator.roles.append(role)
 
         role_objs.append(role)
@@ -83,8 +106,7 @@ def create_authority_roles(roles, owner, plugin_title, creator):
     owner_role = role_service.get_by_name(owner)
     if not owner_role:
         owner_role = role_service.create(
-            owner,
-            description="Auto generated role based on owner: {0}".format(owner)
+            owner, description="Auto generated role based on owner: {0}".format(owner)
         )
 
     role_objs.append(owner_role)
@@ -97,27 +119,29 @@ def create(**kwargs):
     """
     body, private_key, chain, roles = mint(**kwargs)
 
-    kwargs['creator'].roles = list(set(list(kwargs['creator'].roles) + roles))
+    kwargs["creator"].roles = list(set(list(kwargs["creator"].roles) + roles))
 
-    kwargs['body'] = body
-    kwargs['private_key'] = private_key
-    kwargs['chain'] = chain
+    kwargs["body"] = body
+    kwargs["private_key"] = private_key
+    kwargs["chain"] = chain
 
-    if kwargs.get('roles'):
-        kwargs['roles'] += roles
+    if kwargs.get("roles"):
+        kwargs["roles"] += roles
     else:
-        kwargs['roles'] = roles
+        kwargs["roles"] = roles
 
     cert = upload(**kwargs)
-    kwargs['authority_certificate'] = cert
-    if kwargs.get('plugin', {}).get('plugin_options', []):
-        kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options'])
+    kwargs["authority_certificate"] = cert
+    if kwargs.get("plugin", {}).get("plugin_options", []):
+        kwargs["options"] = json.dumps(kwargs["plugin"]["plugin_options"])
 
     authority = Authority(**kwargs)
     authority = database.create(authority)
-    kwargs['creator'].authorities.append(authority)
+    kwargs["creator"].authorities.append(authority)
 
-    metrics.send('authority_created', 'counter', 1, metric_tags=dict(owner=authority.owner))
+    metrics.send(
+        "authority_created", "counter", 1, metric_tags=dict(owner=authority.owner)
+    )
     return authority
 
 
@@ -149,7 +173,7 @@ def get_by_name(authority_name):
     :param authority_name:
     :return:
     """
-    return database.get(Authority, authority_name, field='name')
+    return database.get(Authority, authority_name, field="name")
 
 
 def get_authority_role(ca_name, creator=None):
@@ -172,22 +196,31 @@ def render(args):
     :return:
     """
     query = database.session_query(Authority)
-    filt = args.pop('filter')
+    filt = args.pop("filter")
 
     if filt:
-        terms = filt.split(';')
-        if 'active' in filt:
+        terms = filt.split(";")
+        if "active" in filt:
             query = query.filter(Authority.active == truthiness(terms[1]))
+        elif "cn" in filt:
+            term = "%{0}%".format(terms[1])
+            sub_query = (
+                database.session_query(Certificate.root_authority_id)
+                .filter(Certificate.cn.ilike(term))
+                .subquery()
+            )
+
+            query = query.filter(Authority.id.in_(sub_query))
         else:
             query = database.filter(query, Authority, terms)
 
     # we make sure that a user can only use an authority they either own are a member of - admins can see all
-    if not args['user'].is_admin:
+    if not args["user"].is_admin:
         authority_ids = []
-        for authority in args['user'].authorities:
+        for authority in args["user"].authorities:
             authority_ids.append(authority.id)
 
-        for role in args['user'].roles:
+        for role in args["user"].roles:
             for authority in role.authorities:
                 authority_ids.append(authority.id)
         query = query.filter(Authority.id.in_(authority_ids))

lemur/authorities/views.py

@@ -16,15 +16,21 @@ from lemur.auth.permissions import AuthorityPermission
 from lemur.certificates import service as certificate_service
 
 from lemur.authorities import service
-from lemur.authorities.schemas import authority_input_schema, authority_output_schema, authorities_output_schema, authority_update_schema
+from lemur.authorities.schemas import (
+    authority_input_schema,
+    authority_output_schema,
+    authorities_output_schema,
+    authority_update_schema,
+)
 
 
-mod = Blueprint('authorities', __name__)
+mod = Blueprint("authorities", __name__)
 api = Api(mod)
 
 
 class AuthoritiesList(AuthenticatedResource):
     """ Defines the 'authorities' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(AuthoritiesList, self).__init__()
@@ -90,7 +96,7 @@ class AuthoritiesList(AuthenticatedResource):
                     "owner": "secure@example.com",
                     "id": 43,
                     "description": "This is the ROOT certificate for the TestAuthority certificate authority."
-                }
+                }],
                 "total": 1
               }
 
@@ -107,7 +113,7 @@ class AuthoritiesList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['user'] = g.current_user
+        args["user"] = g.current_user
         return service.render(args)
 
     @validate_schema(authority_input_schema, authority_output_schema)
@@ -124,6 +130,7 @@ class AuthoritiesList(AuthenticatedResource):
               POST /authorities HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "country": "US",
@@ -136,7 +143,7 @@ class AuthoritiesList(AuthenticatedResource):
                  "sensitivity": "medium",
                  "keyType": "RSA2048",
                  "plugin": {
-                    "slug": "cloudca-issuer",
+                     "slug": "cloudca-issuer"
                  },
                  "name": "TimeTestAuthority5",
                  "owner": "secure@example.com",
@@ -149,6 +156,7 @@ class AuthoritiesList(AuthenticatedResource):
                      },
                      "custom": []
                  }
+              }
 
            **Example response**:
 
@@ -210,8 +218,7 @@ class AuthoritiesList(AuthenticatedResource):
            :arg parent: the parent authority if this is to be a subca
            :arg signingAlgorithm: algorithm used to sign the authority
            :arg keyType: key type
-           :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored
-           in an HSM
+           :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored in an HSM
            :arg keyName: name of the key to store in the HSM (CloudCA)
            :arg serialNumber: serial number of the authority
            :arg firstSerial: specifies the starting serial number for certificates issued off of this authority
@@ -219,7 +226,7 @@ class AuthoritiesList(AuthenticatedResource):
            :statuscode 403: unauthenticated
            :statuscode 200: no error
         """
-        data['creator'] = g.current_user
+        data["creator"] = g.current_user
         return service.create(**data)
 
 
@@ -294,6 +301,7 @@ class Authorities(AuthenticatedResource):
               PUT /authorities/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "name": "TestAuthority5",
@@ -387,7 +395,7 @@ class Authorities(AuthenticatedResource):
         authority = service.get(authority_id)
 
         if not authority:
-            return dict(message='Not Found'), 404
+            return dict(message="Not Found"), 404
 
         # all the authority role members should be allowed
         roles = [x.name for x in authority.roles]
@@ -396,10 +404,10 @@ class Authorities(AuthenticatedResource):
         if permission.can():
             return service.update(
                 authority_id,
-                owner=data['owner'],
-                description=data['description'],
-                active=data['active'],
-                roles=data['roles']
+                owner=data["owner"],
+                description=data["description"],
+                active=data["active"],
+                roles=data["roles"],
             )
 
         return dict(message="You are not authorized to update this authority."), 403
@@ -485,6 +493,26 @@ class CertificateAuthority(AuthenticatedResource):
 class AuthorityVisualizations(AuthenticatedResource):
     def get(self, authority_id):
         """
+        .. http:get:: /authorities/1/visualize
+
+           Authority visualization
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /certificates/1/visualize HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
                 {"name": "flare",
                     "children": [
                         {
@@ -499,14 +527,31 @@ class AuthorityVisualizations(AuthenticatedResource):
                                         {"name": "MergeEdge", "size": 743}
                                     ]
                                 }
+                            ]
                         }
-        ]}
+                    ]
+                }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
         """
         authority = service.get(authority_id)
-        return dict(name=authority.name, children=[{"name": c.name} for c in authority.certificates])
+        return dict(
+            name=authority.name,
+            children=[{"name": c.name} for c in authority.certificates],
+        )
 
 
-api.add_resource(AuthoritiesList, '/authorities', endpoint='authorities')
-api.add_resource(Authorities, '/authorities/<int:authority_id>', endpoint='authority')
-api.add_resource(AuthorityVisualizations, '/authorities/<int:authority_id>/visualize', endpoint='authority_visualizations')
-api.add_resource(CertificateAuthority, '/certificates/<int:certificate_id>/authority', endpoint='certificateAuthority')
+api.add_resource(AuthoritiesList, "/authorities", endpoint="authorities")
+api.add_resource(Authorities, "/authorities/<int:authority_id>", endpoint="authority")
+api.add_resource(
+    AuthorityVisualizations,
+    "/authorities/<int:authority_id>/visualize",
+    endpoint="authority_visualizations",
+)
+api.add_resource(
+    CertificateAuthority,
+    "/certificates/<int:certificate_id>/authority",
+    endpoint="certificateAuthority",
+)

lemur/authorizations/models.py

@@ -13,7 +13,7 @@ from lemur.plugins.base import plugins
 
 
 class Authorization(db.Model):
-    __tablename__ = 'pending_dns_authorizations'
+    __tablename__ = "pending_dns_authorizations"
     id = Column(Integer, primary_key=True, autoincrement=True)
     account_number = Column(String(128))
     domains = Column(JSONType)
@@ -25,7 +25,7 @@ class Authorization(db.Model):
         return plugins.get(self.plugin_name)
 
     def __repr__(self):
-        return "Authorization(id={id})".format(label=self.id)
+        return "Authorization(id={id})".format(id=self.id)
 
     def __init__(self, account_number, domains, dns_provider_type, options=None):
         self.account_number = account_number

lemur/certificates/cli.py

@@ -6,38 +6,36 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import sys
-import multiprocessing
-from tabulate import tabulate
-from sqlalchemy import or_
-
 from flask import current_app
-
-from flask_script import Manager
 from flask_principal import Identity, identity_changed
-
+from flask_script import Manager
+from sqlalchemy import or_
+from tabulate import tabulate
+from time import sleep
 
 from lemur import database
-from lemur.extensions import sentry
-from lemur.extensions import metrics
-from lemur.plugins.base import plugins
-from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
-from lemur.deployment import service as deployment_service
-from lemur.endpoints import service as endpoint_service
-from lemur.notifications.messaging import send_rotation_notification
-from lemur.domains.models import Domain
 from lemur.authorities.models import Authority
-from lemur.certificates.schemas import CertificateOutputSchema
+from lemur.authorities.service import get as authorities_get_by_id
 from lemur.certificates.models import Certificate
+from lemur.certificates.schemas import CertificateOutputSchema
 from lemur.certificates.service import (
     reissue_certificate,
     get_certificate_primitives,
     get_all_pending_reissue,
     get_by_name,
-    get_all_certs,
-    get
+    get_all_valid_certs,
+    get,
+    get_all_certs_attached_to_endpoint_without_autorotate,
+    revoke as revoke_certificate,
 )
-
 from lemur.certificates.verify import verify_string
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS, CRLReason
+from lemur.deployment import service as deployment_service
+from lemur.domains.models import Domain
+from lemur.endpoints import service as endpoint_service
+from lemur.extensions import sentry, metrics
+from lemur.notifications.messaging import send_rotation_notification
+from lemur.plugins.base import plugins
 
 manager = Manager(usage="Handles all certificate related tasks.")
 
@@ -56,11 +54,14 @@ def print_certificate_details(details):
         "\t[+] Authority: {authority_name}\n"
         "\t[+] Validity Start: {validity_start}\n"
         "\t[+] Validity End: {validity_end}\n".format(
-            common_name=details['commonName'],
-            sans=",".join(x['value'] for x in details['extensions']['subAltNames']['names']) or None,
-            authority_name=details['authority']['name'],
-            validity_start=details['validityStart'],
-            validity_end=details['validityEnd']
+            common_name=details["commonName"],
+            sans=",".join(
+                x["value"] for x in details["extensions"]["subAltNames"]["names"]
+            )
+            or None,
+            authority_name=details["authority"]["name"],
+            validity_start=details["validityStart"],
+            validity_end=details["validityEnd"],
         )
     )
 
@@ -118,15 +119,20 @@ def request_rotation(endpoint, certificate, message, commit):
             status = SUCCESS_METRIC_STATUS
 
         except Exception as e:
+            sentry.captureException(extra={"certificate_name": str(certificate.name),
+                                           "endpoint": str(endpoint.dnsname)})
+            current_app.logger.exception(
+                f"Error rotating certificate: {certificate.name}", exc_info=True
+            )
             print(
                 "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
-                    endpoint.name,
-                    certificate.name,
-                    e
+                    endpoint.name, certificate.name, e
                 )
             )
 
-    metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': status})
+    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status,
+                                                                 "certificate_name": str(certificate.name),
+                                                                 "endpoint": str(endpoint.dnsname)})
 
 
 def request_reissue(certificate, commit):
@@ -153,22 +159,53 @@ def request_reissue(certificate, commit):
         status = SUCCESS_METRIC_STATUS
 
     except Exception as e:
-        sentry.captureException()
-        current_app.logger.exception("Error reissuing certificate.", exc_info=True)
-        print(
-            "[!] Failed to reissue certificates. Reason: {}".format(
-                e
+        sentry.captureException(extra={"certificate_name": str(certificate.name)})
+        current_app.logger.exception(
+            f"Error reissuing certificate: {certificate.name}", exc_info=True
         )
+        print(f"[!] Failed to reissue certificate: {certificate.name}. Reason: {e}")
+
+    metrics.send(
+        "certificate_reissue",
+        "counter",
+        1,
+        metric_tags={"status": status, "certificate": certificate.name},
     )
 
-    metrics.send('certificate_reissue', 'counter', 1, metric_tags={'status': status})
 
-
-@manager.option('-e', '--endpoint', dest='endpoint_name', help='Name of the endpoint you wish to rotate.')
-@manager.option('-n', '--new-certificate', dest='new_certificate_name', help='Name of the certificate you wish to rotate to.')
-@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to rotate.')
-@manager.option('-a', '--notify', dest='message', action='store_true', help='Send a rotation notification to the certificates owner.')
-@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+@manager.option(
+    "-e",
+    "--endpoint",
+    dest="endpoint_name",
+    help="Name of the endpoint you wish to rotate.",
+)
+@manager.option(
+    "-n",
+    "--new-certificate",
+    dest="new_certificate_name",
+    help="Name of the certificate you wish to rotate to.",
+)
+@manager.option(
+    "-o",
+    "--old-certificate",
+    dest="old_certificate_name",
+    help="Name of the certificate you wish to rotate.",
+)
+@manager.option(
+    "-a",
+    "--notify",
+    dest="message",
+    action="store_true",
+    help="Send a rotation notification to the certificates owner.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
 def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, commit):
     """
     Rotates an endpoint and reissues it if it has not already been replaced. If it has
@@ -181,45 +218,265 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
     status = FAILURE_METRIC_STATUS
 
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+    }
+
     try:
         old_cert = validate_certificate(old_certificate_name)
         new_cert = validate_certificate(new_certificate_name)
         endpoint = validate_endpoint(endpoint_name)
 
         if endpoint and new_cert:
-            print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name))
+            print(
+                f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
+            )
+            log_data["message"] = "Rotating one endpoint"
+            log_data["endpoint"] = endpoint.dnsname
+            log_data["certificate"] = new_cert.name
             request_rotation(endpoint, new_cert, message, commit)
+            current_app.logger.info(log_data)
 
         elif old_cert and new_cert:
-            print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name))
-
+            print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
+            log_data["certificate"] = new_cert.name
+            log_data["certificate_old"] = old_cert.name
+            log_data["message"] = "Rotating endpoint from old to new cert"
             for endpoint in old_cert.endpoints:
-                print("[+] Rotating {0}".format(endpoint.name))
+                print(f"[+] Rotating {endpoint.name}")
+                log_data["endpoint"] = endpoint.dnsname
                 request_rotation(endpoint, new_cert, message, commit)
+                current_app.logger.info(log_data)
 
         else:
+            # No certificate name or endpoint is provided. We will now fetch all endpoints,
+            # which are associated with a certificate that has been replaced
             print("[+] Rotating all endpoints that have new certificates available")
             for endpoint in endpoint_service.get_all_pending_rotation():
-                if len(endpoint.certificate.replaced) == 1:
-                    print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name))
+
+                log_data["message"] = "Rotating endpoint from old to new cert"
+                if len(endpoint.certificate.replaced) > 1:
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
+                                          f"{len(endpoint.certificate.replaced)}"
+
+                log_data["endpoint"] = endpoint.dnsname
+                log_data["certificate"] = endpoint.certificate.replaced[0].name
+                print(
+                    f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
+                )
                 request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
-                else:
-                    metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-                    print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format(
-                        endpoint.name
-                    ))
+                current_app.logger.info(log_data)
 
         status = SUCCESS_METRIC_STATUS
         print("[+] Done!")
 
     except Exception as e:
-        sentry.captureException()
+        sentry.captureException(
+            extra={
+                "old_certificate_name": str(old_certificate_name),
+                "new_certificate_name": str(new_certificate_name),
+                "endpoint_name": str(endpoint_name),
+                "message": str(message),
+            }
+        )
 
-    metrics.send('endpoint_rotation_job', 'counter', 1, metric_tags={'status': status})
+    metrics.send(
+        "endpoint_rotation_job",
+        "counter",
+        1,
+        metric_tags={
+            "status": status,
+            "old_certificate_name": str(old_certificate_name),
+            "new_certificate_name": str(new_certificate_name),
+            "endpoint_name": str(endpoint_name),
+            "message": str(message),
+            "endpoint": str(globals().get("endpoint")),
+        },
+    )
 
 
-@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to reissue.')
-@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+def request_rotation_region(endpoint, new_cert, message, commit, log_data, region):
+    if region in endpoint.dnsname:
+        log_data["message"] = "Rotating endpoint in region"
+        request_rotation(endpoint, new_cert, message, commit)
+    else:
+        log_data["message"] = "Skipping rotation, region mismatch"
+
+    print(log_data)
+    current_app.logger.info(log_data)
+
+
+@manager.option(
+    "-e",
+    "--endpoint",
+    dest="endpoint_name",
+    help="Name of the endpoint you wish to rotate.",
+)
+@manager.option(
+    "-n",
+    "--new-certificate",
+    dest="new_certificate_name",
+    help="Name of the certificate you wish to rotate to.",
+)
+@manager.option(
+    "-o",
+    "--old-certificate",
+    dest="old_certificate_name",
+    help="Name of the certificate you wish to rotate.",
+)
+@manager.option(
+    "-a",
+    "--notify",
+    dest="message",
+    action="store_true",
+    help="Send a rotation notification to the certificates owner.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
+@manager.option(
+    "-r",
+    "--region",
+    dest="region",
+    required=True,
+    help="Region in which to rotate the endpoint.",
+)
+def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, message, commit, region):
+    """
+    Rotates an endpoint in a defined region it if it has not already been replaced. If it has
+    been replaced, will use the replacement certificate for the rotation.
+    :param old_certificate_name: Name of the certificate you wish to rotate.
+    :param new_certificate_name: Name of the certificate you wish to rotate to.
+    :param endpoint_name: Name of the endpoint you wish to rotate.
+    :param message: Send a rotation notification to the certificates owner.
+    :param commit: Persist changes.
+    :param region: Region in which to rotate the endpoint.
+    #todo: merge this method with rotate()
+    """
+    if commit:
+        print("[!] Running in COMMIT mode.")
+
+    print("[+] Starting endpoint rotation.")
+    status = FAILURE_METRIC_STATUS
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "region": region,
+    }
+
+    try:
+        old_cert = validate_certificate(old_certificate_name)
+        new_cert = validate_certificate(new_certificate_name)
+        endpoint = validate_endpoint(endpoint_name)
+
+        if endpoint and new_cert:
+            log_data["endpoint"] = endpoint.dnsname
+            log_data["certificate"] = new_cert.name
+            request_rotation_region(endpoint, new_cert, message, commit, log_data, region)
+
+        elif old_cert and new_cert:
+            log_data["certificate"] = new_cert.name
+            log_data["certificate_old"] = old_cert.name
+            log_data["message"] = "Rotating endpoint from old to new cert"
+            print(log_data)
+            current_app.logger.info(log_data)
+            for endpoint in old_cert.endpoints:
+                log_data["endpoint"] = endpoint.dnsname
+                request_rotation_region(endpoint, new_cert, message, commit, log_data, region)
+
+        else:
+            log_data["message"] = "Rotating all endpoints that have new certificates available"
+            print(log_data)
+            current_app.logger.info(log_data)
+            all_pending_rotation_endpoints = endpoint_service.get_all_pending_rotation()
+            for endpoint in all_pending_rotation_endpoints:
+                log_data["endpoint"] = endpoint.dnsname
+                if region not in endpoint.dnsname:
+                    log_data["message"] = "Skipping rotation, region mismatch"
+                    print(log_data)
+                    current_app.logger.info(log_data)
+                    metrics.send(
+                        "endpoint_rotation_region_skipped",
+                        "counter",
+                        1,
+                        metric_tags={
+                            "region": region,
+                            "new_certificate_name": str(endpoint.certificate.replaced[0].name),
+                            "endpoint_name": str(endpoint.dnsname),
+                        },
+                    )
+                    continue
+
+                log_data["certificate"] = endpoint.certificate.replaced[0].name
+                log_data["message"] = "Rotating all endpoints in region"
+                if len(endpoint.certificate.replaced) > 1:
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
+                                          f"{len(endpoint.certificate.replaced)}"
+
+                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
+                current_app.logger.info(log_data)
+
+                metrics.send(
+                    "endpoint_rotation_region",
+                    "counter",
+                    1,
+                    metric_tags={
+                        "status": FAILURE_METRIC_STATUS,
+                        "new_certificate_name": str(log_data["certificate"]),
+                        "endpoint_name": str(endpoint.dnsname),
+                        "message": str(message),
+                        "region": str(region),
+                    },
+                )
+        status = SUCCESS_METRIC_STATUS
+        print("[+] Done!")
+
+    except Exception as e:
+        sentry.captureException(
+            extra={
+                "old_certificate_name": str(old_certificate_name),
+                "new_certificate_name": str(new_certificate_name),
+                "endpoint": str(endpoint_name),
+                "message": str(message),
+                "region": str(region),
+            }
+        )
+
+    metrics.send(
+        "endpoint_rotation_region_job",
+        "counter",
+        1,
+        metric_tags={
+            "status": status,
+            "old_certificate_name": str(old_certificate_name),
+            "new_certificate_name": str(new_certificate_name),
+            "endpoint_name": str(endpoint_name),
+            "message": str(message),
+            "endpoint": str(globals().get("endpoint")),
+            "region": str(region),
+        },
+    )
+
+
+@manager.option(
+    "-o",
+    "--old-certificate",
+    dest="old_certificate_name",
+    help="Name of the certificate you wish to reissue.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
 def reissue(old_certificate_name, commit):
     """
     Reissues certificate with the same parameters as it was originally issued with.
@@ -247,75 +504,93 @@ def reissue(old_certificate_name, commit):
     except Exception as e:
         sentry.captureException()
         current_app.logger.exception("Error reissuing certificate.", exc_info=True)
-        print(
-            "[!] Failed to reissue certificates. Reason: {}".format(
-                e
-            )
+        print("[!] Failed to reissue certificates. Reason: {}".format(e))
+
+    metrics.send(
+        "certificate_reissue_job", "counter", 1, metric_tags={"status": status}
     )
 
-    metrics.send('certificate_reissue_job', 'counter', 1, metric_tags={'status': status})
 
-
-@manager.option('-f', '--fqdns', dest='fqdns', help='FQDNs to query. Multiple fqdns specified via comma.')
-@manager.option('-i', '--issuer', dest='issuer', help='Issuer to query for.')
-@manager.option('-o', '--owner', dest='owner', help='Owner to query for.')
-@manager.option('-e', '--expired', dest='expired', type=bool, default=False, help='Include expired certificates.')
+@manager.option(
+    "-f",
+    "--fqdns",
+    dest="fqdns",
+    help="FQDNs to query. Multiple fqdns specified via comma.",
+)
+@manager.option("-i", "--issuer", dest="issuer", help="Issuer to query for.")
+@manager.option("-o", "--owner", dest="owner", help="Owner to query for.")
+@manager.option(
+    "-e",
+    "--expired",
+    dest="expired",
+    type=bool,
+    default=False,
+    help="Include expired certificates.",
+)
 def query(fqdns, issuer, owner, expired):
     """Prints certificates that match the query params."""
     table = []
 
     q = database.session_query(Certificate)
-
-    sub_query = database.session_query(Authority.id) \
-        .filter(Authority.name.ilike('%{0}%'.format(issuer))) \
+    if issuer:
+        sub_query = (
+            database.session_query(Authority.id)
+            .filter(Authority.name.ilike("%{0}%".format(issuer)))
             .subquery()
+        )
 
         q = q.filter(
             or_(
-            Certificate.issuer.ilike('%{0}%'.format(issuer)),
-            Certificate.authority_id.in_(sub_query)
+                Certificate.issuer.ilike("%{0}%".format(issuer)),
+                Certificate.authority_id.in_(sub_query),
             )
         )
-
-    q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner)))
+    if owner:
+        q = q.filter(Certificate.owner.ilike("%{0}%".format(owner)))
 
     if not expired:
         q = q.filter(Certificate.expired == False)  # noqa
 
-    for f in fqdns.split(','):
+    if fqdns:
+        for f in fqdns.split(","):
             q = q.filter(
                 or_(
-                Certificate.cn.ilike('%{0}%'.format(f)),
-                Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f)))
+                    Certificate.cn.ilike("%{0}%".format(f)),
+                    Certificate.domains.any(Domain.name.ilike("%{0}%".format(f))),
                 )
             )
 
     for c in q.all():
         table.append([c.id, c.name, c.owner, c.issuer])
 
-    print(tabulate(table, headers=['Id', 'Name', 'Owner', 'Issuer'], tablefmt='csv'))
+    print(tabulate(table, headers=["Id", "Name", "Owner", "Issuer"], tablefmt="csv"))
 
 
 def worker(data, commit, reason):
-    parts = [x for x in data.split(' ') if x]
+    parts = [x for x in data.split(" ") if x]
     try:
         cert = get(int(parts[0].strip()))
-        plugin = plugins.get(cert.authority.plugin_name)
 
-        print('[+] Revoking certificate. Id: {0} Name: {1}'.format(cert.id, cert.name))
+        print("[+] Revoking certificate. Id: {0} Name: {1}".format(cert.id, cert.name))
         if commit:
-            plugin.revoke_certificate(cert, reason)
+            revoke_certificate(cert, reason)
 
-        metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+        metrics.send(
+            "certificate_revoke",
+            "counter",
+            1,
+            metric_tags={"status": SUCCESS_METRIC_STATUS},
+        )
 
     except Exception as e:
         sentry.captureException()
-        metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-        print(
-            "[!] Failed to revoke certificates. Reason: {}".format(
-                e
-            )
+        metrics.send(
+            "certificate_revoke",
+            "counter",
+            1,
+            metric_tags={"status": FAILURE_METRIC_STATUS},
         )
+        print("[!] Failed to revoke certificates. Reason: {}".format(e))
 
 
 @manager.command
@@ -324,27 +599,48 @@ def clear_pending():
     Function clears all pending certificates.
     :return:
     """
-    v = plugins.get('verisign-issuer')
+    v = plugins.get("verisign-issuer")
     v.clear_pending_certificates()
 
 
-@manager.option('-p', '--path', dest='path', help='Absolute file path to a Lemur query csv.')
-@manager.option('-r', '--reason', dest='reason', help='Reason to revoke certificate.')
-@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
-def revoke(path, reason, commit):
+@manager.option("-p", "--path", dest="path", help="Absolute file path to a Lemur query csv.")
+@manager.option("-id", "--certid", dest="cert_id", help="ID of the certificate to be revoked")
+@manager.option("-r", "--reason", dest="reason", default="unspecified", help="CRL Reason as per RFC 5280 section 5.3.1")
+@manager.option("-m", "--message", dest="message", help="Message explaining reason for revocation")
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
+def revoke(path, cert_id, reason, message, commit):
     """
     Revokes given certificate.
     """
+    if not path and not cert_id:
+        print("[!] No input certificates mentioned to revoke")
+        return
+    if path and cert_id:
+        print("[!] Please mention single certificate id (-id) or input file (-p)")
+        return
+
     if commit:
         print("[!] Running in COMMIT mode.")
 
     print("[+] Starting certificate revocation.")
 
-    with open(path, 'r') as f:
-        args = [[x, commit, reason] for x in f.readlines()[2:]]
+    if reason not in CRLReason.__members__:
+        reason = CRLReason.unspecified.name
+    comments = {"comments": message, "crl_reason": reason}
 
-    with multiprocessing.Pool(processes=3) as pool:
-        pool.starmap(worker, args)
+    if cert_id:
+        worker(cert_id, commit, comments)
+    else:
+        with open(path, "r") as f:
+            for x in f.readlines()[2:]:
+                worker(x, commit, comments)
 
 
 @manager.command
@@ -356,18 +652,126 @@ def check_revoked():
     encounters an issue with verification it marks the certificate status
     as `unknown`.
     """
-    for cert in get_all_certs():
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Checking for revoked Certificates"
+    }
+
+    certs = get_all_valid_certs(current_app.config.get("SUPPORTED_REVOCATION_AUTHORITY_PLUGINS", []))
+    for cert in certs:
         try:
             if cert.chain:
                 status = verify_string(cert.body, cert.chain)
             else:
                 status = verify_string(cert.body, "")
 
-            cert.status = 'valid' if status else 'revoked'
+            cert.status = "valid" if status else "revoked"
+
+            if cert.status == "revoked":
+                log_data["valid"] = cert.status
+                log_data["certificate_name"] = cert.name
+                log_data["certificate_id"] = cert.id
+                metrics.send(
+                    "certificate_revoked",
+                    "counter",
+                    1,
+                    metric_tags={"status": log_data["valid"],
+                                 "certificate_name": log_data["certificate_name"],
+                                 "certificate_id": log_data["certificate_id"]},
+                )
+                current_app.logger.info(log_data)
 
         except Exception as e:
             sentry.captureException()
             current_app.logger.exception(e)
-            cert.status = 'unknown'
+            cert.status = "unknown"
 
         database.update(cert)
+
+
+@manager.command
+def automatically_enable_autorotate():
+    """
+    This function automatically enables auto-rotation for unexpired certificates that are
+    attached to an endpoint but do not have autorotate enabled.
+
+    WARNING: This will overwrite the Auto-rotate toggle!
+    """
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Enabling auto-rotate for certificate"
+    }
+
+    permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", [])
+
+    eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate()
+    for cert in eligible_certs:
+
+        if cert.authority_id not in permitted_authorities:
+            continue
+
+        log_data["certificate"] = cert.name
+        log_data["certificate_id"] = cert.id
+        log_data["authority_id"] = cert.authority_id
+        log_data["authority_name"] = authorities_get_by_id(cert.authority_id).name
+        if cert.destinations:
+            log_data["destination_names"] = ', '.join([d.label for d in cert.destinations])
+        else:
+            log_data["destination_names"] = "NONE"
+        current_app.logger.info(log_data)
+        metrics.send("automatically_enable_autorotate",
+                     "counter", 1,
+                     metric_tags={"certificate": log_data["certificate"],
+                                  "certificate_id": log_data["certificate_id"],
+                                  "authority_id": log_data["authority_id"],
+                                  "authority_name": log_data["authority_name"],
+                                  "destination_names": log_data["destination_names"]
+                                  })
+        cert.rotation = True
+        database.update(cert)
+
+
+@manager.command
+def deactivate_entrust_certificates():
+    """
+    Attempt to deactivate test certificates issued by Entrust
+    """
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Deactivating Entrust certificates"
+    }
+
+    certificates = get_all_valid_certs(['entrust-issuer'])
+    entrust_plugin = plugins.get('entrust-issuer')
+    for index, cert in enumerate(certificates):
+        if (index % 10) == 0:
+            # Entrust enforces a 10 request per 30s rate limit
+            sleep(30)
+        try:
+            response = entrust_plugin.deactivate_certificate(cert)
+            if response == 200:
+                cert.status = "revoked"
+            else:
+                cert.status = "unknown"
+
+            log_data["valid"] = cert.status
+            log_data["certificate_name"] = cert.name
+            log_data["certificate_id"] = cert.id
+            metrics.send(
+                "certificate_deactivate",
+                "counter",
+                1,
+                metric_tags={"status": log_data["valid"],
+                             "certificate_name": log_data["certificate_name"],
+                             "certificate_id": log_data["certificate_id"]},
+            )
+            current_app.logger.info(log_data)
+
+            database.update(cert)
+
+        except Exception as e:
+            current_app.logger.info(log_data)
+            sentry.captureException()
+            current_app.logger.exception(e)

lemur/certificates/hooks.py

@@ -12,21 +12,30 @@ import subprocess
 
 from flask import current_app
 
-from lemur.certificates.service import csr_created, csr_imported, certificate_issued, certificate_imported
+from lemur.certificates.service import (
+    csr_created,
+    csr_imported,
+    certificate_issued,
+    certificate_imported,
+)
 
 
 def csr_dump_handler(sender, csr, **kwargs):
     try:
-        subprocess.run(['openssl', 'req', '-text', '-noout', '-reqopt', 'no_sigdump,no_pubkey'],
-                       input=csr.encode('utf8'))
+        subprocess.run(
+            ["openssl", "req", "-text", "-noout", "-reqopt", "no_sigdump,no_pubkey"],
+            input=csr.encode("utf8"),
+        )
     except Exception as err:
         current_app.logger.warning("Error inspecting CSR: %s", err)
 
 
 def cert_dump_handler(sender, certificate, **kwargs):
     try:
-        subprocess.run(['openssl', 'x509', '-text', '-noout', '-certopt', 'no_sigdump,no_pubkey'],
-                       input=certificate.body.encode('utf8'))
+        subprocess.run(
+            ["openssl", "x509", "-text", "-noout", "-certopt", "no_sigdump,no_pubkey"],
+            input=certificate.body.encode("utf8"),
+        )
     except Exception as err:
         current_app.logger.warning("Error inspecting certificate: %s", err)
 

lemur/certificates/models.py

@@ -5,49 +5,55 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import arrow
 from datetime import timedelta
 
-from flask import current_app
-
+import arrow
 from cryptography import x509
-from cryptography.hazmat.primitives.asymmetric import rsa
-
+from flask import current_app
 from idna.core import InvalidCodepoint
-
+from sqlalchemy import (
+    event,
+    Integer,
+    ForeignKey,
+    String,
+    DefaultClause,
+    func,
+    Column,
+    Text,
+    Boolean,
+    Index,
+)
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import case, extract
-from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
-
 from sqlalchemy_utils.types.arrow import ArrowType
+from werkzeug.utils import cached_property
 
-import lemur.common.utils
-
-from lemur.database import db
-from lemur.extensions import sentry
-
-from lemur.utils import Vault
-from lemur.common import defaults
-
-from lemur.plugins.base import plugins
-
-from lemur.extensions import metrics
+from lemur.common import defaults, utils, validators
 from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
-
-from lemur.models import certificate_associations, certificate_source_associations, \
-    certificate_destination_associations, certificate_notification_associations, \
-    certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations
-
+from lemur.database import db
 from lemur.domains.models import Domain
+from lemur.extensions import metrics
+from lemur.extensions import sentry
+from lemur.models import (
+    certificate_associations,
+    certificate_source_associations,
+    certificate_destination_associations,
+    certificate_notification_associations,
+    certificate_replacement_associations,
+    roles_certificates,
+    pending_cert_replacement_associations,
+)
+from lemur.plugins.base import plugins
 from lemur.policies.models import RotationPolicy
+from lemur.utils import Vault
 
 
 def get_sequence(name):
-    if '-' not in name:
+    if "-" not in name:
         return name, None
 
-    parts = name.split('-')
+    parts = name.split("-")
 
     # see if we have an int at the end of our name
     try:
@@ -59,22 +65,26 @@ def get_sequence(name):
     if len(parts[-1]) == 8:
         return name, None
 
-    root = '-'.join(parts[:-1])
+    root = "-".join(parts[:-1])
     return root, seq
 
 
 def get_or_increase_name(name, serial):
-    certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).all()
+    certificates = Certificate.query.filter(Certificate.name == name).all()
 
     if not certificates:
         return name
 
-    serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper())
-    certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(serial_name))).all()
+    serial_name = "{0}-{1}".format(name, hex(int(serial))[2:].upper())
+    certificates = Certificate.query.filter(Certificate.name == serial_name).all()
 
     if not certificates:
         return serial_name
 
+    certificates = Certificate.query.filter(
+        Certificate.name.ilike("{0}%".format(serial_name))
+    ).all()
+
     ends = [0]
     root, end = get_sequence(serial_name)
     for cert in certificates:
@@ -82,12 +92,29 @@ def get_or_increase_name(name, serial):
         if end:
             ends.append(end)
 
-    return '{0}-{1}'.format(root, max(ends) + 1)
+    return "{0}-{1}".format(root, max(ends) + 1)
 
 
 class Certificate(db.Model):
-    __tablename__ = 'certificates'
+    __tablename__ = "certificates"
+    __table_args__ = (
+        Index(
+            "ix_certificates_cn",
+            "cn",
+            postgresql_ops={"cn": "gin_trgm_ops"},
+            postgresql_using="gin",
+        ),
+        Index(
+            "ix_certificates_name",
+            "name",
+            postgresql_ops={"name": "gin_trgm_ops"},
+            postgresql_using="gin",
+        ),
+    )
     id = Column(Integer, primary_key=True)
+    ix = Index(
+        "ix_certificates_id_desc", id.desc(), postgresql_using="btree", unique=True
+    )
     external_id = Column(String(128))
     owner = Column(String(128), nullable=False)
     name = Column(String(256), unique=True)
@@ -96,17 +123,22 @@ class Certificate(db.Model):
 
     body = Column(Text(), nullable=False)
     chain = Column(Text())
+    csr = Column(Text())
     private_key = Column(Vault)
 
     issuer = Column(String(128))
     serial = Column(String(128))
     cn = Column(String(128))
-    deleted = Column(Boolean, index=True)
-    dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='cascade'), nullable=True)
+    deleted = Column(Boolean, index=True, default=False)
+    dns_provider_id = Column(
+        Integer(), ForeignKey("dns_providers.id", ondelete="CASCADE"), nullable=True
+    )
 
     not_before = Column(ArrowType)
     not_after = Column(ArrowType)
-    date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
+    not_after_ix = Index("ix_certificates_not_after", not_after.desc())
+
+    date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False)
 
     signing_algorithm = Column(String(128))
     status = Column(String(128))
@@ -114,35 +146,54 @@ class Certificate(db.Model):
     san = Column(String(1024))  # TODO this should be migrated to boolean
 
     rotation = Column(Boolean, default=False)
-    user_id = Column(Integer, ForeignKey('users.id'))
-    authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
-    root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
-    rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
+    user_id = Column(Integer, ForeignKey("users.id"))
+    authority_id = Column(Integer, ForeignKey("authorities.id", ondelete="CASCADE"))
+    root_authority_id = Column(
+        Integer, ForeignKey("authorities.id", ondelete="CASCADE")
+    )
+    rotation_policy_id = Column(Integer, ForeignKey("rotation_policies.id"))
+    key_type = Column(String(128))
 
-    notifications = relationship('Notification', secondary=certificate_notification_associations, backref='certificate')
-    destinations = relationship('Destination', secondary=certificate_destination_associations, backref='certificate')
-    sources = relationship('Source', secondary=certificate_source_associations, backref='certificate')
-    domains = relationship('Domain', secondary=certificate_associations, backref='certificate')
-    roles = relationship('Role', secondary=roles_certificates, backref='certificate')
-    replaces = relationship('Certificate',
+    notifications = relationship(
+        "Notification",
+        secondary=certificate_notification_associations,
+        backref="certificate",
+    )
+    destinations = relationship(
+        "Destination",
+        secondary=certificate_destination_associations,
+        backref="certificate",
+    )
+    sources = relationship(
+        "Source", secondary=certificate_source_associations, backref="certificate"
+    )
+    domains = relationship(
+        "Domain", secondary=certificate_associations, backref="certificate"
+    )
+    roles = relationship("Role", secondary=roles_certificates, backref="certificate")
+    replaces = relationship(
+        "Certificate",
         secondary=certificate_replacement_associations,
         primaryjoin=id == certificate_replacement_associations.c.certificate_id,  # noqa
-                            secondaryjoin=id == certificate_replacement_associations.c.replaced_certificate_id,  # noqa
-                            backref='replaced')
+        secondaryjoin=id
+        == certificate_replacement_associations.c.replaced_certificate_id,  # noqa
+        backref="replaced",
+    )
 
-    replaced_by_pending = relationship('PendingCertificate',
+    replaced_by_pending = relationship(
+        "PendingCertificate",
         secondary=pending_cert_replacement_associations,
-                                       backref='pending_replace',
-                                       viewonly=True)
+        backref="pending_replace",
+    )
 
-    logs = relationship('Log', backref='certificate')
-    endpoints = relationship('Endpoint', backref='certificate')
+    logs = relationship("Log", backref="certificate")
+    endpoints = relationship("Endpoint", backref="certificate")
     rotation_policy = relationship("RotationPolicy")
-
-    sensitive_fields = ('private_key',)
+    sensitive_fields = ("private_key",)
 
     def __init__(self, **kwargs):
-        cert = lemur.common.utils.parse_certificate(kwargs['body'])
+        self.body = kwargs["body"].strip()
+        cert = self.parsed_cert
 
         self.issuer = defaults.issuer(cert)
         self.cn = defaults.common_name(cert)
@@ -152,72 +203,110 @@ class Certificate(db.Model):
         self.serial = defaults.serial(cert)
 
         # when destinations are appended they require a valid name.
-        if kwargs.get('name'):
-            self.name = get_or_increase_name(defaults.text_to_slug(kwargs['name']), self.serial)
+        if kwargs.get("name"):
+            self.name = get_or_increase_name(
+                defaults.text_to_slug(kwargs["name"]), self.serial
+            )
         else:
             self.name = get_or_increase_name(
-                defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san), self.serial)
+                defaults.certificate_name(
+                    self.cn, self.issuer, self.not_before, self.not_after, self.san
+                ),
+                self.serial,
+            )
 
-        self.owner = kwargs['owner']
-        self.body = kwargs['body'].strip()
+        self.owner = kwargs["owner"]
 
-        if kwargs.get('private_key'):
-            self.private_key = kwargs['private_key'].strip()
+        if kwargs.get("private_key"):
+            self.private_key = kwargs["private_key"].strip()
 
-        if kwargs.get('chain'):
-            self.chain = kwargs['chain'].strip()
+        if kwargs.get("chain"):
+            self.chain = kwargs["chain"].strip()
 
-        self.notify = kwargs.get('notify', True)
-        self.destinations = kwargs.get('destinations', [])
-        self.notifications = kwargs.get('notifications', [])
-        self.description = kwargs.get('description')
-        self.roles = list(set(kwargs.get('roles', [])))
-        self.replaces = kwargs.get('replaces', [])
-        self.rotation = kwargs.get('rotation')
-        self.rotation_policy = kwargs.get('rotation_policy')
+        if kwargs.get("csr"):
+            self.csr = kwargs["csr"].strip()
+
+        self.notify = kwargs.get("notify", True)
+        self.destinations = kwargs.get("destinations", [])
+        self.notifications = kwargs.get("notifications", [])
+        self.description = kwargs.get("description")
+        self.roles = list(set(kwargs.get("roles", [])))
+        self.replaces = kwargs.get("replaces", [])
+        self.rotation = kwargs.get("rotation")
+        self.rotation_policy = kwargs.get("rotation_policy")
+        self.key_type = kwargs.get("key_type")
         self.signing_algorithm = defaults.signing_algorithm(cert)
         self.bits = defaults.bitstrength(cert)
-        self.external_id = kwargs.get('external_id')
-        self.authority_id = kwargs.get('authority_id')
-        self.dns_provider_id = kwargs.get('dns_provider_id')
+        self.external_id = kwargs.get("external_id")
+        self.authority_id = kwargs.get("authority_id")
+        self.dns_provider_id = kwargs.get("dns_provider_id")
 
         for domain in defaults.domains(cert):
             self.domains.append(Domain(name=domain))
 
+        # Check integrity before saving anything into the database.
+        # For user-facing API calls, validation should also be done in schema validators.
+        self.check_integrity()
+
+    def check_integrity(self):
+        """
+        Integrity checks: Does the cert have a valid chain and matching private key?
+        """
+        if self.private_key:
+            validators.verify_private_key_match(
+                utils.parse_private_key(self.private_key),
+                self.parsed_cert,
+                error_class=AssertionError,
+            )
+
+        if self.chain:
+            chain = [self.parsed_cert] + utils.parse_cert_chain(self.chain)
+            validators.verify_cert_chain(chain, error_class=AssertionError)
+
+    @cached_property
+    def parsed_cert(self):
+        assert self.body, "Certificate body not set"
+        return utils.parse_certificate(self.body)
+
     @property
     def active(self):
         return self.notify
 
     @property
     def organization(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        return defaults.organization(cert)
+        return defaults.organization(self.parsed_cert)
 
     @property
     def organizational_unit(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        return defaults.organizational_unit(cert)
+        return defaults.organizational_unit(self.parsed_cert)
 
     @property
     def country(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        return defaults.country(cert)
+        return defaults.country(self.parsed_cert)
 
     @property
     def state(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        return defaults.state(cert)
+        return defaults.state(self.parsed_cert)
 
     @property
     def location(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        return defaults.location(cert)
+        return defaults.location(self.parsed_cert)
 
+    @property
+    def distinguished_name(self):
+        return self.parsed_cert.subject.rfc4514_string()
+
+    """
+    # Commenting this property as key_type is now added as a column. This code can be removed in future.
     @property
     def key_type(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        if isinstance(cert.public_key(), rsa.RSAPublicKey):
-            return 'RSA{key_size}'.format(key_size=cert.public_key().key_size)
+        if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey):
+            return "RSA{key_size}".format(
+                key_size=self.parsed_cert.public_key().key_size
+            )
+        elif isinstance(self.parsed_cert.public_key(), ec.EllipticCurvePublicKey):
+            return get_key_type_from_ec_curve(self.parsed_cert.public_key().curve.name)
+    """
 
     @property
     def validity_remaining(self):
@@ -229,41 +318,38 @@ class Certificate(db.Model):
 
     @property
     def subject(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        return cert.subject
+        return self.parsed_cert.subject
 
     @property
     def public_key(self):
-        cert = lemur.common.utils.parse_certificate(self.body)
-        return cert.public_key()
+        return self.parsed_cert.public_key()
 
     @hybrid_property
     def expired(self):
-        if self.not_after <= arrow.utcnow():
+        # can't compare offset-naive and offset-aware datetimes
+        if arrow.Arrow.fromdatetime(self.not_after) <= arrow.utcnow():
             return True
 
     @expired.expression
     def expired(cls):
-        return case(
-            [
-                (cls.not_after <= arrow.utcnow(), True)
-            ],
-            else_=False
-        )
+        return case([(cls.not_after <= arrow.utcnow(), True)], else_=False)
 
     @hybrid_property
     def revoked(self):
-        if 'revoked' == self.status:
+        if "revoked" == self.status:
             return True
 
     @revoked.expression
     def revoked(cls):
-        return case(
-            [
-                (cls.status == 'revoked', True)
-            ],
-            else_=False
-        )
+        return case([(cls.status == "revoked", True)], else_=False)
+
+    @hybrid_property
+    def has_private_key(self):
+        return self.private_key is not None
+
+    @has_private_key.expression
+    def has_private_key(cls):
+        return case([(cls.private_key.is_(None), True)], else_=False)
 
     @hybrid_property
     def in_rotation_window(self):
@@ -286,67 +372,65 @@ class Certificate(db.Model):
         :return:
         """
         return case(
-            [
-                (extract('day', cls.not_after - func.now()) <= RotationPolicy.days, True)
-            ],
-            else_=False
+            [(extract("day", cls.not_after - func.now()) <= RotationPolicy.days, True)],
+            else_=False,
         )
 
     @property
     def extensions(self):
         # setup default values
-        return_extensions = {
-            'sub_alt_names': {'names': []}
-        }
+        return_extensions = {"sub_alt_names": {"names": []}}
 
         try:
-            cert = lemur.common.utils.parse_certificate(self.body)
-            for extension in cert.extensions:
+            for extension in self.parsed_cert.extensions:
                 value = extension.value
                 if isinstance(value, x509.BasicConstraints):
-                    return_extensions['basic_constraints'] = value
+                    return_extensions["basic_constraints"] = value
 
                 elif isinstance(value, x509.SubjectAlternativeName):
-                    return_extensions['sub_alt_names']['names'] = value
+                    return_extensions["sub_alt_names"]["names"] = value
 
                 elif isinstance(value, x509.ExtendedKeyUsage):
-                    return_extensions['extended_key_usage'] = value
+                    return_extensions["extended_key_usage"] = value
 
                 elif isinstance(value, x509.KeyUsage):
-                    return_extensions['key_usage'] = value
+                    return_extensions["key_usage"] = value
 
                 elif isinstance(value, x509.SubjectKeyIdentifier):
-                    return_extensions['subject_key_identifier'] = {'include_ski': True}
+                    return_extensions["subject_key_identifier"] = {"include_ski": True}
 
                 elif isinstance(value, x509.AuthorityInformationAccess):
-                    return_extensions['certificate_info_access'] = {'include_aia': True}
+                    return_extensions["certificate_info_access"] = {"include_aia": True}
 
                 elif isinstance(value, x509.AuthorityKeyIdentifier):
-                    aki = {
-                        'use_key_identifier': False,
-                        'use_authority_cert': False
-                    }
+                    aki = {"use_key_identifier": False, "use_authority_cert": False}
 
                     if value.key_identifier:
-                        aki['use_key_identifier'] = True
+                        aki["use_key_identifier"] = True
 
                     if value.authority_cert_issuer:
-                        aki['use_authority_cert'] = True
+                        aki["use_authority_cert"] = True
 
-                    return_extensions['authority_key_identifier'] = aki
+                    return_extensions["authority_key_identifier"] = aki
 
                 elif isinstance(value, x509.CRLDistributionPoints):
-                    return_extensions['crl_distribution_points'] = {'include_crl_dp': value}
+                    return_extensions["crl_distribution_points"] = {
+                        "include_crl_dp": value
+                    }
 
                 # TODO: Not supporting custom OIDs yet. https://github.com/Netflix/lemur/issues/665
                 else:
-                    current_app.logger.warning('Custom OIDs not yet supported for clone operation.')
+                    current_app.logger.warning(
+                        "Custom OIDs not yet supported for clone operation."
+                    )
         except InvalidCodepoint as e:
             sentry.captureException()
-            current_app.logger.warning('Unable to parse extensions due to underscore in dns name')
+            current_app.logger.warning(
+                "Unable to parse extensions due to underscore in dns name"
+            )
         except ValueError as e:
             sentry.captureException()
-            current_app.logger.warning('Unable to parse')
+            current_app.logger.warning("Unable to parse")
             current_app.logger.exception(e)
 
         return return_extensions
@@ -355,7 +439,7 @@ class Certificate(db.Model):
         return "Certificate(name={name})".format(name=self.name)
 
 
-@event.listens_for(Certificate.destinations, 'append')
+@event.listens_for(Certificate.destinations, "append")
 def update_destinations(target, value, initiator):
     """
     Attempt to upload certificate to the new destination
@@ -367,18 +451,36 @@ def update_destinations(target, value, initiator):
     """
     destination_plugin = plugins.get(value.plugin_name)
     status = FAILURE_METRIC_STATUS
+
+    if target.expired:
+        return
     try:
-        if target.private_key:
-            destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
+        if target.private_key or not destination_plugin.requires_key:
+            destination_plugin.upload(
+                target.name,
+                target.body,
+                target.private_key,
+                target.chain,
+                value.options,
+            )
             status = SUCCESS_METRIC_STATUS
     except Exception as e:
         sentry.captureException()
+        raise
 
-    metrics.send('destination_upload', 'counter', 1,
-                 metric_tags={'status': status, 'certificate': target.name, 'destination': value.label})
+    metrics.send(
+        "destination_upload",
+        "counter",
+        1,
+        metric_tags={
+            "status": status,
+            "certificate": target.name,
+            "destination": value.label,
+        },
+    )
 
 
-@event.listens_for(Certificate.replaces, 'append')
+@event.listens_for(Certificate.replaces, "append")
 def update_replacement(target, value, initiator):
     """
     When a certificate is marked as 'replaced' we should not notify.

lemur/certificates/schemas.py

@@ -6,19 +6,24 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import current_app
-from marshmallow import fields, validate, validates_schema, post_load, pre_load
+from flask_restful import inputs
+from flask_restful.reqparse import RequestParser
+from marshmallow import fields, validate, validates_schema, post_load, pre_load, post_dump
 from marshmallow.exceptions import ValidationError
 
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
-from lemur.common import validators, missing
+from lemur.certificates import utils as cert_utils
+from lemur.common import missing, utils, validators
 from lemur.common.fields import ArrowDateTime, Hex
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
-from lemur.constants import CERTIFICATE_KEY_TYPES
+from lemur.constants import CERTIFICATE_KEY_TYPES, CRLReason
 from lemur.destinations.schemas import DestinationNestedOutputSchema
+from lemur.dns_providers.schemas import DnsProvidersNestedOutputSchema
 from lemur.domains.schemas import DomainNestedOutputSchema
 from lemur.notifications import service as notification_service
 from lemur.notifications.schemas import NotificationNestedOutputSchema
 from lemur.policies.schemas import RotationPolicyNestedOutputSchema
+from lemur.roles import service as roles_service
 from lemur.roles.schemas import RoleNestedOutputSchema
 from lemur.schemas import (
     AssociatedAuthoritySchema,
@@ -37,20 +42,26 @@ from lemur.users.schemas import UserNestedOutputSchema
 
 class CertificateSchema(LemurInputSchema):
     owner = fields.Email(required=True)
-    description = fields.String(missing='', allow_none=True)
+    description = fields.String(missing="", allow_none=True)
 
 
 class CertificateCreationSchema(CertificateSchema):
     @post_load
     def default_notification(self, data):
-        if not data['notifications']:
-            data['notifications'] += notification_service.create_default_expiration_notifications(
-                "DEFAULT_{0}".format(data['owner'].split('@')[0].upper()),
-                [data['owner']],
+        if not data["notifications"]:
+            data[
+                "notifications"
+            ] += notification_service.create_default_expiration_notifications(
+                "DEFAULT_{0}".format(data["owner"].split("@")[0].upper()),
+                [data["owner"]],
             )
-            data['notifications'] += notification_service.create_default_expiration_notifications(
-                'DEFAULT_SECURITY',
-                current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
+
+            data[
+                "notifications"
+            ] += notification_service.create_default_expiration_notifications(
+                "DEFAULT_SECURITY",
+                current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL"),
+                current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL_INTERVALS", None),
             )
         return data
 
@@ -60,40 +71,61 @@ class CertificateInputSchema(CertificateCreationSchema):
     common_name = fields.String(required=True, validate=validators.common_name)
     authority = fields.Nested(AssociatedAuthoritySchema, required=True)
 
-    validity_start = ArrowDateTime()
-    validity_end = ArrowDateTime()
-    validity_years = fields.Integer()
+    validity_start = ArrowDateTime(allow_none=True)
+    validity_end = ArrowDateTime(allow_none=True)
+    validity_years = fields.Integer(allow_none=True)
 
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
     replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
-    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
+    replacements = fields.Nested(
+        AssociatedCertificateSchema, missing=[], many=True
+    )  # deprecated
     roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
-    dns_provider = fields.Nested(AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False)
+    dns_provider = fields.Nested(
+        AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False
+    )
 
-    csr = fields.String(validate=validators.csr)
+    csr = fields.String(allow_none=True, validate=validators.csr)
 
     key_type = fields.String(
-        validate=validate.OneOf(CERTIFICATE_KEY_TYPES),
-        missing='RSA2048')
+        validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="ECCPRIME256V1"
+    )
 
     notify = fields.Boolean(default=True)
     rotation = fields.Boolean()
-    rotation_policy = fields.Nested(AssociatedRotationPolicySchema, missing={'name': 'default'}, default={'name': 'default'})
+    rotation_policy = fields.Nested(
+        AssociatedRotationPolicySchema,
+        missing={"name": "default"},
+        allow_none=True,
+        default={"name": "default"},
+    )
 
     # certificate body fields
-    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
-    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
-    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
-    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
-    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+    organizational_unit = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATIONAL_UNIT")
+    )
+    organization = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATION")
+    )
+    location = fields.String()
+    country = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_COUNTRY")
+    )
+    state = fields.String(missing=lambda: current_app.config.get("LEMUR_DEFAULT_STATE"))
 
     extensions = fields.Nested(ExtensionSchema)
 
     @validates_schema
     def validate_authority(self, data):
-        if not data['authority'].active:
-            raise ValidationError("The authority is inactive.", ['authority'])
+        if 'authority' not in data:
+            raise ValidationError("Missing Authority.")
+
+        if isinstance(data["authority"], str):
+            raise ValidationError("Authority not found.")
+
+        if not data["authority"].active:
+            raise ValidationError("The authority is inactive.", ["authority"])
 
     @validates_schema
     def validate_dates(self, data):
@@ -101,8 +133,35 @@ class CertificateInputSchema(CertificateCreationSchema):
 
     @pre_load
     def load_data(self, data):
-        if data.get('replacements'):
-            data['replaces'] = data['replacements']  # TODO remove when field is deprecated
+        if data.get("replacements"):
+            data["replaces"] = data[
+                "replacements"
+            ]  # TODO remove when field is deprecated
+        if data.get("csr"):
+            csr_sans = cert_utils.get_sans_from_csr(data["csr"])
+            if not data.get("extensions"):
+                data["extensions"] = {"subAltNames": {"names": []}}
+            elif not data["extensions"].get("subAltNames"):
+                data["extensions"]["subAltNames"] = {"names": []}
+            elif not data["extensions"]["subAltNames"].get("names"):
+                data["extensions"]["subAltNames"]["names"] = []
+
+            data["extensions"]["subAltNames"]["names"] = csr_sans
+
+            common_name = cert_utils.get_cn_from_csr(data["csr"])
+            if common_name:
+                data["common_name"] = common_name
+            key_type = cert_utils.get_key_type_from_csr(data["csr"])
+            if key_type:
+                data["key_type"] = key_type
+
+        # This code will be exercised for certificate import (without CSR)
+        if data.get("key_type") is None:
+            if data.get("body"):
+                data["key_type"] = utils.get_key_type_from_certificate(data["body"])
+            else:
+                data["key_type"] = "ECCPRIME256V1"  # default value
+
         return missing.convert_validity_years(data)
 
 
@@ -115,26 +174,63 @@ class CertificateEditInputSchema(CertificateSchema):
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
     replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
-    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
+    replacements = fields.Nested(
+        AssociatedCertificateSchema, missing=[], many=True
+    )  # deprecated
     roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
 
     @pre_load
     def load_data(self, data):
-        if data.get('replacements'):
-            data['replaces'] = data['replacements']  # TODO remove when field is deprecated
+        if data.get("replacements"):
+            data["replaces"] = data[
+                "replacements"
+            ]  # TODO remove when field is deprecated
+
+        if data.get("owner"):
+            # Check if role already exists. This avoids adding duplicate role.
+            if data.get("roles") and any(r.get("name") == data["owner"] for r in data["roles"]):
+                return data
+
+            # Add required role
+            owner_role = roles_service.get_or_create(
+                data["owner"],
+                description=f"Auto generated role based on owner: {data['owner']}"
+            )
+
+            # Put  role info in correct format using RoleNestedOutputSchema
+            owner_role_dict = RoleNestedOutputSchema().dump(owner_role).data
+            if data.get("roles"):
+                data["roles"].append(owner_role_dict)
+            else:
+                data["roles"] = [owner_role_dict]
+
         return data
 
     @post_load
     def enforce_notifications(self, data):
         """
-        Ensures that when an owner changes, default notifications are added for the new owner.
-        Old owner notifications are retained unless explicitly removed.
+        Add default notification for current owner if none exist.
+        This ensures that the default notifications are added in the event of owner change.
+        Old owner notifications are retained unless explicitly removed later in the code path.
         :param data:
         :return:
         """
-        if data['owner']:
-            notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
-            data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']])
+        if data.get("owner"):
+            notification_name = "DEFAULT_{0}".format(
+                data["owner"].split("@")[0].upper()
+            )
+
+            # Even if one default role exists, return
+            # This allows a User to remove unwanted default notification for current owner
+            if any(n.label.startswith(notification_name) for n in data["notifications"]):
+                return data
+
+            data[
+                "notifications"
+            ] += notification_service.create_default_expiration_notifications(
+                notification_name, [data["owner"]]
+            )
+
         return data
 
 
@@ -151,6 +247,7 @@ class CertificateNestedOutputSchema(LemurOutputSchema):
     bits = fields.Integer()
     body = fields.String()
     chain = fields.String()
+    csr = fields.String()
     active = fields.Boolean()
 
     rotation = fields.Boolean()
@@ -159,13 +256,13 @@ class CertificateNestedOutputSchema(LemurOutputSchema):
 
     # Note aliasing is the first step in deprecating these fields.
     cn = fields.String()  # deprecated
-    common_name = fields.String(attribute='cn')
+    common_name = fields.String(attribute="cn")
 
     not_after = fields.DateTime()  # deprecated
-    validity_end = ArrowDateTime(attribute='not_after')
+    validity_end = ArrowDateTime(attribute="not_after")
 
     not_before = fields.DateTime()  # deprecated
-    validity_start = ArrowDateTime(attribute='not_before')
+    validity_start = ArrowDateTime(attribute="not_before")
 
     issuer = fields.Nested(AuthorityNestedOutputSchema)
 
@@ -182,32 +279,39 @@ class CertificateOutputSchema(LemurOutputSchema):
     bits = fields.Integer()
     body = fields.String()
     chain = fields.String()
+    csr = fields.String()
     deleted = fields.Boolean(default=False)
     description = fields.String()
     issuer = fields.String()
     name = fields.String()
     dns_provider_id = fields.Integer(required=False, allow_none=True)
+    date_created = ArrowDateTime()
+    resolved = fields.Boolean(required=False, allow_none=True)
+    resolved_cert_id = fields.Integer(required=False, allow_none=True)
 
     rotation = fields.Boolean()
 
     # Note aliasing is the first step in deprecating these fields.
     notify = fields.Boolean()
-    active = fields.Boolean(attribute='notify')
+    active = fields.Boolean(attribute="notify")
+    has_private_key = fields.Boolean()
 
     cn = fields.String()
-    common_name = fields.String(attribute='cn')
+    common_name = fields.String(attribute="cn")
+    distinguished_name = fields.String()
 
     not_after = fields.DateTime()
-    validity_end = ArrowDateTime(attribute='not_after')
+    validity_end = ArrowDateTime(attribute="not_after")
 
     not_before = fields.DateTime()
-    validity_start = ArrowDateTime(attribute='not_before')
+    validity_start = ArrowDateTime(attribute="not_before")
 
     owner = fields.Email()
     san = fields.Boolean()
     serial = fields.String()
-    serial_hex = Hex(attribute='serial')
+    serial_hex = Hex(attribute="serial")
     signing_algorithm = fields.String()
+    key_type = fields.String(allow_none=True)
 
     status = fields.String()
     user = fields.Nested(UserNestedOutputSchema)
@@ -220,19 +324,60 @@ class CertificateOutputSchema(LemurOutputSchema):
     notifications = fields.Nested(NotificationNestedOutputSchema, many=True)
     replaces = fields.Nested(CertificateNestedOutputSchema, many=True)
     authority = fields.Nested(AuthorityNestedOutputSchema)
+    dns_provider = fields.Nested(DnsProvidersNestedOutputSchema)
     roles = fields.Nested(RoleNestedOutputSchema, many=True)
     endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
-    replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
+    replaced_by = fields.Nested(
+        CertificateNestedOutputSchema, many=True, attribute="replaced"
+    )
     rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
 
+    country = fields.String()
+    location = fields.String()
+    state = fields.String()
+    organization = fields.String()
+    organizational_unit = fields.String()
+
+    @post_dump
+    def handle_subject_details(self, data):
+        subject_details = ["country", "state", "location", "organization", "organizational_unit"]
+
+        # Remove subject details if authority is CA/Browser Forum compliant. The code will use default set of values in that case.
+        # If CA/Browser Forum compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below
+        # condition checks for 'not False' ==> 'True or None'
+        if data.get("authority"):
+            is_cab_compliant = data.get("authority").get("isCabCompliant")
+
+            if is_cab_compliant is not False:
+                for field in subject_details:
+                    data.pop(field, None)
+
+        # Removing subject fields if None, else it complains in de-serialization
+        for field in subject_details:
+            if field in data and data[field] is None:
+                data.pop(field)
+
+
+class CertificateShortOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    name = fields.String()
+    owner = fields.Email()
+    notify = fields.Boolean()
+    authority = fields.Nested(AuthorityNestedOutputSchema)
+    issuer = fields.String()
+    cn = fields.String()
+
 
 class CertificateUploadInputSchema(CertificateCreationSchema):
     name = fields.String()
+    authority = fields.Nested(AssociatedAuthoritySchema, required=False)
     notify = fields.Boolean(missing=True)
-
-    private_key = fields.String(validate=validators.private_key)
-    body = fields.String(required=True, validate=validators.public_certificate)
-    chain = fields.String(validate=validators.public_certificate, missing=None, allow_none=True)  # TODO this could be multiple certificates
+    external_id = fields.String(missing=None, allow_none=True)
+    private_key = fields.String()
+    body = fields.String(required=True)
+    chain = fields.String(missing=None, allow_none=True)
+    csr = fields.String(required=False, allow_none=True, validate=validators.csr)
+    key_type = fields.String()
 
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
@@ -241,9 +386,54 @@ class CertificateUploadInputSchema(CertificateCreationSchema):
 
     @validates_schema
     def keys(self, data):
-        if data.get('destinations'):
-            if not data.get('private_key'):
-                raise ValidationError('Destinations require private key.')
+        if data.get("destinations"):
+            if not data.get("private_key"):
+                raise ValidationError("Destinations require private key.")
+
+    @validates_schema
+    def validate_cert_private_key_chain(self, data):
+        cert = None
+        key = None
+        if data.get("body"):
+            try:
+                cert = utils.parse_certificate(data["body"])
+            except ValueError:
+                raise ValidationError(
+                    "Public certificate presented is not valid.", field_names=["body"]
+                )
+
+        if data.get("private_key"):
+            try:
+                key = utils.parse_private_key(data["private_key"])
+            except ValueError:
+                raise ValidationError(
+                    "Private key presented is not valid.", field_names=["private_key"]
+                )
+
+        if cert and key:
+            # Throws ValidationError
+            validators.verify_private_key_match(key, cert)
+
+        if data.get("chain"):
+            try:
+                chain = utils.parse_cert_chain(data["chain"])
+            except ValueError:
+                raise ValidationError(
+                    "Invalid certificate in certificate chain.", field_names=["chain"]
+                )
+
+            # Throws ValidationError
+            validators.verify_cert_chain([cert] + chain)
+
+    @pre_load
+    def load_data(self, data):
+        if data.get("body"):
+            try:
+                data["key_type"] = utils.get_key_type_from_certificate(data["body"])
+            except ValueError:
+                raise ValidationError(
+                    "Public certificate presented is not valid.", field_names=["body"]
+                )
 
 
 class CertificateExportInputSchema(LemurInputSchema):
@@ -251,23 +441,40 @@ class CertificateExportInputSchema(LemurInputSchema):
 
 
 class CertificateNotificationOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
     description = fields.String()
     issuer = fields.String()
     name = fields.String()
     owner = fields.Email()
     user = fields.Nested(UserNestedOutputSchema)
-    validity_end = ArrowDateTime(attribute='not_after')
-    replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
+    validity_end = ArrowDateTime(attribute="not_after")
+    replaced_by = fields.Nested(
+        CertificateNestedOutputSchema, many=True, attribute="replaced"
+    )
     endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
 
 
 class CertificateRevokeSchema(LemurInputSchema):
     comments = fields.String()
+    crl_reason = fields.String(validate=validate.OneOf(CRLReason.__members__), missing="unspecified")
+
+
+certificates_list_request_parser = RequestParser()
+certificates_list_request_parser.add_argument("short", type=inputs.boolean, default=False, location="args")
+
+
+def certificates_list_output_schema_factory():
+    args = certificates_list_request_parser.parse_args()
+    if args["short"]:
+        return certificates_short_output_schema
+    else:
+        return certificates_output_schema
 
 
 certificate_input_schema = CertificateInputSchema()
 certificate_output_schema = CertificateOutputSchema()
 certificates_output_schema = CertificateOutputSchema(many=True)
+certificates_short_output_schema = CertificateShortOutputSchema(many=True)
 certificate_upload_input_schema = CertificateUploadInputSchema()
 certificate_export_input_schema = CertificateExportInputSchema()
 certificate_edit_input_schema = CertificateEditInputSchema()

lemur/certificates/service.py

@@ -6,36 +6,38 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
-
-from flask import current_app
-from sqlalchemy import func, or_, not_, cast, Integer
-
+import re
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
+from flask import current_app
+from sqlalchemy import func, or_, not_, cast, Integer
+from sqlalchemy.sql.expression import false, true
 
 from lemur import database
-from lemur.extensions import metrics, signals
-from lemur.plugins.base import plugins
-from lemur.common.utils import generate_private_key, truthiness
-
-from lemur.roles.models import Role
-from lemur.domains.models import Domain
 from lemur.authorities.models import Authority
-from lemur.destinations.models import Destination
 from lemur.certificates.models import Certificate
+from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
+from lemur.common.utils import generate_private_key, truthiness
+from lemur.destinations.models import Destination
+from lemur.domains.models import Domain
+from lemur.endpoints import service as endpoint_service
+from lemur.extensions import metrics, sentry, signals
+from lemur.models import certificate_associations
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
-
-from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
-
+from lemur.plugins.base import plugins
 from lemur.roles import service as role_service
+from lemur.roles.models import Role
 
-
-csr_created = signals.signal('csr_created', "CSR generated")
-csr_imported = signals.signal('csr_imported', "CSR imported from external source")
-certificate_issued = signals.signal('certificate_issued', "Authority issued a certificate")
-certificate_imported = signals.signal('certificate_imported', "Certificate imported from external source")
+csr_created = signals.signal("csr_created", "CSR generated")
+csr_imported = signals.signal("csr_imported", "CSR imported from external source")
+certificate_issued = signals.signal(
+    "certificate_issued", "Authority issued a certificate"
+)
+certificate_imported = signals.signal(
+    "certificate_imported", "Certificate imported from external source"
+)
 
 
 def get(cert_id):
@@ -55,12 +57,12 @@ def get_by_name(name):
     :param name:
     :return:
     """
-    return database.get(Certificate, name, field='name')
+    return database.get(Certificate, name, field="name")
 
 
 def get_by_serial(serial):
     """
-    Retrieves certificate by it's Serial.
+    Retrieves certificate(s) by serial number.
     :param serial:
     :return:
     """
@@ -70,6 +72,22 @@ def get_by_serial(serial):
     return Certificate.query.filter(Certificate.serial == serial).all()
 
 
+def get_by_attributes(conditions):
+    """
+    Retrieves certificate(s) by conditions given in a hash of given key=>value pairs.
+    :param serial:
+    :return:
+    """
+    # Ensure that each of the given conditions corresponds to actual columns
+    # if not, silently remove it
+    for attr in conditions.keys():
+        if attr not in Certificate.__table__.columns:
+            conditions.pop(attr)
+
+    query = database.session_query(Certificate)
+    return database.find_all(query, Certificate, conditions).all()
+
+
 def delete(cert_id):
     """
     Delete's a certificate.
@@ -88,15 +106,95 @@ def get_all_certs():
     return Certificate.query.all()
 
 
-def get_all_pending_cleaning(source):
+def get_all_valid_certs(authority_plugin_name):
     """
-    Retrieves all certificates that are available for cleaning.
+    Retrieves all valid (not expired & not revoked) certificates within Lemur, for the given authority plugin names
+    ignored if no authority_plugin_name provided.
+
+    Note that depending on the DB size retrieving all certificates might an expensive operation
 
-    :param source:
     :return:
     """
-    return Certificate.query.filter(Certificate.sources.any(id=source.id))\
-                            .filter(not_(Certificate.endpoints.any())).all()
+    if authority_plugin_name:
+        return (
+            Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
+                Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
+                Authority.plugin_name.in_(authority_plugin_name)).filter(Certificate.revoked.is_(False)).all()
+        )
+    else:
+        return (
+            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
+                Certificate.revoked.is_(False)).all()
+        )
+
+
+def get_all_pending_cleaning_expired(source):
+    """
+    Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not
+    attached to any endpoints.
+
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.expired)
+        .all()
+    )
+
+
+def get_all_certs_attached_to_endpoint_without_autorotate():
+    """
+        Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled.
+
+        :return: list of certificates attached to an endpoint without autorotate
+        """
+    return (
+        Certificate.query.filter(Certificate.endpoints.any())
+        .filter(Certificate.rotation == false())
+        .filter(Certificate.revoked == false())
+        .filter(Certificate.not_after >= arrow.now())
+        .filter(not_(Certificate.replaced.any()))
+        .all()  # noqa
+    )
+
+
+def get_all_pending_cleaning_expiring_in_days(source, days_to_expire):
+    """
+    Retrieves all certificates that are available for cleaning, not attached to endpoint,
+    and within X days from expiration.
+
+    :param days_to_expire: defines how many days till the certificate is expired
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
+    expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD")
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.not_after < expiration_window)
+        .all()
+    )
+
+
+def get_all_pending_cleaning_issued_since_days(source, days_since_issuance):
+    """
+    Retrieves all certificates that are available for cleaning: not attached to endpoint, and X days since issuance.
+
+    :param days_since_issuance: defines how many days since the certificate is issued
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
+    not_in_use_window = (
+        arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD")
+    )
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.date_created > not_in_use_window)
+        .all()
+    )
 
 
 def get_all_pending_reissue():
@@ -109,9 +207,12 @@ def get_all_pending_reissue():
 
     :return:
     """
-    return Certificate.query.filter(Certificate.rotation == True)\
-        .filter(not_(Certificate.replaced.any()))\
-        .filter(Certificate.in_rotation_window == True).all()  # noqa
+    return (
+        Certificate.query.filter(Certificate.rotation == true())
+        .filter(not_(Certificate.replaced.any()))
+        .filter(Certificate.in_rotation_window == true())
+        .all()
+    )  # noqa
 
 
 def find_duplicates(cert):
@@ -123,10 +224,12 @@ def find_duplicates(cert):
     :param cert:
     :return:
     """
-    if cert['chain']:
-        return Certificate.query.filter_by(body=cert['body'].strip(), chain=cert['chain'].strip()).all()
+    if cert["chain"]:
+        return Certificate.query.filter_by(
+            body=cert["body"].strip(), chain=cert["chain"].strip()
+        ).all()
     else:
-        return Certificate.query.filter_by(body=cert['body'].strip(), chain=None).all()
+        return Certificate.query.filter_by(body=cert["body"].strip(), chain=None).all()
 
 
 def export(cert, export_plugin):
@@ -138,8 +241,10 @@ def export(cert, export_plugin):
     :param cert:
     :return:
     """
-    plugin = plugins.get(export_plugin['slug'])
-    return plugin.export(cert.body, cert.chain, cert.private_key, export_plugin['pluginOptions'])
+    plugin = plugins.get(export_plugin["slug"])
+    return plugin.export(
+        cert.body, cert.chain, cert.private_key, export_plugin["pluginOptions"]
+    )
 
 
 def update(cert_id, **kwargs):
@@ -156,19 +261,33 @@ def update(cert_id, **kwargs):
     return database.update(cert)
 
 
-def create_certificate_roles(**kwargs):
-    # create an role for the owner and assign it
-    owner_role = role_service.get_by_name(kwargs['owner'])
+def cleanup_owner_roles_notification(owner_name, kwargs):
+    kwargs["roles"] = [r for r in kwargs["roles"] if r.name != owner_name]
+    notification_prefix = f"DEFAULT_{owner_name.split('@')[0].upper()}"
+    kwargs["notifications"] = [n for n in kwargs["notifications"] if not n.label.startswith(notification_prefix)]
 
-    if not owner_role:
-        owner_role = role_service.create(
-            kwargs['owner'],
-            description="Auto generated role based on owner: {0}".format(kwargs['owner'])
+
+def update_notify(cert, notify_flag):
+    """
+    Toggle notification value which is a boolean
+    :param notify_flag: new notify value
+    :param cert: Certificate object to be updated
+    :return:
+    """
+    cert.notify = notify_flag
+    return database.update(cert)
+
+
+def create_certificate_roles(**kwargs):
+    # create a role for the owner and assign it
+    owner_role = role_service.get_or_create(
+        kwargs["owner"],
+        description=f"Auto generated role based on owner: {kwargs['owner']}"
     )
 
     # ensure that the authority's owner is also associated with the certificate
-    if kwargs.get('authority'):
-        authority_owner_role = role_service.get_by_name(kwargs['authority'].owner)
+    if kwargs.get("authority"):
+        authority_owner_role = role_service.get_by_name(kwargs["authority"].owner)
         return [owner_role, authority_owner_role]
 
     return [owner_role]
@@ -180,16 +299,16 @@ def mint(**kwargs):
     Support for multiple authorities is handled by individual plugins.
 
     """
-    authority = kwargs['authority']
+    authority = kwargs["authority"]
 
     issuer = plugins.get(authority.plugin_name)
 
     # allow the CSR to be specified by the user
-    if not kwargs.get('csr'):
+    if not kwargs.get("csr"):
         csr, private_key = create_csr(**kwargs)
         csr_created.send(authority=authority, csr=csr)
     else:
-        csr = str(kwargs.get('csr'))
+        csr = str(kwargs.get("csr"))
         private_key = None
         csr_imported.send(authority=authority, csr=csr)
 
@@ -210,8 +329,8 @@ def import_certificate(**kwargs):
 
     :param kwargs:
     """
-    if not kwargs.get('owner'):
-        kwargs['owner'] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0]
+    if not kwargs.get("owner"):
+        kwargs["owner"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")[0]
 
     return upload(**kwargs)
 
@@ -222,21 +341,16 @@ def upload(**kwargs):
     """
     roles = create_certificate_roles(**kwargs)
 
-    if kwargs.get('roles'):
-        kwargs['roles'] += roles
+    if kwargs.get("roles"):
+        kwargs["roles"] += roles
     else:
-        kwargs['roles'] = roles
-
-    if kwargs.get('private_key'):
-        private_key = kwargs['private_key']
-        if not isinstance(private_key, bytes):
-            kwargs['private_key'] = private_key.encode('utf-8')
+        kwargs["roles"] = roles
 
     cert = Certificate(**kwargs)
-
+    cert.authority = kwargs.get("authority")
     cert = database.create(cert)
 
-    kwargs['creator'].certificates.append(cert)
+    kwargs["creator"].certificates.append(cert)
 
     cert = database.update(cert)
     certificate_imported.send(certificate=cert, authority=cert.authority)
@@ -247,34 +361,60 @@ def create(**kwargs):
     """
     Creates a new certificate.
     """
+    try:
         cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs)
-    kwargs['body'] = cert_body
-    kwargs['private_key'] = private_key
-    kwargs['chain'] = cert_chain
-    kwargs['external_id'] = external_id
-    kwargs['csr'] = csr
+    except Exception:
+        log_data = {
+            "message": "Exception minting certificate",
+            "issuer": kwargs["authority"].name,
+            "cn": kwargs["common_name"],
+        }
+        current_app.logger.error(log_data, exc_info=True)
+        sentry.captureException()
+        raise
+    kwargs["body"] = cert_body
+    kwargs["private_key"] = private_key
+    kwargs["chain"] = cert_chain
+    kwargs["external_id"] = external_id
+    kwargs["csr"] = csr
 
     roles = create_certificate_roles(**kwargs)
 
-    if kwargs.get('roles'):
-        kwargs['roles'] += roles
+    if kwargs.get("roles"):
+        kwargs["roles"] += roles
     else:
-        kwargs['roles'] = roles
+        kwargs["roles"] = roles
 
     if cert_body:
         cert = Certificate(**kwargs)
-        kwargs['creator'].certificates.append(cert)
+        kwargs["creator"].certificates.append(cert)
     else:
+        # ACME path
         cert = PendingCertificate(**kwargs)
-        kwargs['creator'].pending_certificates.append(cert)
+        kwargs["creator"].pending_certificates.append(cert)
 
-    cert.authority = kwargs['authority']
+    cert.authority = kwargs["authority"]
 
     database.commit()
 
     if isinstance(cert, Certificate):
         certificate_issued.send(certificate=cert, authority=cert.authority)
-        metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
+        metrics.send(
+            "certificate_issued",
+            "counter",
+            1,
+            metric_tags=dict(owner=cert.owner, issuer=cert.issuer),
+        )
+
+    if isinstance(cert, PendingCertificate):
+        # We need to refresh the pending certificate to avoid "Instance is not bound to a Session; "
+        # "attribute refresh operation cannot proceed"
+        pending_cert = database.session_query(PendingCertificate).get(cert.id)
+        from lemur.common.celery import fetch_acme_cert
+
+        if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False):
+            fetch_acme_cert.apply_async((pending_cert.id,), countdown=5)
+
     return cert
 
 
@@ -287,82 +427,167 @@ def render(args):
     """
     query = database.session_query(Certificate)
 
-    time_range = args.pop('time_range')
-    destination_id = args.pop('destination_id')
-    notification_id = args.pop('notification_id', None)
-    show = args.pop('show')
+    show_expired = args.pop("showExpired")
+    if show_expired != 1:
+        one_month_old = (
+            arrow.now()
+            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))
+            .format("YYYY-MM-DD")
+        )
+        query = query.filter(Certificate.not_after > one_month_old)
+
+    time_range = args.pop("time_range")
+
+    destination_id = args.pop("destination_id")
+    notification_id = args.pop("notification_id", None)
+    show = args.pop("show")
     # owner = args.pop('owner')
     # creator = args.pop('creator')  # TODO we should enabling filtering by owner
 
-    filt = args.pop('filter')
+    filt = args.pop("filter")
 
     if filt:
-        terms = filt.split(';')
-        term = '%{0}%'.format(terms[1])
+        terms = filt.split(";")
+        term = "%{0}%".format(terms[1])
         # Exact matches for quotes. Only applies to name, issuer, and cn
         if terms[1].startswith('"') and terms[1].endswith('"'):
             term = terms[1][1:-1]
 
-        if 'issuer' in terms:
+        if "issuer" in terms:
             # we can't rely on issuer being correct in the cert directly so we combine queries
-            sub_query = database.session_query(Authority.id)\
-                .filter(Authority.name.ilike(term))\
+            sub_query = (
+                database.session_query(Authority.id)
+                .filter(Authority.name.ilike(term))
                 .subquery()
+            )
 
             query = query.filter(
                 or_(
                     Certificate.issuer.ilike(term),
-                    Certificate.authority_id.in_(sub_query)
+                    Certificate.authority_id.in_(sub_query),
                 )
             )
 
-        elif 'destination' in terms:
-            query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
-        elif 'notify' in filt:
+        elif "destination" in terms:
+            query = query.filter(
+                Certificate.destinations.any(Destination.id == terms[1])
+            )
+        elif "notify" in filt:
             query = query.filter(Certificate.notify == truthiness(terms[1]))
-        elif 'active' in filt:
+        elif "active" in filt:
             query = query.filter(Certificate.active == truthiness(terms[1]))
-        elif 'cn' in terms:
+        elif "cn" in terms:
             query = query.filter(
                 or_(
-                    Certificate.cn.ilike(term),
-                    Certificate.domains.any(Domain.name.ilike(term))
+                    func.lower(Certificate.cn).like(term.lower()),
+                    Certificate.id.in_(like_domain_query(term)),
                 )
             )
-        elif 'id' in terms:
+        elif "id" in terms:
             query = query.filter(Certificate.id == cast(terms[1], Integer))
-        elif 'name' in terms:
+        elif "name" in terms:
             query = query.filter(
                 or_(
-                    Certificate.name.ilike(term),
-                    Certificate.domains.any(Domain.name.ilike(term)),
-                    Certificate.cn.ilike(term),
+                    func.lower(Certificate.name).like(term.lower()),
+                    Certificate.id.in_(like_domain_query(term)),
+                    func.lower(Certificate.cn).like(term.lower()),
                 )
             )
+        elif "fixedName" in terms:
+            # only what matches the fixed name directly if a fixedname is provided
+            query = query.filter(Certificate.name == terms[1])
         else:
             query = database.filter(query, Certificate, terms)
 
     if show:
-        sub_query = database.session_query(Role.name).filter(Role.user_id == args['user'].id).subquery()
+        sub_query = (
+            database.session_query(Role.name)
+            .filter(Role.user_id == args["user"].id)
+            .subquery()
+        )
         query = query.filter(
             or_(
-                Certificate.user_id == args['user'].id,
-                Certificate.owner.in_(sub_query)
+                Certificate.user_id == args["user"].id, Certificate.owner.in_(sub_query)
             )
         )
 
     if destination_id:
-        query = query.filter(Certificate.destinations.any(Destination.id == destination_id))
+        query = query.filter(
+            Certificate.destinations.any(Destination.id == destination_id)
+        )
 
     if notification_id:
-        query = query.filter(Certificate.notifications.any(Notification.id == notification_id))
+        query = query.filter(
+            Certificate.notifications.any(Notification.id == notification_id)
+        )
 
     if time_range:
-        to = arrow.now().replace(weeks=+time_range).format('YYYY-MM-DD')
-        now = arrow.now().format('YYYY-MM-DD')
-        query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
+        to = arrow.now().shift(weeks=+time_range).format("YYYY-MM-DD")
+        now = arrow.now().format("YYYY-MM-DD")
+        query = query.filter(Certificate.not_after <= to).filter(
+            Certificate.not_after >= now
+        )
 
-    return database.sort_and_page(query, Certificate, args)
+    if current_app.config.get("ALLOW_CERT_DELETION", False):
+        query = query.filter(Certificate.deleted == false())
+
+    result = database.sort_and_page(query, Certificate, args)
+    return result
+
+
+def like_domain_query(term):
+    domain_query = database.session_query(Domain.id)
+    domain_query = domain_query.filter(func.lower(Domain.name).like(term.lower()))
+    assoc_query = database.session_query(certificate_associations.c.certificate_id)
+    assoc_query = assoc_query.filter(certificate_associations.c.domain_id.in_(domain_query))
+    return assoc_query
+
+
+def query_name(certificate_name, args):
+    """
+    Helper function that queries for a certificate by name
+
+    :param args:
+    :return:
+    """
+    query = database.session_query(Certificate)
+    query = query.filter(Certificate.name == certificate_name)
+    result = database.sort_and_page(query, Certificate, args)
+    return result
+
+
+def query_common_name(common_name, args):
+    """
+    Helper function that queries for not expired certificates by common name (and owner)
+
+    :param common_name:
+    :param args:
+    :return:
+    """
+    owner = args.pop("owner")
+    page = args.pop("page")
+    count = args.pop("count")
+
+    paginate = page and count
+    query = database.session_query(Certificate) if paginate else Certificate.query
+
+    # only not expired certificates
+    current_time = arrow.utcnow()
+    query = query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\
+        .filter(not_(Certificate.revoked))\
+        .filter(not_(Certificate.replaced.any()))  # ignore rotated certificates to avoid duplicates
+
+    if owner:
+        query = query.filter(Certificate.owner.ilike(owner))
+
+    if common_name != "%":
+        # if common_name is a wildcard ('%'), no need to include it in the query
+        query = query.filter(Certificate.cn.ilike(common_name))
+
+    if paginate:
+        return database.paginate(query, page, count)
+
+    return query.all()
 
 
 def create_csr(**csr_config):
@@ -372,65 +597,77 @@ def create_csr(**csr_config):
 
     :param csr_config:
     """
-    private_key = generate_private_key(csr_config.get('key_type'))
+    private_key = generate_private_key(csr_config.get("key_type"))
 
     builder = x509.CertificateSigningRequestBuilder()
-    name_list = [x509.NameAttribute(x509.OID_COMMON_NAME, csr_config['common_name'])]
-    if current_app.config.get('LEMUR_OWNER_EMAIL_IN_SUBJECT', True):
-        name_list.append(x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config['owner']))
-    if 'organization' in csr_config and csr_config['organization'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config['organization']))
-    if 'organizational_unit' in csr_config and csr_config['organizational_unit'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config['organizational_unit']))
-    if 'country' in csr_config and csr_config['country'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config['country']))
-    if 'state' in csr_config and csr_config['state'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config['state']))
-    if 'location' in csr_config and csr_config['location'].strip():
-        name_list.append(x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config['location']))
+    name_list = [x509.NameAttribute(x509.OID_COMMON_NAME, csr_config["common_name"])]
+    if current_app.config.get("LEMUR_OWNER_EMAIL_IN_SUBJECT", True):
+        name_list.append(
+            x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config["owner"])
+        )
+    if "organization" in csr_config and csr_config["organization"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config["organization"])
+        )
+    if (
+        "organizational_unit" in csr_config
+        and csr_config["organizational_unit"].strip()
+    ):
+        name_list.append(
+            x509.NameAttribute(
+                x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config["organizational_unit"]
+            )
+        )
+    if "country" in csr_config and csr_config["country"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config["country"])
+        )
+    if "state" in csr_config and csr_config["state"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config["state"])
+        )
+    if "location" in csr_config and csr_config["location"].strip():
+        name_list.append(
+            x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config["location"])
+        )
     builder = builder.subject_name(x509.Name(name_list))
 
-    extensions = csr_config.get('extensions', {})
-    critical_extensions = ['basic_constraints', 'sub_alt_names', 'key_usage']
-    noncritical_extensions = ['extended_key_usage']
+    extensions = csr_config.get("extensions", {})
+    critical_extensions = ["basic_constraints", "sub_alt_names", "key_usage"]
+    noncritical_extensions = ["extended_key_usage"]
     for k, v in extensions.items():
         if v:
             if k in critical_extensions:
-                current_app.logger.debug('Adding Critical Extension: {0} {1}'.format(k, v))
-                if k == 'sub_alt_names':
-                    if v['names']:
-                        builder = builder.add_extension(v['names'], critical=True)
+                current_app.logger.debug(
+                    "Adding Critical Extension: {0} {1}".format(k, v)
+                )
+                if k == "sub_alt_names":
+                    if v["names"]:
+                        builder = builder.add_extension(v["names"], critical=True)
                 else:
                     builder = builder.add_extension(v, critical=True)
 
             if k in noncritical_extensions:
-                current_app.logger.debug('Adding Extension: {0} {1}'.format(k, v))
+                current_app.logger.debug("Adding Extension: {0} {1}".format(k, v))
                 builder = builder.add_extension(v, critical=False)
 
-    ski = extensions.get('subject_key_identifier', {})
-    if ski.get('include_ski', False):
+    ski = extensions.get("subject_key_identifier", {})
+    if ski.get("include_ski", False):
         builder = builder.add_extension(
             x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
-            critical=False
+            critical=False,
         )
 
-    request = builder.sign(
-        private_key, hashes.SHA256(), default_backend()
-    )
+    request = builder.sign(private_key, hashes.SHA256(), default_backend())
 
     # serialize our private key and CSR
     private_key = private_key.private_bytes(
         encoding=serialization.Encoding.PEM,
         format=serialization.PrivateFormat.TraditionalOpenSSL,  # would like to use PKCS8 but AWS ELBs don't like it
-        encryption_algorithm=serialization.NoEncryption()
-    )
+        encryption_algorithm=serialization.NoEncryption(),
+    ).decode("utf-8")
 
-    if isinstance(private_key, bytes):
-        private_key = private_key.decode('utf-8')
-
-    csr = request.public_bytes(
-        encoding=serialization.Encoding.PEM
-    ).decode('utf-8')
+    csr = request.public_bytes(encoding=serialization.Encoding.PEM).decode("utf-8")
 
     return csr, private_key
 
@@ -442,16 +679,19 @@ def stats(**kwargs):
     :param kwargs:
     :return:
     """
-    if kwargs.get('metric') == 'not_after':
+    if kwargs.get("metric") == "not_after":
         start = arrow.utcnow()
-        end = start.replace(weeks=+32)
-        items = database.db.session.query(Certificate.issuer, func.count(Certificate.id))\
-            .group_by(Certificate.issuer)\
-            .filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \
-            .filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all()
+        end = start.shift(weeks=+32)
+        items = (
+            database.db.session.query(Certificate.issuer, func.count(Certificate.id))
+            .group_by(Certificate.issuer)
+            .filter(Certificate.not_after <= end.format("YYYY-MM-DD"))
+            .filter(Certificate.not_after >= start.format("YYYY-MM-DD"))
+            .all()
+        )
 
     else:
-        attr = getattr(Certificate, kwargs.get('metric'))
+        attr = getattr(Certificate, kwargs.get("metric"))
         query = database.db.session.query(attr, func.count(attr))
 
         items = query.group_by(attr).all()
@@ -462,7 +702,7 @@ def stats(**kwargs):
         keys.append(key)
         values.append(count)
 
-    return {'labels': keys, 'values': values}
+    return {"labels": keys, "values": values}
 
 
 def get_account_number(arn):
@@ -509,20 +749,24 @@ def get_certificate_primitives(certificate):
     certificate via `create`.
     """
     start, end = calculate_reissue_range(certificate.not_before, certificate.not_after)
-    data = CertificateInputSchema().load(CertificateOutputSchema().dump(certificate).data).data
+    ser = CertificateInputSchema().load(
+        CertificateOutputSchema().dump(certificate).data
+    )
+    assert not ser.errors, "Error re-serializing certificate: %s" % ser.errors
+    data = ser.data
 
     # we can't quite tell if we are using a custom name, as this is an automated process (typically)
     # we will rely on the Lemur generated name
-    data.pop('name', None)
+    data.pop("name", None)
 
     # TODO this can be removed once we migrate away from cn
-    data['cn'] = data['common_name']
+    data["cn"] = data["common_name"]
 
     # needed until we move off not_*
-    data['not_before'] = start
-    data['not_after'] = end
-    data['validity_start'] = start
-    data['validity_end'] = end
+    data["not_before"] = start
+    data["not_after"] = end
+    data["validity_start"] = start
+    data["validity_end"] = end
     return data
 
 
@@ -536,15 +780,115 @@ def reissue_certificate(certificate, replace=None, user=None):
     """
     primitives = get_certificate_primitives(certificate)
 
+    if primitives.get("csr"):
+        #  We do not want to re-use the CSR when creating a certificate because this defeats the purpose of rotation.
+        del primitives["csr"]
     if not user:
-        primitives['creator'] = certificate.user
+        primitives["creator"] = certificate.user
 
     else:
-        primitives['creator'] = user
+        primitives["creator"] = user
 
     if replace:
-        primitives['replaces'] = [certificate]
+        primitives["replaces"] = [certificate]
+
+    # Modify description to include the certificate ID being reissued and mention that this is created by Lemur
+    # as part of reissue
+    reissue_message_prefix = "Reissued by Lemur for cert ID "
+    reissue_message = re.compile(f"{reissue_message_prefix}([0-9]+)")
+    if primitives["description"]:
+        match = reissue_message.search(primitives["description"])
+        if match:
+            primitives["description"] = primitives["description"].replace(match.group(1), str(certificate.id))
+        else:
+            primitives["description"] = f"{reissue_message_prefix}{certificate.id}, {primitives['description']}"
+    else:
+        primitives["description"] = f"{reissue_message_prefix}{certificate.id}"
+
+    # Rotate the certificate to ECCPRIME256V1 if cert owner is present in the configured list
+    # This is a temporary change intending to rotate certificates to ECC, if opted in by certificate owners
+    # Unless identified a use case, this will be removed in mid-Q2 2021
+    ecc_reissue_owner_list = current_app.config.get("ROTATE_TO_ECC_OWNER_LIST", [])
+    ecc_reissue_exclude_cn_list = current_app.config.get("ECC_NON_COMPATIBLE_COMMON_NAMES", [])
+
+    if (certificate.owner in ecc_reissue_owner_list) and (certificate.cn not in ecc_reissue_exclude_cn_list):
+        primitives["key_type"] = "ECCPRIME256V1"
 
     new_cert = create(**primitives)
 
     return new_cert
+
+
+def is_attached_to_endpoint(certificate_name, endpoint_name):
+    """
+    Find if given certificate is attached to the endpoint. Both, certificate and endpoint, are identified by name.
+    This method talks to elb and finds the real time information.
+    :param certificate_name:
+    :param endpoint_name:
+    :return: True if certificate is attached to the given endpoint, False otherwise
+    """
+    endpoint = endpoint_service.get_by_name(endpoint_name)
+    attached_certificates = endpoint.source.plugin.get_endpoint_certificate_names(endpoint)
+    return certificate_name in attached_certificates
+
+
+def remove_from_destination(certificate, destination):
+    """
+    Remove the certificate from given destination if clean() is implemented
+    :param certificate:
+    :param destination:
+    :return:
+    """
+    plugin = plugins.get(destination.plugin_name)
+    if not hasattr(plugin, "clean"):
+        info_text = f"Cannot clean certificate {certificate.name}, {destination.plugin_name} plugin does not implement 'clean()'"
+        current_app.logger.warning(info_text)
+    else:
+        plugin.clean(certificate=certificate, options=destination.options)
+
+
+def revoke(certificate, reason):
+    plugin = plugins.get(certificate.authority.plugin_name)
+    plugin.revoke_certificate(certificate, reason)
+
+    # Perform cleanup after revoke
+    return cleanup_after_revoke(certificate)
+
+
+def cleanup_after_revoke(certificate):
+    """
+    Perform the needed cleanup for a revoked certificate. This includes -
+    1. Disabling notification
+    2. Disabling auto-rotation
+    3. Update certificate status to 'revoked'
+    4. Remove from AWS
+    :param certificate: Certificate object to modify and update in DB
+    :return: None
+    """
+    certificate.notify = False
+    certificate.rotation = False
+    certificate.status = 'revoked'
+
+    error_message = ""
+
+    for destination in list(certificate.destinations):
+        try:
+            remove_from_destination(certificate, destination)
+            certificate.destinations.remove(destination)
+        except Exception as e:
+            # This cleanup is the best-effort since certificate is already revoked at this point.
+            # We will capture the exception and move on to the next destination
+            sentry.captureException()
+            error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
+
+    database.update(certificate)
+    return error_message
+
+
+def get_issued_cert_count_for_authority(authority):
+    """
+    Returns the count of certs issued by the specified authority.
+
+    :return:
+    """
+    return database.db.session.query(Certificate).filter(Certificate.authority_id == authority.id).count()

lemur/certificates/utils.py

@@ -0,0 +1,85 @@
+"""
+Utils to parse certificate data.
+
+.. module: lemur.certificates.hooks
+    :platform: Unix
+    :copyright: (c) 2019 by Javier Ramos, see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Javier Ramos <javier.ramos@booking.com>
+"""
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from marshmallow.exceptions import ValidationError
+from cryptography.hazmat.primitives.asymmetric import rsa, ec
+from lemur.common.utils import get_key_type_from_ec_curve
+
+
+def get_sans_from_csr(data):
+    """
+    Fetches SubjectAlternativeNames from CSR.
+    Works with any kind of SubjectAlternativeName
+    :param data: PEM-encoded string with CSR
+    :return: List of LemurAPI-compatible subAltNames
+    """
+    sub_alt_names = []
+    try:
+        request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
+    except Exception:
+        raise ValidationError("CSR presented is not valid.")
+
+    try:
+        alt_names = request.extensions.get_extension_for_class(
+            x509.SubjectAlternativeName
+        )
+        for alt_name in alt_names.value:
+            sub_alt_names.append(
+                {"nameType": type(alt_name).__name__, "value": alt_name.value}
+            )
+    except x509.ExtensionNotFound:
+        pass
+
+    return sub_alt_names
+
+
+def get_cn_from_csr(data):
+    """
+    Fetches common name (CN) from CSR.
+    Works with any kind of SubjectAlternativeName
+    :param data: PEM-encoded string with CSR
+    :return: the common name
+    """
+    try:
+        request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
+    except Exception:
+        raise ValidationError("CSR presented is not valid.")
+
+    common_name = request.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)
+    return common_name[0].value
+
+
+def get_key_type_from_csr(data):
+    """
+    Fetches key_type from CSR.
+    Works with any kind of SubjectAlternativeName
+    :param data: PEM-encoded string with CSR
+    :return: key_type
+    """
+    try:
+        request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
+    except Exception:
+        raise ValidationError("CSR presented is not valid.")
+
+    try:
+        if isinstance(request.public_key(), rsa.RSAPublicKey):
+            return "RSA{key_size}".format(
+                key_size=request.public_key().key_size
+            )
+        elif isinstance(request.public_key(), ec.EllipticCurvePublicKey):
+            return get_key_type_from_ec_curve(request.public_key().curve.name)
+        else:
+            raise Exception("Unsupported key type")
+
+    except NotImplemented:
+        raise NotImplementedError

lemur/certificates/verify.py

@@ -7,6 +7,8 @@
 """
 import requests
 import subprocess
+from flask import current_app
+from lemur.extensions import sentry
 from requests.exceptions import ConnectionError, InvalidSchema
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -14,56 +16,88 @@ from cryptography.hazmat.backends import default_backend
 from lemur.utils import mktempfile
 from lemur.common.utils import parse_certificate
 
+crl_cache = {}
 
-def ocsp_verify(cert_path, issuer_chain_path):
+
+def ocsp_verify(cert, cert_path, issuer_chain_path):
     """
     Attempts to verify a certificate via OCSP. OCSP is a more modern version
     of CRL in that it will query the OCSP URI in order to determine if the
     certificate has been revoked
 
+    :param cert:
     :param cert_path:
     :param issuer_chain_path:
     :return bool: True if certificate is valid, False otherwise
     """
-    command = ['openssl', 'x509', '-noout', '-ocsp_uri', '-in', cert_path]
+    command = ["openssl", "x509", "-noout", "-ocsp_uri", "-in", cert_path]
     p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     url, err = p1.communicate()
 
-    p2 = subprocess.Popen(['openssl', 'ocsp', '-issuer', issuer_chain_path,
-                           '-cert', cert_path, "-url", url.strip()], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if not url:
+        current_app.logger.debug(
+            "No OCSP URL in certificate {}".format(cert.serial_number)
+        )
+        return None
+
+    p2 = subprocess.Popen(
+        [
+            "openssl",
+            "ocsp",
+            "-issuer",
+            issuer_chain_path,
+            "-cert",
+            cert_path,
+            "-url",
+            url.strip(),
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
 
     message, err = p2.communicate()
 
-    p_message = message.decode('utf-8')
+    p_message = message.decode("utf-8")
 
-    if 'error' in p_message or 'Error' in p_message:
+    if "error" in p_message or "Error" in p_message:
         raise Exception("Got error when parsing OCSP url")
 
-    elif 'revoked' in p_message:
-        return
+    elif "revoked" in p_message:
+        current_app.logger.debug(
+            "OCSP reports certificate revoked: {}".format(cert.serial_number)
+        )
+        return False
 
-    elif 'good' not in p_message:
+    elif "good" not in p_message:
         raise Exception("Did not receive a valid response")
 
     return True
 
 
-def crl_verify(cert_path):
+def crl_verify(cert, cert_path):
     """
     Attempts to verify a certificate using CRL.
 
+    :param cert:
     :param cert_path:
     :return: True if certificate is valid, False otherwise
     :raise Exception: If certificate does not have CRL
     """
-    with open(cert_path, 'rt') as c:
-        cert = parse_certificate(c.read())
-
-    distribution_points = cert.extensions.get_extension_for_oid(x509.OID_CRL_DISTRIBUTION_POINTS).value
+    try:
+        distribution_points = cert.extensions.get_extension_for_oid(
+            x509.OID_CRL_DISTRIBUTION_POINTS
+        ).value
+    except x509.ExtensionNotFound:
+        current_app.logger.debug(
+            "No CRLDP extension in certificate {}".format(cert.serial_number)
+        )
+        return None
 
     for p in distribution_points:
         point = p.full_name[0].value
 
+        if point not in crl_cache:
+            current_app.logger.debug("Retrieving CRL: {}".format(point))
             try:
                 response = requests.get(point)
 
@@ -75,20 +109,29 @@ def crl_verify(cert_path):
             except ConnectionError:
                 raise Exception("Unable to retrieve CRL: {0}".format(point))
 
-        crl = x509.load_der_x509_crl(response.content, backend=default_backend())
+            crl_cache[point] = x509.load_der_x509_crl(
+                response.content, backend=default_backend()
+            )
+        else:
+            current_app.logger.debug("CRL point is cached {}".format(point))
 
-        for r in crl:
-            if cert.serial == r.serial_number:
+        for r in crl_cache[point]:
+            if cert.serial_number == r.serial_number:
                 try:
                     reason = r.extensions.get_extension_for_class(x509.CRLReason).value
-                    # Handle "removeFromCRL" revoke reason as unrevoked; continue with the next distribution point.
-                    # Per RFC 5280 section 6.3.3 (k): https://tools.ietf.org/html/rfc5280#section-6.3.3
+                    # Handle "removeFromCRL" revoke reason as unrevoked;
+                    # continue with the next distribution point.
+                    # Per RFC 5280 section 6.3.3 (k):
+                    #  https://tools.ietf.org/html/rfc5280#section-6.3.3
                     if reason == x509.ReasonFlags.remove_from_crl:
                         break
                 except x509.ExtensionNotFound:
                     pass
 
-                return
+                current_app.logger.debug(
+                    "CRL reports certificate " "revoked: {}".format(cert.serial_number)
+                )
+                return False
 
     return True
 
@@ -101,15 +144,33 @@ def verify(cert_path, issuer_chain_path):
     :param issuer_chain_path:
     :return: True if valid, False otherwise
     """
+    with open(cert_path, "rt") as c:
+        try:
+            cert = parse_certificate(c.read())
+        except ValueError as e:
+            current_app.logger.error(e)
+            return None
+
     # OCSP is our main source of truth, in a lot of cases CRLs
     # have been deprecated and are no longer updated
+    verify_result = None
     try:
-        return ocsp_verify(cert_path, issuer_chain_path)
+        verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
     except Exception as e:
+        sentry.captureException()
+        current_app.logger.exception(e)
+
+    if verify_result is None:
         try:
-            return crl_verify(cert_path)
+            verify_result = crl_verify(cert, cert_path)
         except Exception as e:
-            raise Exception("Failed to verify")
+            sentry.captureException()
+            current_app.logger.exception(e)
+
+    if verify_result is None:
+        current_app.logger.debug("Failed to verify {}".format(cert.serial_number))
+
+    return verify_result
 
 
 def verify_string(cert_string, issuer_string):
@@ -121,10 +182,10 @@ def verify_string(cert_string, issuer_string):
     :return: True if valid, False otherwise
     """
     with mktempfile() as cert_tmp:
-        with open(cert_tmp, 'w') as f:
+        with open(cert_tmp, "w") as f:
             f.write(cert_string)
         with mktempfile() as issuer_tmp:
-            with open(issuer_tmp, 'w') as f:
+            with open(issuer_tmp, "w") as f:
                 f.write(issuer_string)
             status = verify(cert_tmp, issuer_tmp)
     return status

lemur/certificates/views.py

@@ -8,8 +8,8 @@
 import base64
 from builtins import str
 
-from flask import Blueprint, make_response, jsonify, g
-from flask_restful import reqparse, Api
+from flask import Blueprint, make_response, jsonify, g, current_app
+from flask_restful import reqparse, Api, inputs
 
 from lemur.common.schema import validate_schema
 from lemur.common.utils import paginated_parser
@@ -19,35 +19,141 @@ from lemur.auth.permissions import AuthorityPermission, CertificatePermission
 
 from lemur.certificates import service
 from lemur.certificates.models import Certificate
-from lemur.plugins.base import plugins
+from lemur.extensions import sentry
 from lemur.certificates.schemas import (
     certificate_input_schema,
     certificate_output_schema,
     certificate_upload_input_schema,
     certificates_output_schema,
     certificate_export_input_schema,
-    certificate_edit_input_schema
+    certificate_edit_input_schema,
+    certificates_list_output_schema_factory,
+    certificate_revoke_schema,
 )
 
 from lemur.roles import service as role_service
 from lemur.logs import service as log_service
 
 
-mod = Blueprint('certificates', __name__)
+mod = Blueprint("certificates", __name__)
 api = Api(mod)
 
 
-class CertificatesList(AuthenticatedResource):
-    """ Defines the 'certificates' endpoint """
+class CertificatesListValid(AuthenticatedResource):
+    """ Defines the 'certificates/valid' endpoint """
 
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
-        super(CertificatesList, self).__init__()
+        super(CertificatesListValid, self).__init__()
 
     @validate_schema(None, certificates_output_schema)
     def get(self):
         """
-        .. http:get:: /certificates
+        .. http:get:: /certificates/valid/<query>
+
+           The current list of not-expired certificates for a given common name, and owner. The API offers
+           optional pagination. One can send page number(>=1) and desired count per page. The returned data
+           contains total number of certificates which can help in determining the last page. Pagination
+           will not be offered if page or count info is not sent or if it is zero.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com&page=1&count=20 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response (with single cert to be concise)**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [{
+                    "status": null,
+                    "cn": "*.test.example.net",
+                    "chain": "",
+                    "csr": "-----BEGIN CERTIFICATE REQUEST-----"
+                    "authority": {
+                        "active": true,
+                        "owner": "secure@example.com",
+                        "id": 1,
+                        "description": "verisign test authority",
+                        "name": "verisign"
+                    },
+                    "owner": "joe@example.com",
+                    "serial": "82311058732025924142789179368889309156",
+                    "id": 2288,
+                    "issuer": "SymantecCorporation",
+                    "dateCreated": "2016-06-03T06:09:42.133769+00:00",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }],
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "replaced": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
+                "total": 1
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        # using non-paginated parser to ensure backward compatibility
+        self.reqparse.add_argument("filter", type=str, location="args")
+        self.reqparse.add_argument("owner", type=str, location="args")
+        self.reqparse.add_argument("count", type=int, location="args")
+        self.reqparse.add_argument("page", type=int, location="args")
+
+        args = self.reqparse.parse_args()
+        args["user"] = g.user
+        common_name = args.pop("filter").split(";")[1]
+        return service.query_common_name(common_name, args)
+
+
+class CertificatesNameQuery(AuthenticatedResource):
+    """ Defines the 'certificates/name' endpoint """
+
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificatesNameQuery, self).__init__()
+
+    @validate_schema(None, certificates_output_schema)
+    def get(self, certificate_name):
+        """
+        .. http:get:: /certificates/name/<query>
 
            The current list of certificates
 
@@ -55,7 +161,7 @@ class CertificatesList(AuthenticatedResource):
 
            .. sourcecode:: http
 
-              GET /certificates HTTP/1.1
+              GET /certificates/name/WILDCARD.test.example.net-SymantecCorporation-20160603-20180112 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
 
@@ -72,6 +178,7 @@ class CertificatesList(AuthenticatedResource):
                     "status": null,
                     "cn": "*.test.example.net",
                     "chain": "",
+                    "csr": "-----BEGIN CERTIFICATE REQUEST-----"
                     "authority": {
                         "active": true,
                         "owner": "secure@example.com",
@@ -83,6 +190,7 @@ class CertificatesList(AuthenticatedResource):
                     "serial": "82311058732025924142789179368889309156",
                     "id": 2288,
                     "issuer": "SymantecCorporation",
+                    "dateCreated": "2016-06-03T06:09:42.133769+00:00",
                     "notBefore": "2016-06-03T00:00:00+00:00",
                     "notAfter": "2018-01-12T23:59:59+00:00",
                     "destinations": [],
@@ -130,16 +238,129 @@ class CertificatesList(AuthenticatedResource):
 
         """
         parser = paginated_parser.copy()
-        parser.add_argument('timeRange', type=int, dest='time_range', location='args')
-        parser.add_argument('owner', type=bool, location='args')
-        parser.add_argument('id', type=str, location='args')
-        parser.add_argument('active', type=bool, location='args')
-        parser.add_argument('destinationId', type=int, dest="destination_id", location='args')
-        parser.add_argument('creator', type=str, location='args')
-        parser.add_argument('show', type=str, location='args')
+        parser.add_argument("timeRange", type=int, dest="time_range", location="args")
+        parser.add_argument("owner", type=inputs.boolean, location="args")
+        parser.add_argument("id", type=str, location="args")
+        parser.add_argument("active", type=inputs.boolean, location="args")
+        parser.add_argument(
+            "destinationId", type=int, dest="destination_id", location="args"
+        )
+        parser.add_argument("creator", type=str, location="args")
+        parser.add_argument("show", type=str, location="args")
 
         args = parser.parse_args()
-        args['user'] = g.user
+        args["user"] = g.user
+        return service.query_name(certificate_name, args)
+
+
+class CertificatesList(AuthenticatedResource):
+    """ Defines the 'certificates' endpoint """
+
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificatesList, self).__init__()
+
+    @validate_schema(None, certificates_list_output_schema_factory)
+    def get(self):
+        """
+        .. http:get:: /certificates
+
+           The current list of certificates
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /certificates HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [{
+                    "status": null,
+                    "cn": "*.test.example.net",
+                    "chain": "",
+                    "csr": "-----BEGIN CERTIFICATE REQUEST-----"
+                    "authority": {
+                        "active": true,
+                        "owner": "secure@example.com",
+                        "id": 1,
+                        "description": "verisign test authority",
+                        "name": "verisign"
+                    },
+                    "owner": "joe@example.com",
+                    "serial": "82311058732025924142789179368889309156",
+                    "id": 2288,
+                    "issuer": "SymantecCorporation",
+                    "dateCreated": "2016-06-03T06:09:42.133769+00:00",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }],
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "replaced": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
+                "total": 1
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int. default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number. default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        parser = paginated_parser.copy()
+        parser.add_argument("timeRange", type=int, dest="time_range", location="args")
+        parser.add_argument("owner", type=inputs.boolean, location="args")
+        parser.add_argument("id", type=str, location="args")
+        parser.add_argument("active", type=inputs.boolean, location="args")
+        parser.add_argument(
+            "destinationId", type=int, dest="destination_id", location="args"
+        )
+        parser.add_argument("creator", type=str, location="args")
+        parser.add_argument("show", type=str, location="args")
+        parser.add_argument("showExpired", type=int, location="args")
+
+        args = parser.parse_args()
+        args["user"] = g.user
         return service.render(args)
 
     @validate_schema(certificate_input_schema, certificate_output_schema)
@@ -156,6 +377,7 @@ class CertificatesList(AuthenticatedResource):
               POST /certificates HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                   "owner": "secure@example.net",
@@ -214,6 +436,7 @@ class CertificatesList(AuthenticatedResource):
                 "serial": "82311058732025924142789179368889309156",
                 "id": 2288,
                 "issuer": "SymantecCorporation",
+                "dateCreated": "2016-06-03T06:09:42.133769+00:00",
                 "notBefore": "2016-06-03T00:00:00+00:00",
                 "notAfter": "2018-01-12T23:59:59+00:00",
                 "destinations": [],
@@ -256,24 +479,31 @@ class CertificatesList(AuthenticatedResource):
            :statuscode 403: unauthenticated
 
         """
-        role = role_service.get_by_name(data['authority'].owner)
+        role = role_service.get_by_name(data["authority"].owner)
 
         # all the authority role members should be allowed
-        roles = [x.name for x in data['authority'].roles]
+        roles = [x.name for x in data["authority"].roles]
 
         # allow "owner" roles by team DL
         roles.append(role)
-        authority_permission = AuthorityPermission(data['authority'].id, roles)
+        authority_permission = AuthorityPermission(data["authority"].id, roles)
 
         if authority_permission.can():
-            data['creator'] = g.user
+            data["creator"] = g.user
             cert = service.create(**data)
             if isinstance(cert, Certificate):
                 # only log if created, not pending
-                log_service.create(g.user, 'create_cert', certificate=cert)
+                log_service.create(g.user, "create_cert", certificate=cert)
             return cert
 
-        return dict(message="You are not authorized to use the authority: {0}".format(data['authority'].name)), 403
+        return (
+            dict(
+                message="You are not authorized to use the authority: {0}".format(
+                    data["authority"].name
+                )
+            ),
+            403,
+        )
 
 
 class CertificatesUpload(AuthenticatedResource):
@@ -297,12 +527,14 @@ class CertificatesUpload(AuthenticatedResource):
               POST /certificates/upload HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "owner": "joe@example.com",
                  "body": "-----BEGIN CERTIFICATE-----...",
                  "chain": "-----BEGIN CERTIFICATE-----...",
                  "privateKey": "-----BEGIN RSA PRIVATE KEY-----..."
+                 "csr": "-----BEGIN CERTIFICATE REQUEST-----..."
                  "destinations": [],
                  "notifications": [],
                  "replacements": [],
@@ -334,6 +566,7 @@ class CertificatesUpload(AuthenticatedResource):
                 "serial": "82311058732025924142789179368889309156",
                 "id": 2288,
                 "issuer": "SymantecCorporation",
+                "dateCreated": "2016-06-03T06:09:42.133769+00:00",
                 "notBefore": "2016-06-03T00:00:00+00:00",
                 "notAfter": "2018-01-12T23:59:59+00:00",
                 "destinations": [],
@@ -374,12 +607,14 @@ class CertificatesUpload(AuthenticatedResource):
            :statuscode 200: no error
 
         """
-        data['creator'] = g.user
-        if data.get('destinations'):
-            if data.get('private_key'):
+        data["creator"] = g.user
+        if data.get("destinations"):
+            if data.get("private_key"):
                 return service.upload(**data)
             else:
-                raise Exception("Private key must be provided in order to upload certificate to AWS")
+                raise Exception(
+                    "Private key must be provided in order to upload certificate to AWS"
+                )
         return service.upload(**data)
 
 
@@ -391,10 +626,12 @@ class CertificatesStats(AuthenticatedResource):
         super(CertificatesStats, self).__init__()
 
     def get(self):
-        self.reqparse.add_argument('metric', type=str, location='args')
-        self.reqparse.add_argument('range', default=32, type=int, location='args')
-        self.reqparse.add_argument('destinationId', dest='destination_id', location='args')
-        self.reqparse.add_argument('active', type=str, default='true', location='args')
+        self.reqparse.add_argument("metric", type=str, location="args")
+        self.reqparse.add_argument("range", default=32, type=int, location="args")
+        self.reqparse.add_argument(
+            "destinationId", dest="destination_id", location="args"
+        )
+        self.reqparse.add_argument("active", type=str, default="true", location="args")
 
         args = self.reqparse.parse_args()
 
@@ -446,12 +683,12 @@ class CertificatePrivateKey(AuthenticatedResource):
             permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
 
             if not permission.can():
-                return dict(message='You are not authorized to view this key'), 403
+                return dict(message="You are not authorized to view this key"), 403
 
-        log_service.create(g.current_user, 'key_view', certificate=cert)
+        log_service.create(g.current_user, "key_view", certificate=cert)
         response = make_response(jsonify(key=cert.private_key), 200)
-        response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store'
-        response.headers['pragma'] = 'no-cache'
+        response.headers["cache-control"] = "private, max-age=0, no-cache, no-store"
+        response.headers["pragma"] = "no-cache"
         return response
 
 
@@ -487,6 +724,7 @@ class Certificates(AuthenticatedResource):
                 "status": null,
                 "cn": "*.test.example.net",
                 "chain": "",
+                "csr": "-----BEGIN CERTIFICATE REQUEST-----"
                 "authority": {
                     "active": true,
                     "owner": "secure@example.com",
@@ -498,6 +736,7 @@ class Certificates(AuthenticatedResource):
                 "serial": "82311058732025924142789179368889309156",
                 "id": 2288,
                 "issuer": "SymantecCorporation",
+                "dateCreated": "2016-06-03T06:09:42.133769+00:00",
                 "notBefore": "2016-06-03T00:00:00+00:00",
                 "notAfter": "2018-01-12T23:59:59+00:00",
                 "destinations": [],
@@ -555,6 +794,7 @@ class Certificates(AuthenticatedResource):
               PUT /certificates/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "owner": "jimbob@example.com",
@@ -587,6 +827,7 @@ class Certificates(AuthenticatedResource):
                 "serial": "82311058732025924142789179368889309156",
                 "id": 2288,
                 "issuer": "SymantecCorporation",
+                "dateCreated": "2016-06-03T06:09:42.133769+00:00",
                 "notBefore": "2016-06-03T00:00:00+00:00",
                 "notAfter": "2018-01-12T23:59:59+00:00",
                 "destinations": [],
@@ -638,21 +879,204 @@ class Certificates(AuthenticatedResource):
             permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
 
             if not permission.can():
-                return dict(message='You are not authorized to update this certificate'), 403
+                return (
+                    dict(message="You are not authorized to update this certificate"),
+                    403,
+                )
 
-        for destination in data['destinations']:
+        for destination in data["destinations"]:
             if destination.plugin.requires_key:
                 if not cert.private_key:
-                    return dict(
-                        message='Unable to add destination: {0}. Certificate does not have required private key.'.format(
+                    return (
+                        dict(
+                            message="Unable to add destination: {0}. Certificate does not have required private key.".format(
                                 destination.label
                             )
-                    ), 400
+                        ),
+                        400,
+                    )
 
+        # if owner is changed, remove all notifications and roles associated with old owner
+        if cert.owner != data["owner"]:
+            service.cleanup_owner_roles_notification(cert.owner, data)
+
+        error_message = ""
+        # if destination is removed, cleanup the certificate from AWS
+        for destination in cert.destinations:
+            if destination not in data["destinations"]:
+                try:
+                    service.remove_from_destination(cert, destination)
+                except Exception as e:
+                    sentry.captureException()
+                    # Add the removed destination back
+                    data["destinations"].append(destination)
+                    error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
+
+        # go ahead with DB update
         cert = service.update(certificate_id, **data)
-        log_service.create(g.current_user, 'update_cert', certificate=cert)
+        log_service.create(g.current_user, "update_cert", certificate=cert)
+
+        if error_message:
+            return dict(message=f"Edit Successful except -\n\n {error_message}"), 400
         return cert
 
+    @validate_schema(certificate_edit_input_schema, certificate_output_schema)
+    def post(self, certificate_id, data=None):
+        """
+        .. http:post:: /certificates/1/update/notify
+
+           Update certificate notification
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              POST /certificates/1/update/notify HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                 "notify": false
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "status": null,
+                "cn": "*.test.example.net",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "dateCreated": "2016-06-03T06:09:42.133769+00:00",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
+                "bits": 2048,
+                "body": "-----BEGIN CERTIFICATE-----...",
+                "description": null,
+                "deleted": null,
+                "notify": false,
+                "notifications": [{
+                    "id": 1
+                }]
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "rotation": true,
+                "rotationPolicy": {"name": "default"},
+                "san": null
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        cert = service.get(certificate_id)
+
+        if not cert:
+            return dict(message="Cannot find specified certificate"), 404
+
+        # allow creators
+        if g.current_user != cert.user:
+            owner_role = role_service.get_by_name(cert.owner)
+            permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+
+            if not permission.can():
+                return (
+                    dict(message="You are not authorized to update this certificate"),
+                    403,
+                )
+
+        cert = service.update_notify(cert, data.get("notify"))
+        log_service.create(g.current_user, "update_cert", certificate=cert)
+        return cert
+
+    def delete(self, certificate_id, data=None):
+        """
+        .. http:delete:: /certificates/1
+
+           Delete a certificate
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              DELETE /certificates/1 HTTP/1.1
+              Host: example.com
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 204 OK
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 204: no error
+           :statuscode 403: unauthenticated
+           :statuscode 404: certificate not found
+           :statuscode 405: certificate deletion is disabled
+
+        """
+        if not current_app.config.get("ALLOW_CERT_DELETION", False):
+            return dict(message="Certificate deletion is disabled"), 405
+
+        cert = service.get(certificate_id)
+
+        if not cert:
+            return dict(message="Cannot find specified certificate"), 404
+
+        if cert.deleted:
+            return dict(message="Certificate is already deleted"), 412
+
+        # allow creators
+        if g.current_user != cert.user:
+            owner_role = role_service.get_by_name(cert.owner)
+            permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+
+            if not permission.can():
+                return (
+                    dict(message="You are not authorized to delete this certificate"),
+                    403,
+                )
+
+        service.update(certificate_id, deleted=True)
+        log_service.create(g.current_user, "delete_cert", certificate=cert)
+        return "Certificate deleted", 204
+
 
 class NotificationCertificatesList(AuthenticatedResource):
     """ Defines the 'certificates' endpoint """
@@ -689,6 +1113,7 @@ class NotificationCertificatesList(AuthenticatedResource):
                     "status": null,
                     "cn": "*.test.example.net",
                     "chain": "",
+                    "csr": "-----BEGIN CERTIFICATE REQUEST-----"
                     "authority": {
                         "active": true,
                         "owner": "secure@example.com",
@@ -700,6 +1125,7 @@ class NotificationCertificatesList(AuthenticatedResource):
                     "serial": "82311058732025924142789179368889309156",
                     "id": 2288,
                     "issuer": "SymantecCorporation",
+                    "dateCreated": "2016-06-03T06:09:42.133769+00:00",
                     "notBefore": "2016-06-03T00:00:00+00:00",
                     "notAfter": "2018-01-12T23:59:59+00:00",
                     "destinations": [],
@@ -749,17 +1175,20 @@ class NotificationCertificatesList(AuthenticatedResource):
 
         """
         parser = paginated_parser.copy()
-        parser.add_argument('timeRange', type=int, dest='time_range', location='args')
-        parser.add_argument('owner', type=bool, location='args')
-        parser.add_argument('id', type=str, location='args')
-        parser.add_argument('active', type=bool, location='args')
-        parser.add_argument('destinationId', type=int, dest="destination_id", location='args')
-        parser.add_argument('creator', type=str, location='args')
-        parser.add_argument('show', type=str, location='args')
+        parser.add_argument("timeRange", type=int, dest="time_range", location="args")
+        parser.add_argument("owner", type=inputs.boolean, location="args")
+        parser.add_argument("id", type=str, location="args")
+        parser.add_argument("active", type=inputs.boolean, location="args")
+        parser.add_argument(
+            "destinationId", type=int, dest="destination_id", location="args"
+        )
+        parser.add_argument("creator", type=str, location="args")
+        parser.add_argument("show", type=str, location="args")
+        parser.add_argument("showExpired", type=int, location="args")
 
         args = parser.parse_args()
-        args['notification_id'] = notification_id
-        args['user'] = g.current_user
+        args["notification_id"] = notification_id
+        args["user"] = g.current_user
         return service.render(args)
 
 
@@ -796,6 +1225,7 @@ class CertificatesReplacementsList(AuthenticatedResource):
                     "status": null,
                     "cn": "*.test.example.net",
                     "chain": "",
+                    "csr": "-----BEGIN CERTIFICATE REQUEST-----",
                     "authority": {
                         "active": true,
                         "owner": "secure@example.com",
@@ -807,6 +1237,7 @@ class CertificatesReplacementsList(AuthenticatedResource):
                     "serial": "82311058732025924142789179368889309156",
                     "id": 2288,
                     "issuer": "SymantecCorporation",
+                    "dateCreated": "2016-06-03T06:09:42.133769+00:00",
                     "notBefore": "2016-06-03T00:00:00+00:00",
                     "notAfter": "2018-01-12T23:59:59+00:00",
                     "destinations": [],
@@ -872,6 +1303,7 @@ class CertificateExport(AuthenticatedResource):
               PUT /certificates/1/export HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "export": {
@@ -930,30 +1362,48 @@ class CertificateExport(AuthenticatedResource):
         if not cert:
             return dict(message="Cannot find specified certificate"), 404
 
-        plugin = data['plugin']['plugin_object']
+        plugin = data["plugin"]["plugin_object"]
 
         if plugin.requires_key:
             if not cert.private_key:
-                return dict(
-                    message='Unable to export certificate, plugin: {0} requires a private key but no key was found.'.format(
-                        plugin.slug)), 400
+                return (
+                    dict(
+                        message="Unable to export certificate, plugin: {0} requires a private key but no key was found.".format(
+                            plugin.slug
+                        )
+                    ),
+                    400,
+                )
 
             else:
                 # allow creators
                 if g.current_user != cert.user:
                     owner_role = role_service.get_by_name(cert.owner)
-                    permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+                    permission = CertificatePermission(
+                        owner_role, [x.name for x in cert.roles]
+                    )
 
                     if not permission.can():
-                        return dict(message='You are not authorized to export this certificate.'), 403
+                        return (
+                            dict(
+                                message="You are not authorized to export this certificate."
+                            ),
+                            403,
+                        )
 
-        options = data['plugin']['plugin_options']
+        options = data["plugin"]["plugin_options"]
 
-        log_service.create(g.current_user, 'key_view', certificate=cert)
-        extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
+        log_service.create(g.current_user, "key_view", certificate=cert)
+        extension, passphrase, data = plugin.export(
+            cert.body, cert.chain, cert.private_key, options
+        )
 
         # we take a hit in message size when b64 encoding
-        return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data).decode('utf-8'))
+        return dict(
+            extension=extension,
+            passphrase=passphrase,
+            data=base64.b64encode(data).decode("utf-8"),
+        )
 
 
 class CertificateRevoke(AuthenticatedResource):
@@ -961,7 +1411,7 @@ class CertificateRevoke(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(CertificateRevoke, self).__init__()
 
-    @validate_schema(None, None)
+    @validate_schema(certificate_revoke_schema, None)
     def put(self, certificate_id, data=None):
         """
         .. http:put:: /certificates/1/revoke
@@ -975,6 +1425,12 @@ class CertificateRevoke(AuthenticatedResource):
               POST /certificates/1/revoke HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                "crlReason": "affiliationChanged",
+                "comments": "Additional details if any"
+              }
 
            **Example response**:
 
@@ -985,12 +1441,13 @@ class CertificateRevoke(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                'id': 1
+                "id": 1
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
-           :statuscode 403: unauthenticated
+           :statuscode 403: unauthenticated or cert attached to LB
+           :statuscode 400: encountered error, more details in error message
 
         """
         cert = service.get(certificate_id)
@@ -1004,28 +1461,79 @@ class CertificateRevoke(AuthenticatedResource):
             permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
 
             if not permission.can():
-                return dict(message='You are not authorized to revoke this certificate.'), 403
+                return (
+                    dict(message="You are not authorized to revoke this certificate."),
+                    403,
+                )
 
         if not cert.external_id:
-            return dict(message='Cannot revoke certificate. No external id found.'), 400
+            return dict(message="Cannot revoke certificate. No external id found."), 400
 
         if cert.endpoints:
-            return dict(message='Cannot revoke certificate. Endpoints are deployed with the given certificate.'), 403
+            for endpoint in cert.endpoints:
+                if service.is_attached_to_endpoint(cert.name, endpoint.name):
+                    return (
+                        dict(
+                            message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
+                        ),
+                        403,
+                    )
 
-        plugin = plugins.get(cert.authority.plugin_name)
-        plugin.revoke_certificate(cert, data)
-        log_service.create(g.current_user, 'revoke_cert', certificate=cert)
+        try:
+            error_message = service.revoke(cert, data)
+            log_service.create(g.current_user, "revoke_cert", certificate=cert)
+
+            if error_message:
+                return dict(message=f"Certificate (id:{cert.id}) is revoked - {error_message}"), 400
             return dict(id=cert.id)
+        except NotImplementedError as ne:
+            return dict(message="Revoke is not implemented for issuer of this certificate"), 400
+        except Exception as e:
+            sentry.captureException()
+            return dict(message=f"Failed to revoke: {str(e)}"), 400
 
 
-api.add_resource(CertificateRevoke, '/certificates/<int:certificate_id>/revoke', endpoint='revokeCertificate')
-api.add_resource(CertificatesList, '/certificates', endpoint='certificates')
-api.add_resource(Certificates, '/certificates/<int:certificate_id>', endpoint='certificate')
-api.add_resource(CertificatesStats, '/certificates/stats', endpoint='certificateStats')
-api.add_resource(CertificatesUpload, '/certificates/upload', endpoint='certificateUpload')
-api.add_resource(CertificatePrivateKey, '/certificates/<int:certificate_id>/key', endpoint='privateKeyCertificates')
-api.add_resource(CertificateExport, '/certificates/<int:certificate_id>/export', endpoint='exportCertificate')
-api.add_resource(NotificationCertificatesList, '/notifications/<int:notification_id>/certificates',
-                 endpoint='notificationCertificates')
-api.add_resource(CertificatesReplacementsList, '/certificates/<int:certificate_id>/replacements',
-                 endpoint='replacements')
+api.add_resource(
+    CertificateRevoke,
+    "/certificates/<int:certificate_id>/revoke",
+    endpoint="revokeCertificate",
+)
+api.add_resource(
+    CertificatesNameQuery,
+    "/certificates/name/<string:certificate_name>",
+    endpoint="certificatesNameQuery",
+)
+api.add_resource(CertificatesList, "/certificates", endpoint="certificates")
+api.add_resource(
+    CertificatesListValid, "/certificates/valid", endpoint="certificatesListValid"
+)
+api.add_resource(
+    Certificates, "/certificates/<int:certificate_id>", endpoint="certificate"
+)
+api.add_resource(
+    Certificates, "/certificates/<int:certificate_id>/update/notify", endpoint="certificateUpdateNotify"
+)
+api.add_resource(CertificatesStats, "/certificates/stats", endpoint="certificateStats")
+api.add_resource(
+    CertificatesUpload, "/certificates/upload", endpoint="certificateUpload"
+)
+api.add_resource(
+    CertificatePrivateKey,
+    "/certificates/<int:certificate_id>/key",
+    endpoint="privateKeyCertificates",
+)
+api.add_resource(
+    CertificateExport,
+    "/certificates/<int:certificate_id>/export",
+    endpoint="exportCertificate",
+)
+api.add_resource(
+    NotificationCertificatesList,
+    "/notifications/<int:notification_id>/certificates",
+    endpoint="notificationCertificates",
+)
+api.add_resource(
+    CertificatesReplacementsList,
+    "/certificates/<int:certificate_id>/replacements",
+    endpoint="replacements",
+)

lemur/common/celery.py

@@ -0,0 +1,955 @@
+"""
+This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
+when invoked.
+
+When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
+beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
+command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
+
+"""
+import copy
+import sys
+import time
+from celery import Celery
+from celery.app.task import Context
+from celery.exceptions import SoftTimeLimitExceeded
+from celery.signals import task_failure, task_received, task_revoked, task_success
+from datetime import datetime, timezone, timedelta
+from flask import current_app
+
+from lemur.authorities.service import get as get_authority
+from lemur.certificates import cli as cli_certificate
+from lemur.common.redis import RedisHandler
+from lemur.constants import ACME_ADDITIONAL_ATTEMPTS
+from lemur.destinations import service as destinations_service
+from lemur.dns_providers import cli as cli_dns_providers
+from lemur.endpoints import cli as cli_endpoints
+from lemur.extensions import metrics, sentry
+from lemur.factory import create_app
+from lemur.notifications import cli as cli_notification
+from lemur.notifications.messaging import send_pending_failure_notification
+from lemur.pending_certificates import service as pending_certificate_service
+from lemur.plugins.base import plugins
+from lemur.sources.cli import clean, sync, validate_sources
+from lemur.sources.service import add_aws_destination_to_sources
+
+if current_app:
+    flask_app = current_app
+else:
+    flask_app = create_app()
+
+red = RedisHandler().redis()
+
+
+def make_celery(app):
+    celery = Celery(
+        app.import_name,
+        backend=app.config.get("CELERY_RESULT_BACKEND"),
+        broker=app.config.get("CELERY_BROKER_URL"),
+    )
+    celery.conf.update(app.config)
+    TaskBase = celery.Task
+
+    class ContextTask(TaskBase):
+        abstract = True
+
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return TaskBase.__call__(self, *args, **kwargs)
+
+    celery.Task = ContextTask
+    return celery
+
+
+celery = make_celery(flask_app)
+
+
+def is_task_active(fun, task_id, args):
+    from celery.task.control import inspect
+
+    if not args:
+        args = "()"  # empty args
+
+    i = inspect()
+    active_tasks = i.active()
+    for _, tasks in active_tasks.items():
+        for task in tasks:
+            if task.get("id") == task_id:
+                continue
+            if task.get("name") == fun and task.get("args") == str(args):
+                return True
+    return False
+
+
+def get_celery_request_tags(**kwargs):
+    request = kwargs.get("request")
+    sender_hostname = "unknown"
+    sender = kwargs.get("sender")
+    if sender:
+        try:
+            sender_hostname = sender.hostname
+        except AttributeError:
+            sender_hostname = vars(sender.request).get("origin", "unknown")
+    if request and not isinstance(
+        request, Context
+    ):  # unlike others, task_revoked sends a Context for `request`
+        task_name = request.name
+        task_id = request.id
+        receiver_hostname = request.hostname
+    else:
+        task_name = sender.name
+        task_id = sender.request.id
+        receiver_hostname = sender.request.hostname
+
+    tags = {
+        "task_name": task_name,
+        "task_id": task_id,
+        "sender_hostname": sender_hostname,
+        "receiver_hostname": receiver_hostname,
+    }
+    if kwargs.get("exception"):
+        tags["error"] = repr(kwargs["exception"])
+    return tags
+
+
+@celery.task()
+def report_celery_last_success_metrics():
+    """
+    For each celery task, this will determine the number of seconds since it has last been successful.
+
+    Celery tasks should be emitting redis stats with a deterministic key (In our case, `f"{task}.last_success"`.
+    report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful.
+    Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted
+    from this function.
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "recurrent task",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_time = int(time.time())
+    schedule = current_app.config.get("CELERYBEAT_SCHEDULE")
+    for _, t in schedule.items():
+        task = t.get("task")
+        last_success = int(red.get(f"{task}.last_success") or 0)
+        metrics.send(
+            f"{task}.time_since_last_success", "gauge", current_time - last_success
+        )
+    red.set(
+        f"{function}.last_success", int(time.time())
+    )  # Alert if this metric is not seen
+    metrics.send(f"{function}.success", "counter", 1)
+
+
+@task_received.connect
+def report_number_pending_tasks(**kwargs):
+    """
+    Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used
+    for autoscaling workers.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received
+    """
+    with flask_app.app_context():
+        metrics.send(
+            "celery.new_pending_task",
+            "TIMER",
+            1,
+            metric_tags=get_celery_request_tags(**kwargs),
+        )
+
+
+@task_success.connect
+def report_successful_task(**kwargs):
+    """
+    Report a generic success metric as tasks to our metrics broker every time a task finished correctly.
+    This metric can be used for autoscaling workers.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success
+    """
+    with flask_app.app_context():
+        tags = get_celery_request_tags(**kwargs)
+        red.set(f"{tags['task_name']}.last_success", int(time.time()))
+        metrics.send("celery.successful_task", "TIMER", 1, metric_tags=tags)
+
+
+@task_failure.connect
+def report_failed_task(**kwargs):
+    """
+    Report a generic failure metric as tasks to our metrics broker every time a task fails.
+    This metric can be used for alerting.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure
+    """
+    with flask_app.app_context():
+        log_data = {
+            "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+            "Message": "Celery Task Failure",
+        }
+
+        # Add traceback if exception info is in the kwargs
+        einfo = kwargs.get("einfo")
+        if einfo:
+            log_data["traceback"] = einfo.traceback
+
+        error_tags = get_celery_request_tags(**kwargs)
+
+        log_data.update(error_tags)
+        current_app.logger.error(log_data)
+        metrics.send("celery.failed_task", "TIMER", 1, metric_tags=error_tags)
+
+
+@task_revoked.connect
+def report_revoked_task(**kwargs):
+    """
+    Report a generic failure metric as tasks to our metrics broker every time a task is revoked.
+    This metric can be used for alerting.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked
+    """
+    with flask_app.app_context():
+        log_data = {
+            "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+            "Message": "Celery Task Revoked",
+        }
+
+        error_tags = get_celery_request_tags(**kwargs)
+
+        log_data.update(error_tags)
+        current_app.logger.error(log_data)
+        metrics.send("celery.revoked_task", "TIMER", 1, metric_tags=error_tags)
+
+
+@celery.task(soft_time_limit=600)
+def fetch_acme_cert(id):
+    """
+    Attempt to get the full certificate for the pending certificate listed.
+
+    Args:
+        id: an id of a PendingCertificate
+    """
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    log_data = {
+        "function": function,
+        "message": "Resolving pending certificate {}".format(id),
+        "task_id": task_id,
+        "id": id,
+    }
+
+    current_app.logger.debug(log_data)
+
+    if task_id and is_task_active(log_data["function"], task_id, (id,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    pending_certs = pending_certificate_service.get_pending_certs([id])
+    new = 0
+    failed = 0
+    wrong_issuer = 0
+    acme_certs = []
+
+    # We only care about certs using the acme-issuer plugin
+    for cert in pending_certs:
+        cert_authority = get_authority(cert.authority_id)
+        if cert_authority.plugin_name == "acme-issuer":
+            acme_certs.append(cert)
+        else:
+            wrong_issuer += 1
+
+    authority = plugins.get("acme-issuer")
+    resolved_certs = authority.get_ordered_certificates(acme_certs)
+
+    for cert in resolved_certs:
+        real_cert = cert.get("cert")
+        # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
+        pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
+        if not pending_cert or pending_cert.resolved:
+            # pending_cert is cleared or it was resolved by another process
+            log_data[
+                "message"
+            ] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
+            current_app.logger.error(log_data)
+            continue
+        if real_cert:
+            # If a real certificate was returned from issuer, then create it in Lemur and mark
+            # the pending certificate as resolved
+            final_cert = pending_certificate_service.create_certificate(
+                pending_cert, real_cert, pending_cert.user
+            )
+            pending_certificate_service.update(
+                cert.get("pending_cert").id, resolved_cert_id=final_cert.id
+            )
+            pending_certificate_service.update(
+                cert.get("pending_cert").id, resolved=True
+            )
+            # add metrics to metrics extension
+            new += 1
+        else:
+            failed += 1
+            error_log = copy.deepcopy(log_data)
+            error_log["message"] = "Pending certificate creation failure"
+            error_log["pending_cert_id"] = pending_cert.id
+            error_log["last_error"] = cert.get("last_error")
+            error_log["cn"] = pending_cert.cn
+
+            if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
+                error_log["message"] = "Deleting pending certificate"
+                send_pending_failure_notification(
+                    pending_cert, notify_owner=pending_cert.notify
+                )
+                # Mark the pending cert as resolved
+                pending_certificate_service.update(
+                    cert.get("pending_cert").id, resolved=True
+                )
+            else:
+                pending_certificate_service.increment_attempt(pending_cert)
+                pending_certificate_service.update(
+                    cert.get("pending_cert").id, status=str(cert.get("last_error"))
+                )
+                # Add failed pending cert task back to queue
+                fetch_acme_cert.delay(id)
+            current_app.logger.error(error_log)
+    log_data["message"] = "Complete"
+    log_data["new"] = new
+    log_data["failed"] = failed
+    log_data["wrong_issuer"] = wrong_issuer
+    current_app.logger.debug(log_data)
+    metrics.send(f"{function}.resolved", "gauge", new)
+    metrics.send(f"{function}.failed", "gauge", failed)
+    metrics.send(f"{function}.wrong_issuer", "gauge", wrong_issuer)
+    print(
+        "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
+            new=new, failed=failed, wrong_issuer=wrong_issuer
+        )
+    )
+    return log_data
+
+
+@celery.task()
+def fetch_all_pending_acme_certs():
+    """Instantiate celery workers to resolve all pending Acme certificates"""
+
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Starting job.",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
+
+    # We only care about certs using the acme-issuer plugin
+    for cert in pending_certs:
+        cert_authority = get_authority(cert.authority_id)
+        if cert_authority.plugin_name == "acme-issuer":
+            if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5):
+                log_data["message"] = "Triggering job for cert {}".format(cert.name)
+                log_data["cert_name"] = cert.name
+                log_data["cert_id"] = cert.id
+                current_app.logger.debug(log_data)
+                fetch_acme_cert.delay(cert.id)
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task()
+def remove_old_acme_certs():
+    """Prune old pending acme certificates from the database"""
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Starting job.",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    pending_certs = pending_certificate_service.get_pending_certs("all")
+
+    # Delete pending certs more than a week old
+    for cert in pending_certs:
+        if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
+            log_data["pending_cert_id"] = cert.id
+            log_data["pending_cert_name"] = cert.name
+            log_data["message"] = "Deleting pending certificate"
+            current_app.logger.debug(log_data)
+            pending_certificate_service.delete(cert)
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task()
+def clean_all_sources():
+    """
+    This function will clean unused certificates from sources. This is a destructive operation and should only
+    be ran periodically. This function triggers one celery task per source.
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Creating celery task to clean source",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    sources = validate_sources("all")
+    for source in sources:
+        log_data["source"] = source.label
+        current_app.logger.debug(log_data)
+        clean_source.delay(source.label)
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def clean_source(source):
+    """
+    This celery task will clean the specified source. This is a destructive operation that will delete unused
+    certificates from each source.
+
+    :param source:
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Cleaning source",
+        "source": source,
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, (source,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        clean([source], True)
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Clean source: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+    return log_data
+
+
+@celery.task()
+def sync_all_sources():
+    """
+    This function will sync certificates from all sources. This function triggers one celery task per source.
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "creating celery task to sync source",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    sources = validate_sources("all")
+    for source in sources:
+        log_data["source"] = source.label
+        current_app.logger.debug(log_data)
+        sync_source.delay(source.label)
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=7200)
+def sync_source(source):
+    """
+    This celery task will sync the specified source.
+
+    :param source:
+    :return:
+    """
+
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "Syncing source",
+        "source": source,
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, (source,)):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        sync([source])
+        metrics.send(
+            f"{function}.success", "counter", 1, metric_tags={"source": source}
+        )
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Error syncing source: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send(
+            "sync_source_timeout", "counter", 1, metric_tags={"source": source}
+        )
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    log_data["message"] = "Done syncing source"
+    current_app.logger.debug(log_data)
+    metrics.send(f"{function}.success", "counter", 1, metric_tags={"source": source})
+    return log_data
+
+
+@celery.task()
+def sync_source_destination():
+    """
+    This celery task will sync destination and source, to make sure all new destinations are also present as source.
+    Some destinations do not qualify as sources, and hence should be excluded from being added as sources
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "syncing AWS destinations and sources",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    for dst in destinations_service.get_all():
+        if add_aws_destination_to_sources(dst):
+            log_data["message"] = "new source added"
+            log_data["source"] = dst.label
+            current_app.logger.debug(log_data)
+
+    log_data["message"] = "completed Syncing AWS destinations and sources"
+    current_app.logger.debug(log_data)
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def certificate_reissue():
+    """
+    This celery task reissues certificates which are pending reissue
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "reissuing certificates",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.reissue(None, True)
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Certificate reissue: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    log_data["message"] = "reissuance completed"
+    current_app.logger.debug(log_data)
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def certificate_rotate(**kwargs):
+
+    """
+    This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    region = kwargs.get("region")
+    log_data = {
+        "function": function,
+        "message": "rotating certificates",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        notify = current_app.config.get("ENABLE_ROTATION_NOTIFICATION", None)
+        if region:
+            log_data["region"] = region
+            cli_certificate.rotate_region(None, None, None, notify, True, region)
+        else:
+            cli_certificate.rotate(None, None, None, notify, True)
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Certificate rotate: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    log_data["message"] = "rotation completed"
+    current_app.logger.debug(log_data)
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def endpoints_expire():
+    """
+    This celery task removes all endpoints that have not been recently updated
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "endpoints expire",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_endpoints.expire(2)  # Time in hours
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "endpoint expire: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=600)
+def get_all_zones():
+    """
+    This celery syncs all zones from the available dns providers
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "refresh all zones from available DNS providers",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_dns_providers.get_all_zones()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "get all zones: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def check_revoked():
+    """
+    This celery task attempts to check if any certs are expired
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "check if any valid certificate is revoked",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.check_revoked()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Checking revoked: Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def notify_expirations():
+    """
+    This celery task notifies about expiring certs
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "notify for cert expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_notification.expirations(
+            current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])
+        )
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Notify expiring Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def notify_authority_expirations():
+    """
+    This celery task notifies about expiring certificate authority certs
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "notify for certificate authority cert expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_notification.authority_expirations()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Notify expiring CA Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def send_security_expiration_summary():
+    """
+    This celery task sends a summary about expiring certificates to the security team.
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "send summary for certificate expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_notification.security_expiration_summary(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Send summary for expiring certs Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def enable_autorotate_for_certs_attached_to_endpoint():
+    """
+    This celery task automatically enables autorotation for unexpired certificates that are
+    attached to an endpoint but do not have autorotate enabled.
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "task_id": task_id,
+        "message": "Enabling autorotate to eligible certificates",
+    }
+    current_app.logger.debug(log_data)
+
+    cli_certificate.automatically_enable_autorotate()
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def deactivate_entrust_test_certificates():
+    """
+    This celery task attempts to deactivate all not yet deactivated Entrust certificates, and should only run in TEST
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "deactivate entrust certificates",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.deactivate_entrust_certificates()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data

lemur/common/defaults.py

@@ -2,23 +2,31 @@ import re
 import unicodedata
 
 from cryptography import x509
+from cryptography.hazmat.primitives.serialization import Encoding
 from flask import current_app
+
+from lemur.common.utils import is_selfsigned
 from lemur.extensions import sentry
 from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
 
 
-def text_to_slug(value):
-    """Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters."""
+def text_to_slug(value, joiner="-"):
+    """
+    Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters.
+    A series of non-alphanumeric characters is replaced with the joiner character.
+    """
 
     # Strip all character accents: decompose Unicode characters and then drop combining chars.
-    value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c))
+    value = "".join(
+        c for c in unicodedata.normalize("NFKD", value) if not unicodedata.combining(c)
+    )
 
-    # Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash.
-    # Except, keep 'xn--' used in IDNA domain names as is.
-    value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', '-', value)
+    # Replace all remaining non-alphanumeric characters with joiner string. Multiple characters get collapsed into a
+    # single joiner. Except, keep 'xn--' used in IDNA domain names as is.
+    value = re.sub(r"[^A-Za-z0-9.]+(?<!xn--)", joiner, value)
 
     # '-' in the beginning or end of string looks ugly.
-    return value.strip('-')
+    return value.strip(joiner)
 
 
 def certificate_name(common_name, issuer, not_before, not_after, san):
@@ -43,12 +51,12 @@ def certificate_name(common_name, issuer, not_before, not_after, san):
 
     temp = t.format(
         subject=common_name,
-        issuer=issuer.replace(' ', ''),
-        not_before=not_before.strftime('%Y%m%d'),
-        not_after=not_after.strftime('%Y%m%d')
+        issuer=issuer.replace(" ", ""),
+        not_before=not_before.strftime("%Y%m%d"),
+        not_after=not_after.strftime("%Y%m%d"),
     )
 
-    temp = temp.replace('*', "WILDCARD")
+    temp = temp.replace("*", "WILDCARD")
     return text_to_slug(temp)
 
 
@@ -64,12 +72,20 @@ def common_name(cert):
     :return: Common name or None
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_COMMON_NAME
-        )[0].value.strip()
+        subject_oid = cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)
+        if len(subject_oid) > 0:
+            return subject_oid[0].value.strip()
+        return None
     except Exception as e:
         sentry.captureException()
-        current_app.logger.error("Unable to get common name! {0}".format(e))
+        current_app.logger.error(
+            {
+                "message": "Unable to get common name",
+                "error": e,
+                "public_key": cert.public_bytes(Encoding.PEM).decode("utf-8")
+            },
+            exc_info=True
+        )
 
 
 def organization(cert):
@@ -79,9 +95,11 @@ def organization(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_ORGANIZATION_NAME
-        )[0].value.strip()
+        o = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
+        if not o:
+            return None
+
+        return o[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get organization! {0}".format(e))
@@ -94,9 +112,11 @@ def organizational_unit(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_ORGANIZATIONAL_UNIT_NAME
-        )[0].value.strip()
+        ou = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)
+        if not ou:
+            return None
+
+        return ou[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get organizational unit! {0}".format(e))
@@ -109,9 +129,11 @@ def country(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_COUNTRY_NAME
-        )[0].value.strip()
+        c = cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)
+        if not c:
+            return None
+
+        return c[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get country! {0}".format(e))
@@ -124,9 +146,11 @@ def state(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_STATE_OR_PROVINCE_NAME
-        )[0].value.strip()
+        s = cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)
+        if not s:
+            return None
+
+        return s[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get state! {0}".format(e))
@@ -139,9 +163,11 @@ def location(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(
-            x509.OID_LOCALITY_NAME
-        )[0].value.strip()
+        loc = cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)
+        if not loc:
+            return None
+
+        return loc[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get location! {0}".format(e))
@@ -219,29 +245,34 @@ def bitstrength(cert):
         return cert.public_key().key_size
     except AttributeError:
         sentry.captureException()
-        current_app.logger.debug('Unable to get bitstrength.')
+        current_app.logger.debug("Unable to get bitstrength.")
 
 
 def issuer(cert):
     """
-    Gets a sane issuer name from a given certificate.
+    Gets a sane issuer slug from a given certificate, stripping non-alphanumeric characters.
 
-    :param cert:
-    :return: Issuer
+    For self-signed certificates, the special value '<selfsigned>' is returned.
+    If issuer cannot be determined, '<unknown>' is returned.
+
+    :param cert: Parsed certificate object
+    :return: Issuer slug
     """
-    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
-    try:
-        # Try organization name or fall back to CN
-        issuer = (cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
-                  or cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME))
-        issuer = str(issuer[0].value)
-        for c in delchars:
-            issuer = issuer.replace(c, "")
-        return issuer
-    except Exception as e:
-        sentry.captureException()
-        current_app.logger.error("Unable to get issuer! {0}".format(e))
-        return "Unknown"
+    # If certificate is self-signed, we return a special value -- there really is no distinct "issuer" for it
+    if is_selfsigned(cert):
+        return "<selfsigned>"
+
+    # Try Common Name or fall back to Organization name
+    attrs = cert.issuer.get_attributes_for_oid(
+        x509.OID_COMMON_NAME
+    ) or cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
+    if not attrs:
+        current_app.logger.error(
+            "Unable to get issuer! Cert serial {:x}".format(cert.serial_number)
+        )
+        return "<unknown>"
+
+    return text_to_slug(attrs[0].value, "")
 
 
 def not_before(cert):

lemur/common/fields.py

@@ -25,6 +25,7 @@ class Hex(Field):
     """
     A hex formatted string.
     """
+
     def _serialize(self, value, attr, obj):
         if value:
             value = hex(int(value))[2:].upper()
@@ -48,25 +49,25 @@ class ArrowDateTime(Field):
     """
 
     DATEFORMAT_SERIALIZATION_FUNCS = {
-        'iso': utils.isoformat,
-        'iso8601': utils.isoformat,
-        'rfc': utils.rfcformat,
-        'rfc822': utils.rfcformat,
+        "iso": utils.isoformat,
+        "iso8601": utils.isoformat,
+        "rfc": utils.rfcformat,
+        "rfc822": utils.rfcformat,
     }
 
     DATEFORMAT_DESERIALIZATION_FUNCS = {
-        'iso': utils.from_iso,
-        'iso8601': utils.from_iso,
-        'rfc': utils.from_rfc,
-        'rfc822': utils.from_rfc,
+        "iso": utils.from_iso,
+        "iso8601": utils.from_iso,
+        "rfc": utils.from_rfc,
+        "rfc822": utils.from_rfc,
     }
 
-    DEFAULT_FORMAT = 'iso'
+    DEFAULT_FORMAT = "iso"
 
     localtime = False
     default_error_messages = {
-        'invalid': 'Not a valid datetime.',
-        'format': '"{input}" cannot be formatted as a datetime.',
+        "invalid": "Not a valid datetime.",
+        "format": '"{input}" cannot be formatted as a datetime.',
     }
 
     def __init__(self, format=None, **kwargs):
@@ -89,34 +90,36 @@ class ArrowDateTime(Field):
             try:
                 return format_func(value, localtime=self.localtime)
             except (AttributeError, ValueError) as err:
-                self.fail('format', input=value)
+                self.fail("format", input=value)
         else:
             return value.strftime(self.dateformat)
 
     def _deserialize(self, value, attr, data):
         if not value:  # Falsy values, e.g. '', None, [] are not valid
-            raise self.fail('invalid')
+            raise self.fail("invalid")
         self.dateformat = self.dateformat or self.DEFAULT_FORMAT
         func = self.DATEFORMAT_DESERIALIZATION_FUNCS.get(self.dateformat)
         if func:
             try:
                 return arrow.get(func(value))
             except (TypeError, AttributeError, ValueError):
-                raise self.fail('invalid')
+                raise self.fail("invalid")
         elif self.dateformat:
             try:
                 return dt.datetime.strptime(value, self.dateformat)
             except (TypeError, AttributeError, ValueError):
-                raise self.fail('invalid')
+                raise self.fail("invalid")
         elif utils.dateutil_available:
             try:
                 return arrow.get(utils.from_datestring(value))
             except TypeError:
-                raise self.fail('invalid')
+                raise self.fail("invalid")
         else:
-            warnings.warn('It is recommended that you install python-dateutil '
-                          'for improved datetime deserialization.')
-            raise self.fail('invalid')
+            warnings.warn(
+                "It is recommended that you install python-dateutil "
+                "for improved datetime deserialization."
+            )
+            raise self.fail("invalid")
 
 
 class KeyUsageExtension(Field):
@@ -131,73 +134,75 @@ class KeyUsageExtension(Field):
 
     def _serialize(self, value, attr, obj):
         return {
-            'useDigitalSignature': value.digital_signature,
-            'useNonRepudiation': value.content_commitment,
-            'useKeyEncipherment': value.key_encipherment,
-            'useDataEncipherment': value.data_encipherment,
-            'useKeyAgreement': value.key_agreement,
-            'useKeyCertSign': value.key_cert_sign,
-            'useCRLSign': value.crl_sign,
-            'useEncipherOnly': value._encipher_only,
-            'useDecipherOnly': value._decipher_only
+            "useDigitalSignature": value.digital_signature,
+            "useNonRepudiation": value.content_commitment,
+            "useKeyEncipherment": value.key_encipherment,
+            "useDataEncipherment": value.data_encipherment,
+            "useKeyAgreement": value.key_agreement,
+            "useKeyCertSign": value.key_cert_sign,
+            "useCRLSign": value.crl_sign,
+            "useEncipherOnly": value._encipher_only,
+            "useDecipherOnly": value._decipher_only,
         }
 
     def _deserialize(self, value, attr, data):
         keyusages = {
-            'digital_signature': False,
-            'content_commitment': False,
-            'key_encipherment': False,
-            'data_encipherment': False,
-            'key_agreement': False,
-            'key_cert_sign': False,
-            'crl_sign': False,
-            'encipher_only': False,
-            'decipher_only': False
+            "digital_signature": False,
+            "content_commitment": False,
+            "key_encipherment": False,
+            "data_encipherment": False,
+            "key_agreement": False,
+            "key_cert_sign": False,
+            "crl_sign": False,
+            "encipher_only": False,
+            "decipher_only": False,
         }
 
         for k, v in value.items():
-            if k == 'useDigitalSignature':
-                keyusages['digital_signature'] = v
+            if k == "useDigitalSignature":
+                keyusages["digital_signature"] = v
 
-            elif k == 'useNonRepudiation':
-                keyusages['content_commitment'] = v
+            elif k == "useNonRepudiation":
+                keyusages["content_commitment"] = v
 
-            elif k == 'useKeyEncipherment':
-                keyusages['key_encipherment'] = v
+            elif k == "useKeyEncipherment":
+                keyusages["key_encipherment"] = v
 
-            elif k == 'useDataEncipherment':
-                keyusages['data_encipherment'] = v
+            elif k == "useDataEncipherment":
+                keyusages["data_encipherment"] = v
 
-            elif k == 'useKeyCertSign':
-                keyusages['key_cert_sign'] = v
+            elif k == "useKeyCertSign":
+                keyusages["key_cert_sign"] = v
 
-            elif k == 'useCRLSign':
-                keyusages['crl_sign'] = v
+            elif k == "useCRLSign":
+                keyusages["crl_sign"] = v
 
-            elif k == 'useKeyAgreement':
-                keyusages['key_agreement'] = v
+            elif k == "useKeyAgreement":
+                keyusages["key_agreement"] = v
 
-            elif k == 'useEncipherOnly' and v:
-                keyusages['encipher_only'] = True
-                keyusages['key_agreement'] = True
+            elif k == "useEncipherOnly" and v:
+                keyusages["encipher_only"] = True
+                keyusages["key_agreement"] = True
 
-            elif k == 'useDecipherOnly' and v:
-                keyusages['decipher_only'] = True
-                keyusages['key_agreement'] = True
+            elif k == "useDecipherOnly" and v:
+                keyusages["decipher_only"] = True
+                keyusages["key_agreement"] = True
 
-        if keyusages['encipher_only'] and keyusages['decipher_only']:
-            raise ValidationError('A certificate cannot have both Encipher Only and Decipher Only Extended Key Usages.')
+        if keyusages["encipher_only"] and keyusages["decipher_only"]:
+            raise ValidationError(
+                "A certificate cannot have both Encipher Only and Decipher Only Extended Key Usages."
+            )
 
         return x509.KeyUsage(
-            digital_signature=keyusages['digital_signature'],
-            content_commitment=keyusages['content_commitment'],
-            key_encipherment=keyusages['key_encipherment'],
-            data_encipherment=keyusages['data_encipherment'],
-            key_agreement=keyusages['key_agreement'],
-            key_cert_sign=keyusages['key_cert_sign'],
-            crl_sign=keyusages['crl_sign'],
-            encipher_only=keyusages['encipher_only'],
-            decipher_only=keyusages['decipher_only']
+            digital_signature=keyusages["digital_signature"],
+            content_commitment=keyusages["content_commitment"],
+            key_encipherment=keyusages["key_encipherment"],
+            data_encipherment=keyusages["data_encipherment"],
+            key_agreement=keyusages["key_agreement"],
+            key_cert_sign=keyusages["key_cert_sign"],
+            crl_sign=keyusages["crl_sign"],
+            encipher_only=keyusages["encipher_only"],
+            decipher_only=keyusages["decipher_only"],
         )
 
 
@@ -216,69 +221,77 @@ class ExtendedKeyUsageExtension(Field):
         usage_list = {}
         for usage in usages:
             if usage == x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH:
-                usage_list['useClientAuthentication'] = True
+                usage_list["useClientAuthentication"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.SERVER_AUTH:
-                usage_list['useServerAuthentication'] = True
+                usage_list["useServerAuthentication"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.CODE_SIGNING:
-                usage_list['useCodeSigning'] = True
+                usage_list["useCodeSigning"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION:
-                usage_list['useEmailProtection'] = True
+                usage_list["useEmailProtection"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.TIME_STAMPING:
-                usage_list['useTimestamping'] = True
+                usage_list["useTimestamping"] = True
 
             elif usage == x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING:
-                usage_list['useOCSPSigning'] = True
+                usage_list["useOCSPSigning"] = True
 
-            elif usage.dotted_string == '1.3.6.1.5.5.7.3.14':
-                usage_list['useEapOverLAN'] = True
+            elif usage.dotted_string == "1.3.6.1.5.5.7.3.14":
+                usage_list["useEapOverLAN"] = True
 
-            elif usage.dotted_string == '1.3.6.1.5.5.7.3.13':
-                usage_list['useEapOverPPP'] = True
+            elif usage.dotted_string == "1.3.6.1.5.5.7.3.13":
+                usage_list["useEapOverPPP"] = True
 
-            elif usage.dotted_string == '1.3.6.1.4.1.311.20.2.2':
-                usage_list['useSmartCardLogon'] = True
+            elif usage.dotted_string == "1.3.6.1.4.1.311.20.2.2":
+                usage_list["useSmartCardLogon"] = True
 
             else:
-                current_app.logger.warning('Unable to serialize ExtendedKeyUsage with OID: {usage}'.format(usage=usage.dotted_string))
+                current_app.logger.warning(
+                    "Unable to serialize ExtendedKeyUsage with OID: {usage}".format(
+                        usage=usage.dotted_string
+                    )
+                )
 
         return usage_list
 
     def _deserialize(self, value, attr, data):
         usage_oids = []
         for k, v in value.items():
-            if k == 'useClientAuthentication' and v:
+            if k == "useClientAuthentication" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH)
 
-            elif k == 'useServerAuthentication' and v:
+            elif k == "useServerAuthentication" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.SERVER_AUTH)
 
-            elif k == 'useCodeSigning' and v:
+            elif k == "useCodeSigning" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.CODE_SIGNING)
 
-            elif k == 'useEmailProtection' and v:
+            elif k == "useEmailProtection" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION)
 
-            elif k == 'useTimestamping' and v:
+            elif k == "useTimestamping" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.TIME_STAMPING)
 
-            elif k == 'useOCSPSigning' and v:
+            elif k == "useOCSPSigning" and v:
                 usage_oids.append(x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING)
 
-            elif k == 'useEapOverLAN' and v:
+            elif k == "useEapOverLAN" and v:
                 usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.14"))
 
-            elif k == 'useEapOverPPP' and v:
+            elif k == "useEapOverPPP" and v:
                 usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.13"))
 
-            elif k == 'useSmartCardLogon' and v:
+            elif k == "useSmartCardLogon" and v:
                 usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.4.1.311.20.2.2"))
 
             else:
-                current_app.logger.warning('Unable to deserialize ExtendedKeyUsage with name: {key}'.format(key=k))
+                current_app.logger.warning(
+                    "Unable to deserialize ExtendedKeyUsage with name: {key}".format(
+                        key=k
+                    )
+                )
 
         return x509.ExtendedKeyUsage(usage_oids)
 
@@ -294,15 +307,17 @@ class BasicConstraintsExtension(Field):
     """
 
     def _serialize(self, value, attr, obj):
-        return {'ca': value.ca, 'path_length': value.path_length}
+        return {"ca": value.ca, "path_length": value.path_length}
 
     def _deserialize(self, value, attr, data):
-        ca = value.get('ca', False)
-        path_length = value.get('path_length', None)
+        ca = value.get("ca", False)
+        path_length = value.get("path_length", None)
 
         if ca:
             if not isinstance(path_length, (type(None), int)):
-                raise ValidationError('A CA certificate path_length (for BasicConstraints) must be None or an integer.')
+                raise ValidationError(
+                    "A CA certificate path_length (for BasicConstraints) must be None or an integer."
+                )
             return x509.BasicConstraints(ca=True, path_length=path_length)
         else:
             return x509.BasicConstraints(ca=False, path_length=None)
@@ -317,6 +332,7 @@ class SubjectAlternativeNameExtension(Field):
     :param kwargs: The same keyword arguments that :class:`Field` receives.
 
     """
+
     def _serialize(self, value, attr, obj):
         general_names = []
         name_type = None
@@ -326,52 +342,59 @@ class SubjectAlternativeNameExtension(Field):
                 value = name.value
 
                 if isinstance(name, x509.DNSName):
-                    name_type = 'DNSName'
+                    name_type = "DNSName"
 
                 elif isinstance(name, x509.IPAddress):
                     if isinstance(value, ipaddress.IPv4Network):
-                        name_type = 'IPNetwork'
+                        name_type = "IPNetwork"
                     else:
-                        name_type = 'IPAddress'
+                        name_type = "IPAddress"
 
                     value = str(value)
 
                 elif isinstance(name, x509.UniformResourceIdentifier):
-                    name_type = 'uniformResourceIdentifier'
+                    name_type = "uniformResourceIdentifier"
 
                 elif isinstance(name, x509.DirectoryName):
-                    name_type = 'directoryName'
+                    name_type = "directoryName"
 
                 elif isinstance(name, x509.RFC822Name):
-                    name_type = 'rfc822Name'
+                    name_type = "rfc822Name"
 
                 elif isinstance(name, x509.RegisteredID):
-                    name_type = 'registeredID'
+                    name_type = "registeredID"
                     value = value.dotted_string
                 else:
-                    current_app.logger.warning('Unknown SubAltName type: {name}'.format(name=name))
+                    current_app.logger.warning(
+                        "Unknown SubAltName type: {name}".format(name=name)
+                    )
+                    continue
 
-                general_names.append({'nameType': name_type, 'value': value})
+                general_names.append({"nameType": name_type, "value": value})
 
         return general_names
 
     def _deserialize(self, value, attr, data):
         general_names = []
         for name in value:
-            if name['nameType'] == 'DNSName':
-                validators.sensitive_domain(name['value'])
-                general_names.append(x509.DNSName(name['value']))
+            if name["nameType"] == "DNSName":
+                validators.sensitive_domain(name["value"])
+                general_names.append(x509.DNSName(name["value"]))
 
-            elif name['nameType'] == 'IPAddress':
-                general_names.append(x509.IPAddress(ipaddress.ip_address(name['value'])))
+            elif name["nameType"] == "IPAddress":
+                general_names.append(
+                    x509.IPAddress(ipaddress.ip_address(name["value"]))
+                )
 
-            elif name['nameType'] == 'IPNetwork':
-                general_names.append(x509.IPAddress(ipaddress.ip_network(name['value'])))
+            elif name["nameType"] == "IPNetwork":
+                general_names.append(
+                    x509.IPAddress(ipaddress.ip_network(name["value"]))
+                )
 
-            elif name['nameType'] == 'uniformResourceIdentifier':
-                general_names.append(x509.UniformResourceIdentifier(name['value']))
+            elif name["nameType"] == "uniformResourceIdentifier":
+                general_names.append(x509.UniformResourceIdentifier(name["value"]))
 
-            elif name['nameType'] == 'directoryName':
+            elif name["nameType"] == "directoryName":
                 # TODO: Need to parse a string in name['value'] like:
                 # 'CN=Common Name, O=Org Name, OU=OrgUnit Name, C=US, ST=ST, L=City/emailAddress=person@example.com'
                 # or
@@ -389,26 +412,32 @@ class SubjectAlternativeNameExtension(Field):
                 # general_names.append(x509.DirectoryName(x509.Name(BLAH))))
                 pass
 
-            elif name['nameType'] == 'rfc822Name':
-                general_names.append(x509.RFC822Name(name['value']))
+            elif name["nameType"] == "rfc822Name":
+                general_names.append(x509.RFC822Name(name["value"]))
 
-            elif name['nameType'] == 'registeredID':
-                general_names.append(x509.RegisteredID(x509.ObjectIdentifier(name['value'])))
+            elif name["nameType"] == "registeredID":
+                general_names.append(
+                    x509.RegisteredID(x509.ObjectIdentifier(name["value"]))
+                )
 
-            elif name['nameType'] == 'otherName':
+            elif name["nameType"] == "otherName":
                 # This has two inputs (type and value), so it doesn't fit the mold of the rest of these GeneralName entities.
                 # general_names.append(x509.OtherName(name['type'], bytes(name['value']), 'utf-8'))
                 pass
 
-            elif name['nameType'] == 'x400Address':
+            elif name["nameType"] == "x400Address":
                 # The Python Cryptography library doesn't support x400Address types (yet?)
                 pass
 
-            elif name['nameType'] == 'EDIPartyName':
+            elif name["nameType"] == "EDIPartyName":
                 # The Python Cryptography library doesn't support EDIPartyName types (yet?)
                 pass
 
             else:
-                current_app.logger.warning('Unable to deserialize SubAltName with type: {name_type}'.format(name_type=name['nameType']))
+                current_app.logger.warning(
+                    "Unable to deserialize SubAltName with type: {name_type}".format(
+                        name_type=name["nameType"]
+                    )
+                )
 
         return x509.SubjectAlternativeName(general_names)

lemur/common/health.py

@@ -10,20 +10,20 @@ from flask import Blueprint
 from lemur.database import db
 from lemur.extensions import sentry
 
-mod = Blueprint('healthCheck', __name__)
+mod = Blueprint("healthCheck", __name__)
 
 
-@mod.route('/healthcheck')
+@mod.route("/healthcheck")
 def health():
     try:
         if healthcheck(db):
-            return 'ok'
+            return "ok"
     except Exception:
         sentry.captureException()
-        return 'db check failed'
+        return "db check failed"
 
 
 def healthcheck(db):
     with db.engine.connect() as connection:
-        connection.execute('SELECT 1;')
+        connection.execute("SELECT 1;")
     return True

lemur/common/managers.py

@@ -52,7 +52,7 @@ class InstanceManager(object):
 
         results = []
         for cls_path in class_list:
-            module_name, class_name = cls_path.rsplit('.', 1)
+            module_name, class_name = cls_path.rsplit(".", 1)
             try:
                 module = __import__(module_name, {}, {}, class_name)
                 cls = getattr(module, class_name)
@@ -62,10 +62,14 @@ class InstanceManager(object):
                     results.append(cls)
 
             except InvalidConfiguration as e:
-                current_app.logger.warning("Plugin '{0}' may not work correctly. {1}".format(class_name, e))
+                current_app.logger.warning(
+                    "Plugin '{0}' may not work correctly. {1}".format(class_name, e)
+                )
 
             except Exception as e:
-                current_app.logger.exception("Unable to import {0}. Reason: {1}".format(cls_path, e))
+                current_app.logger.exception(
+                    "Unable to import {0}. Reason: {1}".format(cls_path, e)
+                )
                 continue
 
         self.cache = results

lemur/common/missing.py

@@ -11,14 +11,15 @@ def convert_validity_years(data):
     :param data:
     :return:
     """
-    if data.get('validity_years'):
+    if data.get("validity_years"):
         now = arrow.utcnow()
-        data['validity_start'] = now.isoformat()
+        data["validity_start"] = now.isoformat()
 
-        end = now.replace(years=+int(data['validity_years']))
-        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+        end = now.shift(years=+int(data["validity_years"]))
+
+        if not current_app.config.get("LEMUR_ALLOW_WEEKEND_EXPIRATION", True):
             if is_weekend(end):
-                end = end.replace(days=-2)
+                end = end.shift(days=-2)
 
-        data['validity_end'] = end.isoformat()
+        data["validity_end"] = end.isoformat()
     return data

lemur/common/redis.py

@@ -0,0 +1,52 @@
+"""
+Helper Class for Redis
+
+"""
+import redis
+import sys
+from flask import current_app
+from lemur.extensions import sentry
+from lemur.factory import create_app
+
+if current_app:
+    flask_app = current_app
+else:
+    flask_app = create_app()
+
+
+class RedisHandler:
+    def __init__(self, host=flask_app.config.get('REDIS_HOST', 'localhost'),
+                 port=flask_app.config.get('REDIS_PORT', 6379),
+                 db=flask_app.config.get('REDIS_DB', 0)):
+        self.host = host
+        self.port = port
+        self.db = db
+
+    def redis(self, db=0):
+        # The decode_responses flag here directs the client to convert the responses from Redis into Python strings
+        # using the default encoding utf-8.  This is client specific.
+        function = f"{__name__}.{sys._getframe().f_code.co_name}"
+        try:
+            red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True)
+            red.set("test", 0)
+        except redis.ConnectionError:
+            log_data = {
+                "function": function,
+                "message": "Redis Connection error",
+                "host": self.host,
+                "port": self.port
+            }
+            current_app.logger.error(log_data)
+            sentry.captureException()
+        return red
+
+
+def redis_get(key, default=None):
+    red = RedisHandler().redis()
+    try:
+        v = red.get(key)
+    except redis.exceptions.ConnectionError:
+        v = None
+    if not v:
+        return default
+    return v

lemur/common/schema.py

@@ -22,27 +22,26 @@ class LemurSchema(Schema):
     """
     Base schema from which all grouper schema's inherit
     """
+
     __envelope__ = True
 
     def under(self, data, many=None):
         items = []
         if many:
             for i in data:
-                items.append(
-                    {underscore(key): value for key, value in i.items()}
-                )
+                items.append({underscore(key): value for key, value in i.items()})
             return items
-        return {
-            underscore(key): value
-            for key, value in data.items()
-        }
+        return {underscore(key): value for key, value in data.items()}
 
     def camel(self, data, many=None):
         items = []
         if many:
             for i in data:
                 items.append(
-                    {camelize(key, uppercase_first_letter=False): value for key, value in i.items()}
+                    {
+                        camelize(key, uppercase_first_letter=False): value
+                        for key, value in i.items()
+                    }
                 )
             return items
         return {
@@ -52,14 +51,16 @@ class LemurSchema(Schema):
 
     def wrap_with_envelope(self, data, many):
         if many:
-            if 'total' in self.context.keys():
-                return dict(total=self.context['total'], items=data)
+            if "total" in self.context.keys():
+                return dict(total=self.context["total"], items=data)
         return data
 
 
 class LemurInputSchema(LemurSchema):
     @pre_load(pass_many=True)
     def preprocess(self, data, many):
+        if isinstance(data, dict) and data.get("owner"):
+            data["owner"] = data["owner"].lower()
         return self.under(data, many=many)
 
 
@@ -72,17 +73,17 @@ class LemurOutputSchema(LemurSchema):
 
     def unwrap_envelope(self, data, many):
         if many:
-            if data['items']:
+            if data["items"]:
                 if isinstance(data, InstrumentedList) or isinstance(data, list):
-                    self.context['total'] = len(data)
+                    self.context["total"] = len(data)
                     return data
                 else:
-                    self.context['total'] = data['total']
+                    self.context["total"] = data["total"]
             else:
-                self.context['total'] = 0
-                data = {'items': []}
+                self.context["total"] = 0
+                data = {"items": []}
 
-            return data['items']
+            return data["items"]
 
         return data
 
@@ -108,11 +109,11 @@ def format_errors(messages):
 
 
 def wrap_errors(messages):
-    errors = dict(message='Validation Error.')
-    if messages.get('_schema'):
-        errors['reasons'] = {'Schema': {'rule': messages['_schema']}}
+    errors = dict(message="Validation Error.")
+    if messages.get("_schema"):
+        errors["reasons"] = {"Schema": {"rule": messages["_schema"]}}
     else:
-        errors['reasons'] = format_errors(messages)
+        errors["reasons"] = format_errors(messages)
     return errors
 
 
@@ -121,19 +122,19 @@ def unwrap_pagination(data, output_schema):
         return data
 
     if isinstance(data, dict):
-        if 'total' in data.keys():
-            if data.get('total') == 0:
+        if "total" in data.keys():
+            if data.get("total") == 0:
                 return data
 
-            marshaled_data = {'total': data['total']}
-            marshaled_data['items'] = output_schema.dump(data['items'], many=True).data
+            marshaled_data = {"total": data["total"]}
+            marshaled_data["items"] = output_schema.dump(data["items"], many=True).data
             return marshaled_data
 
         return output_schema.dump(data).data
 
     elif isinstance(data, list):
-        marshaled_data = {'total': len(data)}
-        marshaled_data['items'] = output_schema.dump(data, many=True).data
+        marshaled_data = {"total": len(data)}
+        marshaled_data["items"] = output_schema.dump(data, many=True).data
         return marshaled_data
     return output_schema.dump(data).data
 
@@ -153,7 +154,7 @@ def validate_schema(input_schema, output_schema):
                 if errors:
                     return wrap_errors(errors), 400
 
-                kwargs['data'] = data
+                kwargs["data"] = data
 
             try:
                 resp = f(*args, **kwargs)
@@ -168,7 +169,13 @@ def validate_schema(input_schema, output_schema):
             if not resp:
                 return dict(message="No data found"), 404
 
-            return unwrap_pagination(resp, output_schema), 200
+            if callable(output_schema):
+                output_schema_to_use = output_schema()
+            else:
+                output_schema_to_use = output_schema
+
+            return unwrap_pagination(resp, output_schema_to_use), 200
 
         return decorated_function
+
     return decorator

lemur/common/utils.py

@@ -7,12 +7,18 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import random
+import re
 import string
+import pem
+import base64
 
 import sqlalchemy
 from cryptography import x509
+from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric import rsa, ec
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding
+from cryptography.hazmat.primitives.serialization import load_pem_private_key, Encoding, pkcs7
 from flask_restful.reqparse import RequestParser
 from sqlalchemy import and_, func
 
@@ -21,21 +27,28 @@ from lemur.exceptions import InvalidConfiguration
 
 paginated_parser = RequestParser()
 
-paginated_parser.add_argument('count', type=int, default=10, location='args')
-paginated_parser.add_argument('page', type=int, default=1, location='args')
-paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
-paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
-paginated_parser.add_argument('filter', type=str, location='args')
+paginated_parser.add_argument("count", type=int, default=10, location="args")
+paginated_parser.add_argument("page", type=int, default=1, location="args")
+paginated_parser.add_argument("sortDir", type=str, dest="sort_dir", location="args")
+paginated_parser.add_argument("sortBy", type=str, dest="sort_by", location="args")
+paginated_parser.add_argument("filter", type=str, location="args")
+paginated_parser.add_argument("owner", type=str, location="args")
+
+
+def base64encode(string):
+    # Performs Base64 encoding of string to string using the base64.b64encode() function
+    # which encodes bytes to bytes.
+    return base64.b64encode(string.encode()).decode()
 
 
 def get_psuedo_random_string():
     """
     Create a random and strongish challenge.
     """
-    challenge = ''.join(random.choice(string.ascii_uppercase) for x in range(6))  # noqa
-    challenge += ''.join(random.choice("~!@#$%^&*()_+") for x in range(6))  # noqa
-    challenge += ''.join(random.choice(string.ascii_lowercase) for x in range(6))
-    challenge += ''.join(random.choice(string.digits) for x in range(6))  # noqa
+    challenge = "".join(random.choice(string.ascii_uppercase) for x in range(6))  # noqa
+    challenge += "".join(random.choice("~!@#$%^&*()_+") for x in range(6))  # noqa
+    challenge += "".join(random.choice(string.ascii_lowercase) for x in range(6))
+    challenge += "".join(random.choice(string.digits) for x in range(6))  # noqa
     return challenge
 
 
@@ -46,10 +59,63 @@ def parse_certificate(body):
     :param body:
     :return:
     """
-    if isinstance(body, str):
-        body = body.encode('utf-8')
+    assert isinstance(body, str)
 
-    return x509.load_pem_x509_certificate(body, default_backend())
+    return x509.load_pem_x509_certificate(body.encode("utf-8"), default_backend())
+
+
+def parse_private_key(private_key):
+    """
+    Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm).
+
+    Raises ValueError for an invalid string. Raises AssertionError when passed value is not str-type.
+
+    :param private_key: String containing PEM private key
+    """
+    assert isinstance(private_key, str)
+
+    return load_pem_private_key(
+        private_key.encode("utf8"), password=None, backend=default_backend()
+    )
+
+
+def get_key_type_from_certificate(body):
+    """
+
+    Helper function to determine key type by pasrding given PEM certificate
+
+    :param body: PEM string
+    :return: Key type string
+    """
+    parsed_cert = parse_certificate(body)
+    if isinstance(parsed_cert.public_key(), rsa.RSAPublicKey):
+        return "RSA{key_size}".format(
+            key_size=parsed_cert.public_key().key_size
+        )
+    elif isinstance(parsed_cert.public_key(), ec.EllipticCurvePublicKey):
+        return get_key_type_from_ec_curve(parsed_cert.public_key().curve.name)
+
+
+def split_pem(data):
+    """
+    Split a string of several PEM payloads to a list of strings.
+
+    :param data: String
+    :return: List of strings
+    """
+    return re.split("\n(?=-----BEGIN )", data)
+
+
+def parse_cert_chain(pem_chain):
+    """
+    Helper function to split and parse a series of PEM certificates.
+
+    :param pem_chain: string
+    :return: List of parsed certificates
+    """
+    if pem_chain is None:
+        return []
+    return [parse_certificate(cert) for cert in split_pem(pem_chain) if cert]
 
 
 def parse_csr(csr):
@@ -59,20 +125,53 @@ def parse_csr(csr):
     :param csr:
     :return:
     """
-    if isinstance(csr, str):
-        csr = csr.encode('utf-8')
+    assert isinstance(csr, str)
 
-    return x509.load_pem_x509_csr(csr, default_backend())
+    return x509.load_pem_x509_csr(csr.encode("utf-8"), default_backend())
 
 
 def get_authority_key(body):
     """Returns the authority key for a given certificate in hex format"""
     parsed_cert = parse_certificate(body)
     authority_key = parsed_cert.extensions.get_extension_for_class(
-        x509.AuthorityKeyIdentifier).value.key_identifier
+        x509.AuthorityKeyIdentifier
+    ).value.key_identifier
     return authority_key.hex()
 
 
+def get_key_type_from_ec_curve(curve_name):
+    """
+    Give an EC curve name, return the matching key_type.
+
+    :param: curve_name
+    :return: key_type
+    """
+
+    _CURVE_TYPES = {
+        ec.SECP192R1().name: "ECCPRIME192V1",
+        ec.SECP256R1().name: "ECCPRIME256V1",
+        ec.SECP224R1().name: "ECCSECP224R1",
+        ec.SECP384R1().name: "ECCSECP384R1",
+        ec.SECP521R1().name: "ECCSECP521R1",
+        ec.SECP256K1().name: "ECCSECP256K1",
+        ec.SECT163K1().name: "ECCSECT163K1",
+        ec.SECT233K1().name: "ECCSECT233K1",
+        ec.SECT283K1().name: "ECCSECT283K1",
+        ec.SECT409K1().name: "ECCSECT409K1",
+        ec.SECT571K1().name: "ECCSECT571K1",
+        ec.SECT163R2().name: "ECCSECT163R2",
+        ec.SECT233R1().name: "ECCSECT233R1",
+        ec.SECT283R1().name: "ECCSECT283R1",
+        ec.SECT409R1().name: "ECCSECT409R1",
+        ec.SECT571R1().name: "ECCSECT571R2",
+    }
+
+    if curve_name in _CURVE_TYPES.keys():
+        return _CURVE_TYPES[curve_name]
+    else:
+        return None
+
+
 def generate_private_key(key_type):
     """
     Generates a new private key based on key_type.
@@ -87,22 +186,19 @@ def generate_private_key(key_type):
     """
 
     _CURVE_TYPES = {
-        "ECCPRIME192V1": ec.SECP192R1(),
-        "ECCPRIME256V1": ec.SECP256R1(),
-
-        "ECCSECP192R1": ec.SECP192R1(),
+        "ECCPRIME192V1": ec.SECP192R1(),  # duplicate
+        "ECCPRIME256V1": ec.SECP256R1(),  # duplicate
+        "ECCSECP192R1": ec.SECP192R1(),  # duplicate
         "ECCSECP224R1": ec.SECP224R1(),
-        "ECCSECP256R1": ec.SECP256R1(),
+        "ECCSECP256R1": ec.SECP256R1(),  # duplicate
         "ECCSECP384R1": ec.SECP384R1(),
         "ECCSECP521R1": ec.SECP521R1(),
         "ECCSECP256K1": ec.SECP256K1(),
-
         "ECCSECT163K1": ec.SECT163K1(),
         "ECCSECT233K1": ec.SECT233K1(),
         "ECCSECT283K1": ec.SECT283K1(),
         "ECCSECT409K1": ec.SECT409K1(),
         "ECCSECT571K1": ec.SECT571K1(),
-
         "ECCSECT163R2": ec.SECT163R2(),
         "ECCSECT233R1": ec.SECT233R1(),
         "ECCSECT283R1": ec.SECT283R1(),
@@ -111,25 +207,74 @@ def generate_private_key(key_type):
     }
 
     if key_type not in CERTIFICATE_KEY_TYPES:
-        raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format(
-            key_type=key_type,
-            choices=",".join(CERTIFICATE_KEY_TYPES)
-        ))
+        raise Exception(
+            "Invalid key type: {key_type}. Supported key types: {choices}".format(
+                key_type=key_type, choices=",".join(CERTIFICATE_KEY_TYPES)
+            )
+        )
 
-    if 'RSA' in key_type:
+    if "RSA" in key_type:
         key_size = int(key_type[3:])
         return rsa.generate_private_key(
-            public_exponent=65537,
-            key_size=key_size,
-            backend=default_backend()
+            public_exponent=65537, key_size=key_size, backend=default_backend()
         )
-    elif 'ECC' in key_type:
+    elif "ECC" in key_type:
         return ec.generate_private_key(
-            curve=_CURVE_TYPES[key_type],
-            backend=default_backend()
+            curve=_CURVE_TYPES[key_type], backend=default_backend()
         )
 
 
+def check_cert_signature(cert, issuer_public_key):
+    """
+    Check a certificate's signature against an issuer public key.
+    Before EC validation, make sure we support the algorithm, otherwise raise UnsupportedAlgorithm
+    On success, returns None; on failure, raises UnsupportedAlgorithm or InvalidSignature.
+    """
+    if isinstance(issuer_public_key, rsa.RSAPublicKey):
+        # RSA requires padding, just to make life difficult for us poor developers :(
+        if cert.signature_algorithm_oid == x509.SignatureAlgorithmOID.RSASSA_PSS:
+            # In 2005, IETF devised a more secure padding scheme to replace PKCS #1 v1.5. To make sure that
+            # nobody can easily support or use it, they mandated lots of complicated parameters, unlike any
+            # other X.509 signature scheme.
+            # https://tools.ietf.org/html/rfc4056
+            raise UnsupportedAlgorithm("RSASSA-PSS not supported")
+        else:
+            padder = padding.PKCS1v15()
+        issuer_public_key.verify(
+            cert.signature,
+            cert.tbs_certificate_bytes,
+            padder,
+            cert.signature_hash_algorithm,
+        )
+    elif isinstance(issuer_public_key, ec.EllipticCurvePublicKey) and isinstance(
+        ec.ECDSA(cert.signature_hash_algorithm), ec.ECDSA
+    ):
+        issuer_public_key.verify(
+            cert.signature,
+            cert.tbs_certificate_bytes,
+            ec.ECDSA(cert.signature_hash_algorithm),
+        )
+    else:
+        raise UnsupportedAlgorithm(
+            "Unsupported Algorithm '{var}'.".format(
+                var=cert.signature_algorithm_oid._name
+            )
+        )
+
+
+def is_selfsigned(cert):
+    """
+    Returns True if the certificate is self-signed.
+    Returns False for failed verification or unsupported signing algorithm.
+    """
+    try:
+        check_cert_signature(cert, cert.public_key())
+        # If verification was successful, it's self-signed.
+        return True
+    except InvalidSignature:
+        return False
+
+
 def is_weekend(date):
     """
     Determines if a given date is on a weekend.
@@ -149,8 +294,10 @@ def validate_conf(app, required_vars):
     :param required_vars: list
     """
     for var in required_vars:
-        if not app.config.get(var):
-            raise InvalidConfiguration("Required variable '{var}' is not set in Lemur's conf.".format(var=var))
+        if var not in app.config:
+            raise InvalidConfiguration(
+                "Required variable '{var}' is not set in Lemur's conf.".format(var=var)
+            )
 
 
 # https://bitbucket.org/zzzeek/sqlalchemy/wiki/UsageRecipes/WindowedRangeQuery
@@ -169,18 +316,15 @@ def column_windows(session, column, windowsize):
     be computed.
 
     """
+
     def int_for_range(start_id, end_id):
         if end_id:
-            return and_(
-                column >= start_id,
-                column < end_id
-            )
+            return and_(column >= start_id, column < end_id)
         else:
             return column >= start_id
 
     q = session.query(
-        column,
-        func.row_number().over(order_by=column).label('rownum')
+        column, func.row_number().over(order_by=column).label("rownum")
     ).from_self(column)
 
     if windowsize > 1:
@@ -200,9 +344,7 @@ def column_windows(session, column, windowsize):
 def windowed_query(q, column, windowsize):
     """"Break a Query into windows on a given column."""
 
-    for whereclause in column_windows(
-            q.session,
-            column, windowsize):
+    for whereclause in column_windows(q.session, column, windowsize):
         for row in q.filter(whereclause).order_by(column):
             yield row
 
@@ -210,4 +352,32 @@ def windowed_query(q, column, windowsize):
 def truthiness(s):
     """If input string resembles something truthy then return True, else False."""
 
-    return s.lower() in ('true', 'yes', 'on', 't', '1')
+    return s.lower() in ("true", "yes", "on", "t", "1")
+
+
+def find_matching_certificates_by_hash(cert, matching_certs):
+    """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs),
+    determine if any of the certificate hashes match and return the matches."""
+    matching = []
+    for c in matching_certs:
+        if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(
+            hashes.SHA256()
+        ):
+            matching.append(c)
+    return matching
+
+
+def convert_pkcs7_bytes_to_pem(certs_pkcs7):
+    """
+    Given a list of certificates in pkcs7 encoding (bytes), covert them into a list of PEM encoded files
+    :raises ValueError or ValidationError
+    :param certs_pkcs7:
+    :return: list of certs in PEM format
+    """
+
+    certificates = pkcs7.load_pem_pkcs7_certificates(certs_pkcs7)
+    certificates_pem = []
+    for cert in certificates:
+        certificates_pem.append(pem.parse(cert.public_bytes(encoding=Encoding.PEM))[0])
+
+    return certificates_pem

lemur/common/validators.py

@@ -1,45 +1,14 @@
 import re
 
 from cryptography import x509
+from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
 from cryptography.x509 import NameOID
 from flask import current_app
 from marshmallow.exceptions import ValidationError
 
 from lemur.auth.permissions import SensitiveDomainPermission
-from lemur.common.utils import parse_certificate, is_weekend
-from lemur.domains import service as domain_service
-
-
-def public_certificate(body):
-    """
-    Determines if specified string is valid public certificate.
-
-    :param body:
-    :return:
-    """
-    try:
-        parse_certificate(body)
-    except Exception as e:
-        current_app.logger.exception(e)
-        raise ValidationError('Public certificate presented is not valid.')
-
-
-def private_key(key):
-    """
-    User to validate that a given string is a RSA private key
-
-    :param key:
-    :return: :raise ValueError:
-    """
-    try:
-        if isinstance(key, bytes):
-            serialization.load_pem_private_key(key, None, backend=default_backend())
-        else:
-            serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
-    except Exception:
-        raise ValidationError('Private key presented is not valid.')
+from lemur.common.utils import check_cert_signature, is_weekend
 
 
 def common_name(value):
@@ -47,13 +16,13 @@ def common_name(value):
     # Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client
     # certificates). As a simple heuristic, we assume that human-readable names always include a space.
     # However, to avoid confusion for humans, we also don't count spaces at the beginning or end of the string.
-    if ' ' not in value.strip():
+    if " " not in value.strip():
         return sensitive_domain(value)
 
 
 def sensitive_domain(domain):
     """
-    Checks if user has the admin role, the domain does not match sensitive domains and whitelisted domain patterns.
+    Checks if user has the admin role, the domain does not match sensitive domains and allowed domain patterns.
     :param domain: domain name (str)
     :return:
     """
@@ -61,14 +30,21 @@ def sensitive_domain(domain):
         # User has permission, no need to check anything
         return
 
-    whitelist = current_app.config.get('LEMUR_WHITELISTED_DOMAINS', [])
-    if whitelist and not any(re.match(pattern, domain) for pattern in whitelist):
-        raise ValidationError('Domain {0} does not match whitelisted domain patterns. '
-                              'Contact an administrator to issue the certificate.'.format(domain))
+    allowlist = current_app.config.get("LEMUR_ALLOWED_DOMAINS", [])
+    if allowlist and not any(re.match(pattern, domain) for pattern in allowlist):
+        raise ValidationError(
+            "Domain {0} does not match allowed domain patterns. "
+            "Contact an administrator to issue the certificate.".format(domain)
+        )
 
-    if any(d.sensitive for d in domain_service.get_by_name(domain)):
-        raise ValidationError('Domain {0} has been marked as sensitive. '
-                              'Contact an administrator to issue the certificate.'.format(domain))
+    # Avoid circular import.
+    from lemur.domains import service as domain_service
+
+    if domain_service.is_domain_sensitive(domain):
+        raise ValidationError(
+            "Domain {0} has been marked as sensitive. "
+            "Contact an administrator to issue the certificate.".format(domain)
+        )
 
 
 def encoding(oid_encoding):
@@ -77,9 +53,13 @@ def encoding(oid_encoding):
     :param oid_encoding:
     :return:
     """
-    valid_types = ['b64asn1', 'string', 'ia5string']
+    valid_types = ["b64asn1", "string", "ia5string"]
     if oid_encoding.lower() not in [o_type.lower() for o_type in valid_types]:
-        raise ValidationError('Invalid Oid Encoding: {0} choose from {1}'.format(oid_encoding, ",".join(valid_types)))
+        raise ValidationError(
+            "Invalid Oid Encoding: {0} choose from {1}".format(
+                oid_encoding, ",".join(valid_types)
+            )
+        )
 
 
 def sub_alt_type(alt_type):
@@ -88,10 +68,23 @@ def sub_alt_type(alt_type):
     :param alt_type:
     :return:
     """
-    valid_types = ['DNSName', 'IPAddress', 'uniFormResourceIdentifier', 'directoryName', 'rfc822Name', 'registrationID',
-                   'otherName', 'x400Address', 'EDIPartyName']
+    valid_types = [
+        "DNSName",
+        "IPAddress",
+        "uniFormResourceIdentifier",
+        "directoryName",
+        "rfc822Name",
+        "registrationID",
+        "otherName",
+        "x400Address",
+        "EDIPartyName",
+    ]
     if alt_type.lower() not in [a_type.lower() for a_type in valid_types]:
-        raise ValidationError('Invalid SubAltName Type: {0} choose from {1}'.format(type, ",".join(valid_types)))
+        raise ValidationError(
+            "Invalid SubAltName Type: {0} choose from {1}".format(
+                type, ",".join(valid_types)
+            )
+        )
 
 
 def csr(data):
@@ -101,16 +94,22 @@ def csr(data):
     :return:
     """
     try:
-        request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
+        request = x509.load_pem_x509_csr(data.encode("utf-8"), default_backend())
     except Exception:
-        raise ValidationError('CSR presented is not valid.')
+        raise ValidationError("CSR presented is not valid.")
 
     # Validate common name and SubjectAltNames
+    try:
         for name in request.subject.get_attributes_for_oid(NameOID.COMMON_NAME):
             common_name(name.value)
+    except ValueError as err:
+        current_app.logger.info("Error parsing Subject from CSR: %s", err)
+        raise ValidationError("Invalid Subject value in supplied CSR")
 
     try:
-        alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
+        alt_names = request.extensions.get_extension_for_class(
+            x509.SubjectAlternativeName
+        )
 
         for name in alt_names.value.get_values_for_type(x509.DNSName):
             sensitive_domain(name)
@@ -119,25 +118,87 @@ def csr(data):
 
 
 def dates(data):
-    if not data.get('validity_start') and data.get('validity_end'):
-        raise ValidationError('If validity start is specified so must validity end.')
+    if not data.get("validity_start") and data.get("validity_end"):
+        raise ValidationError("If validity start is specified so must validity end.")
 
-    if not data.get('validity_end') and data.get('validity_start'):
-        raise ValidationError('If validity end is specified so must validity start.')
+    if not data.get("validity_end") and data.get("validity_start"):
+        raise ValidationError("If validity end is specified so must validity start.")
 
-    if data.get('validity_start') and data.get('validity_end'):
-        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
-            if is_weekend(data.get('validity_end')):
-                raise ValidationError('Validity end must not land on a weekend.')
+    if data.get("validity_start") and data.get("validity_end"):
+        if not current_app.config.get("LEMUR_ALLOW_WEEKEND_EXPIRATION", True):
+            if is_weekend(data.get("validity_end")):
+                raise ValidationError("Validity end must not land on a weekend.")
 
-        if not data['validity_start'] < data['validity_end']:
-            raise ValidationError('Validity start must be before validity end.')
+        if not data["validity_start"] < data["validity_end"]:
+            raise ValidationError("Validity start must be before validity end.")
 
-        if data.get('authority'):
-            if data.get('validity_start').date() < data['authority'].authority_certificate.not_before.date():
-                raise ValidationError('Validity start must not be before {0}'.format(data['authority'].authority_certificate.not_before))
+        if data.get("authority"):
+            if (
+                data.get("validity_start").date()
+                < data["authority"].authority_certificate.not_before.date()
+            ):
+                raise ValidationError(
+                    "Validity start must not be before {0}".format(
+                        data["authority"].authority_certificate.not_before
+                    )
+                )
 
-            if data.get('validity_end').date() > data['authority'].authority_certificate.not_after.date():
-                raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
+            if (
+                data.get("validity_end").date()
+                > data["authority"].authority_certificate.not_after.date()
+            ):
+                raise ValidationError(
+                    "Validity end must not be after {0}".format(
+                        data["authority"].authority_certificate.not_after
+                    )
+                )
 
     return data
+
+
+def verify_private_key_match(key, cert, error_class=ValidationError):
+    """
+    Checks that the supplied private key matches the certificate.
+
+    :param cert: Parsed certificate
+    :param key: Parsed private key
+    :param error_class: Exception class to raise on error
+    """
+    if key.public_key().public_numbers() != cert.public_key().public_numbers():
+        raise error_class("Private key does not match certificate.")
+
+
+def verify_cert_chain(certs, error_class=ValidationError):
+    """
+    Verifies that the certificates in the chain are correct.
+
+    We don't bother with full cert validation but just check that certs in the chain are signed by the next, to avoid
+    basic human errors -- such as pasting the wrong certificate.
+
+    :param certs: List of parsed certificates, use parse_cert_chain()
+    :param error_class: Exception class to raise on error
+    """
+    cert = certs[0]
+    for issuer in certs[1:]:
+        # Use the current cert's public key to verify the previous signature.
+        # "certificate validation is a complex problem that involves much more than just signature checks"
+        try:
+            check_cert_signature(cert, issuer.public_key())
+
+        except InvalidSignature:
+            # Avoid circular import.
+            from lemur.common import defaults
+
+            raise error_class(
+                "Incorrect chain certificate(s) provided: '%s' is not signed by '%s'"
+                % (
+                    defaults.common_name(cert) or "Unknown",
+                    defaults.common_name(issuer),
+                )
+            )
+
+        except UnsupportedAlgorithm as err:
+            current_app.logger.warning("Skipping chain validation: %s", err)
+
+        # Next loop will validate that *this issuer* cert is signed by the next chain cert.
+        cert = issuer

lemur/constants.py

@@ -3,32 +3,51 @@
     :copyright: (c) 2018 by Netflix Inc.
     :license: Apache, see LICENSE for more details.
 """
+from enum import IntEnum
+
 SAN_NAMING_TEMPLATE = "SAN-{subject}-{issuer}-{not_before}-{not_after}"
 DEFAULT_NAMING_TEMPLATE = "{subject}-{issuer}-{not_before}-{not_after}"
 NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
 
-SUCCESS_METRIC_STATUS = 'success'
-FAILURE_METRIC_STATUS = 'failure'
+SUCCESS_METRIC_STATUS = "success"
+FAILURE_METRIC_STATUS = "failure"
+
+# when ACME attempts to resolve a certificate try in total 3 times
+ACME_ADDITIONAL_ATTEMPTS = 2
 
 CERTIFICATE_KEY_TYPES = [
-    'RSA2048',
-    'RSA4096',
-    'ECCPRIME192V1',
-    'ECCPRIME256V1',
-    'ECCSECP192R1',
-    'ECCSECP224R1',
-    'ECCSECP256R1',
-    'ECCSECP384R1',
-    'ECCSECP521R1',
-    'ECCSECP256K1',
-    'ECCSECT163K1',
-    'ECCSECT233K1',
-    'ECCSECT283K1',
-    'ECCSECT409K1',
-    'ECCSECT571K1',
-    'ECCSECT163R2',
-    'ECCSECT233R1',
-    'ECCSECT283R1',
-    'ECCSECT409R1',
-    'ECCSECT571R2'
+    "RSA2048",
+    "RSA4096",
+    "ECCPRIME192V1",
+    "ECCPRIME256V1",
+    "ECCSECP192R1",
+    "ECCSECP224R1",
+    "ECCSECP256R1",
+    "ECCSECP384R1",
+    "ECCSECP521R1",
+    "ECCSECP256K1",
+    "ECCSECT163K1",
+    "ECCSECT233K1",
+    "ECCSECT283K1",
+    "ECCSECT409K1",
+    "ECCSECT571K1",
+    "ECCSECT163R2",
+    "ECCSECT233R1",
+    "ECCSECT283R1",
+    "ECCSECT409R1",
+    "ECCSECT571R2",
 ]
+
+
+# As per RFC 5280 section 5.3.1 (https://tools.ietf.org/html/rfc5280#section-5.3.1)
+class CRLReason(IntEnum):
+    unspecified = 0,
+    keyCompromise = 1,
+    cACompromise = 2,
+    affiliationChanged = 3,
+    superseded = 4,
+    cessationOfOperation = 5,
+    certificateHold = 6,
+    removeFromCRL = 8,
+    privilegeWithdrawn = 9,
+    aACompromise = 10

lemur/database.py

@@ -9,13 +9,14 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import math
 from inflection import underscore
-from sqlalchemy import exc, func
+from sqlalchemy import exc, func, distinct
+from sqlalchemy.orm import make_transient, lazyload
 from sqlalchemy.sql import and_, or_
-from sqlalchemy.orm import make_transient
 
-from lemur.extensions import db
 from lemur.exceptions import AttrNotFound, DuplicateError
+from lemur.extensions import db
 
 
 def filter_none(kwargs):
@@ -43,7 +44,7 @@ def session_query(model):
     :param model: sqlalchemy model
     :return: query object for model
     """
-    return model.query if hasattr(model, 'query') else db.session.query(model)
+    return model.query if hasattr(model, "query") else db.session.query(model)
 
 
 def create_query(model, kwargs):
@@ -77,7 +78,7 @@ def add(model):
 
 
 def get_model_column(model, field):
-    if field in getattr(model, 'sensitive_fields', ()):
+    if field in getattr(model, "sensitive_fields", ()):
         raise AttrNotFound(field)
     column = model.__table__.columns._data.get(field, None)
     if column is None:
@@ -100,7 +101,7 @@ def find_all(query, model, kwargs):
     kwargs = filter_none(kwargs)
     for attr, value in kwargs.items():
         if not isinstance(value, list):
-            value = value.split(',')
+            value = value.split(",")
 
         conditions.append(get_model_column(model, attr).in_(value))
 
@@ -200,7 +201,7 @@ def filter(query, model, terms):
     :return:
     """
     column = get_model_column(model, underscore(terms[0]))
-    return query.filter(column.ilike('%{}%'.format(terms[1])))
+    return query.filter(column.ilike("%{}%".format(terms[1])))
 
 
 def sort(query, model, field, direction):
@@ -214,18 +215,25 @@ def sort(query, model, field, direction):
     :param direction:
     """
     column = get_model_column(model, underscore(field))
-    return query.order_by(column.desc() if direction == 'desc' else column.asc())
+    return query.order_by(column.desc() if direction == "desc" else column.asc())
 
 
 def paginate(query, page, count):
     """
-    Returns the items given the count and page specified
+    Returns the items given the count and page specified. The items would be an empty list
+    if page number exceeds max page number based on count per page and total number of records.
 
-    :param query:
-    :param page:
-    :param count:
+    :param query: search query
+    :param page: current page number
+    :param count: results per page
     """
-    return query.paginate(page, count)
+    total = get_count(query)
+    # Check if input page is higher than total number of pages based on count per page and total
+    # In such a case Flask-SQLAlchemy pagination call results in 404
+    if math.ceil(total / count) < page:
+        return dict(items=[], total=total)
+    items = query.paginate(page, count).items
+    return dict(items=items, total=total)
 
 
 def update_list(model, model_attr, item_model, items):
@@ -247,10 +255,10 @@ def update_list(model, model_attr, item_model, items):
 
     for i in items:
         for item in getattr(model, model_attr):
-            if item.id == i['id']:
+            if item.id == i["id"]:
                 break
         else:
-            getattr(model, model_attr).append(get(item_model, i['id']))
+            getattr(model, model_attr).append(get(item_model, i["id"]))
 
     return model
 
@@ -273,7 +281,35 @@ def get_count(q):
     :param q:
     :return:
     """
-    count_q = q.statement.with_only_columns([func.count()]).order_by(None)
+    disable_group_by = False
+    if len(q._entities) > 1:
+        # currently support only one entity
+        raise Exception("only one entity is supported for get_count, got: %s" % q)
+    entity = q._entities[0]
+    if hasattr(entity, "column"):
+        # _ColumnEntity has column attr - on case: query(Model.column)...
+        col = entity.column
+        if q._group_by and q._distinct:
+            # which query can have both?
+            raise NotImplementedError
+        if q._group_by or q._distinct:
+            col = distinct(col)
+        if q._group_by:
+            # need to disable group_by and enable distinct - we can do this because we have only 1 entity
+            disable_group_by = True
+        count_func = func.count(col)
+    else:
+        # _MapperEntity doesn't have column attr - on case: query(Model)...
+        count_func = func.count()
+    if q._group_by and not disable_group_by:
+        count_func = count_func.over(None)
+    count_q = (
+        q.options(lazyload("*"))
+        .statement.with_only_columns([count_func])
+        .order_by(None)
+    )
+    if disable_group_by:
+        count_q = count_q.group_by(None)
     count = q.session.execute(count_q).scalar()
     return count
 
@@ -287,13 +323,13 @@ def sort_and_page(query, model, args):
     :param args:
     :return:
     """
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
+    sort_by = args.pop("sort_by")
+    sort_dir = args.pop("sort_dir")
+    page = args.pop("page")
+    count = args.pop("count")
 
-    if args.get('user'):
-        user = args.pop('user')
+    if args.get("user"):
+        user = args.pop("user")
 
     query = find_all(query, model, args)