add OpenSSH plugin

release 0.9.0 changelog

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# These owners will be the default owners for everything in the repo.
+*                            @hosseinsh @csine-nflx @charhate @jtschladen

.github/dependabot.yml

@@ -0,0 +1,15 @@
+  version: 2
+  updates:
+    - directory: "/"
+      schedule:
+        interval: "weekly"
+        day: "monday"
+        time: "08:00"
+        timezone: "America/Los_Angeles"
+      package-ecosystem: "pip"
+      reviewers:
+        - "hosseinsh"
+        - "csine-nflx"
+        - "charhate"
+        - "jtschladen"
+      versioning-strategy: lockfile-only

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '15 16 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    
+    # Install prerequisites for python-ldap. See: https://www.python-ldap.org/en/python-ldap-3.3.0/installing.html#build-prerequisites
+    - name: Install python-ldap prerequisites
+      run: sudo apt-get install libldap2-dev libsasl2-dev
+    
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dependabot-auto-merge.yml

@@ -0,0 +1,14 @@
+name: dependabot-auto-merge
+
+on:
+  pull_request:
+
+jobs:
+  auto-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ahmadnassri/action-dependabot-auto-merge@v2
+        with:
+          target: minor
+          github-token: ${{ secrets.DEPENDABOT_GITHUB_TOKEN }}

.github/workflows/lemur-publish-release-pypi.yml

@@ -0,0 +1,41 @@
+# This workflow will upload a Python Package using Twine when a Lemur release is created via github
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Publish Lemur's latest package to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Autobump version
+      run: |
+        # from refs/tags/v0.8.1 get 0.8.1
+        VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+        PLACEHOLDER='__version__ = "develop"'
+        VERSION_FILE='lemur/__about__.py'
+        # in case placeholder is missing, exists with code 1 and github actions aborts the build
+        grep "$PLACEHOLDER" "$VERSION_FILE"
+        sed -i "s/$PLACEHOLDER/__version__ = \"${VERSION}\"/g" "$VERSION_FILE"
+      shell: bash
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.LEMUR_PYPI_API_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.LEMUR_PYPI_API_TOKEN }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.gitignore

@@ -39,3 +39,4 @@ lemur/tests/tmp
 
 /lemur/plugins/lemur_email/tests/expiration-rendered.html
 /lemur/plugins/lemur_email/tests/rotation-rendered.html
+.celerybeat-schedule

.readthedocs.yml

@@ -0,0 +1,23 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/conf.py
+   fail_on_warning: true
+
+# Build docs in all formats (html, pdf, epub)
+formats: all
+
+# Set the version of Python and requirements required to build the docs
+python:
+   version: 3.7
+   install:
+   - requirements: requirements-docs.txt
+   - method: setuptools
+     path: .
+   system_packages: true

.travis.yml

@@ -1,16 +1,50 @@
-language: python
-dist: bionic
-
 node_js:
-  - "6.2.0"
+  - "10"
 
-addons:
+jobs:
-  postgresql: "9.4"
-
-matrix:
   include:
-    - python: "3.7"
+    - name: "python3.7-postgresql-9.4-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
       env: TOXENV=py37
+      addons:
+        postgresql: "9.4"
+        chrome: stable
+      services:
+        - xvfb
+    - name: "python3.7-postgresql-10-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
+      env: TOXENV=py37
+      addons:
+        postgresql: '10'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-10
+            - postgresql-client-10
+            - postgresql-server-dev-10
+      services:
+        - postgresql
+        - xvfb
+    - name: "python3.8-postgresql-12-focal"
+      dist: focal
+      language: python
+      python: "3.8"
+      env: TOXENV=py38
+      addons:
+        postgresql: '12'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-12
+            - postgresql-client-12
+            - postgresql-server-dev-12
+      services:
+        - postgresql
+        - xvfb
 
 cache:
   directories:
@@ -26,13 +60,23 @@ env:
     # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
     - BOTO_CONFIG=/doesnotexist
 
+before_install:
+  - export CHROME_BIN=/usr/bin/google-chrome
+
 before_script:
+  - sudo systemctl stop postgresql
+  # the port may have been auto-configured to use 5433 if it thought 5422 was already in use,
+  # for some reason it happens very often
+  # https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_setup_postgresql.bash#L52
+  - sudo sed -i -e 's/5433/5432/' /etc/postgresql/*/main/postgresql.conf
+  - sudo systemctl restart postgresql
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
   - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
   - npm config set registry https://registry.npmjs.org
-  - npm install -g bower
+  - npm install -g npm@latest bower
   - pip install --upgrade setuptools
+  - export DISPLAY=:99.0
 
 install:
   - pip install coveralls
@@ -41,6 +85,7 @@ install:
 script:
   - make test
   - bandit -r . -ll -ii -x lemur/tests/,docs
+  - make test-js
 
 after_success:
   - coveralls
@@ -51,3 +96,4 @@ notifications:
       - lemur@netflix.com
     on_success: never
     on_failure: always
+    on_cancel: never # Dependbot cancels Travis before rebase and triggers too many emails

CHANGELOG.rst

@@ -1,8 +1,46 @@
 Changelog
 =========
 
+0.9.0 - `2021-03-17`
+~~~~~~~~~~~~~~~~~~~~
+
+This release fixes three critical vulnerabilities where an authenticated user could retrieve/access
+unauthorized information. (Issue `#3463 <https://github.com/Netflix/lemur/issues/3463>`_)
+
+
+0.8.1 - `2021-03-12`
+~~~~~~~~~~~~~~~~~~~~
+
+This release includes improvements on many fronts, such as:
+
+- Notifications:
+    - Enhanced SNS flow
+    - Expiration Summary
+    - CA expiration email
+- EC algorithm as the default
+- Improved revocation flow
+- Localized AWS STS option
+- Improved Lemur doc building
+- ACME:
+    - reduced failed attempts to 3x trials
+    - support for selecting the chain (Let's Encrypt X1 transition)
+    - revocation
+    - http01 documentation
+- Entrust:
+    - Support for cross-signed intermediate CA
+- Revised disclosure process
+- Dependency updates and conflict resolutions
+
+Special thanks to all who contributed to this release, notably:
+
+- `peschmae  <https://github.com/peschmae>`_
+- `atugushev  <https://github.com/atugushev>`_
+- `sirferl   <https://github.com/sirferl>`_
+
+
+
 0.8.0 - `2020-11-13`
-~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 
 This release comes after more than two years and contains many interesting new features and improvements.
 In addition to multiple new plugins, such as ACME-http01, ADCS, PowerDNS, UltraDNS, Entrust, SNS, many of Lemur's existing
@@ -84,7 +122,7 @@ Upgrading
 
 
 0.7 - `2018-05-07`
-~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality.
 The linux_dst plugin will also be deprecated and removed.
@@ -121,8 +159,7 @@ Happy Holidays! This is a big release with lots of bug fixes and features. Below
 
 Features:
 
-* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates.
+* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used.
-is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used.
 * Adds per-user API Keys, allows users to issue multiple long-lived API tokens with the same permission as the user creating them.
 * Adds the ability to revoke certificates from the Lemur UI/API, this is currently only supported for the digicert CIS and cfssl plugins.
 * Allow destinations to support an export function. Useful for file system destinations e.g. S3 to specify the export plugin you wish to run before being sent to the destination.
@@ -166,13 +203,9 @@ Big thanks to neilschelly for quite a lot of improvements to the `lemur-cryptogr
 
 Other Highlights:
 
-* Closed `#501 <https://github.com/Netflix/lemur/issues/501>`_ - Endpoint resource as now kept in sync via an
+* Closed `#501 <https://github.com/Netflix/lemur/issues/501>`_ - Endpoint resource as now kept in sync via an expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never removed from Lemur.
-expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never
+* Closed `#551 <https://github.com/Netflix/lemur/pull/551>`_ - Added the ability to create a 4096 bit key during certificate creation. Closed `#528 <https://github.com/Netflix/lemur/pull/528>`_ to ensure that issuer plugins supported the new 4096 bit keys.
-removed from Lemur.
+* Closed `#566 <https://github.com/Netflix/lemur/issues/566>`_ - Fixed an issue changing the notification status for  certificates without private keys.
-* Closed `#551 <https://github.com/Netflix/lemur/pull/551>`_ - Added the ability to create a 4096 bit key during certificate
-creation. Closed `#528 <https://github.com/Netflix/lemur/pull/528>`_ to ensure that issuer plugins supported the new 4096 bit keys.
-* Closed `#566 <https://github.com/Netflix/lemur/issues/566>`_ - Fixed an issue changing the notification status for  certificates
-without private keys.
 * Closed `#594 <https://github.com/Netflix/lemur/issues/594>`_ - Added `replaced` field indicating if a certificate has been superseded.
 * Closed `#602 <https://github.com/Netflix/lemur/issues/602>`_ - AWS plugin added support for ALBs for endpoint tracking.
 
@@ -196,12 +229,8 @@ Upgrading
 
 There have been quite a few issues closed in this release. Some notables:
 
-* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated
+* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for future enhancements of Lemur's certificate 'deployment' capabilities.
-AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for
+* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability to restrict certificate expiration dates to weekdays.
-future enhancements of Lemur's certificate 'deployment' capabilities.
-
-* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability
-to restrict certificate expiration dates to weekdays.
 
 Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!)
 
@@ -256,7 +285,7 @@ these keys should be fairly trivial, additionally pull requests have been submit
     should be easier to determine what authorities are available and when an authority has actually been selected.
 * Closed `#254 <https://github.com/Netflix/lemur/issues/254>`_ - Forces certificate names to be generally unique. If a certificate name
     (generated or otherwise) is found to be a duplicate we increment by appending a counter.
-* Closed `#254 <https://github.com/Netflix/lemur/issues/275>`_ - Switched to using Fernet generated passphrases for exported items.
+* Closed `#275 <https://github.com/Netflix/lemur/issues/275>`_ - Switched to using Fernet generated passphrases for exported items.
     These are more sounds that pseudo random passphrases generated before and have the nice property of being in base64.
 * Closed `#278 <https://github.com/Netflix/lemur/issues/278>`_ - Added ability to specify a custom name to certificate creation, previously
     this was only available in the certificate import wizard.

Makefile

@@ -115,10 +115,10 @@ endif
 	@echo "--> Updating Python requirements"
 	pip install --upgrade pip
 	pip install --upgrade pip-tools
-	pip-compile --output-file requirements.txt requirements.in -U --no-index
+	pip-compile --output-file requirements.txt requirements.in -U --no-emit-index-url
-	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
+	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-emit-index-url
-	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
+	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-emit-index-url
-	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
+	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-emit-index-url
 	@echo "--> Done updating Python requirements"
 	@echo "--> Removing python-ldap from requirements-docs.txt"
 	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

bower.json

@@ -29,7 +29,7 @@
     "satellizer": "~0.13.4",
     "angular-ui-router": "~0.2.15",
     "font-awesome": "~4.5.0",
-    "lodash": "~4.0.1",
+    "lodash": "~4.17.20",
     "underscore": "~1.8.3",
     "angular-smart-table": "2.1.8",
     "angular-strap": ">= 2.2.2",

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.8
+FROM python:3.7.9-alpine3.12
 
 ARG VERSION
 ENV VERSION master
@@ -12,7 +12,7 @@ ENV group lemur
 
 RUN addgroup -S ${group} -g ${gid} && \
     adduser -D -S ${user} -G ${group} -u ${uid} && \
-    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk add --no-cache --update python3 py-pip libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
     apk --update add --virtual build-dependencies \
                 git \
                 tar \
@@ -42,7 +42,9 @@ RUN addgroup -S ${group} -g ${gid} && \
 
 WORKDIR /opt/lemur
 
-RUN npm install --unsafe-perm && \
+RUN echo "Running with python:" && python -c 'import platform; print(platform.python_version())' && \
+    echo "Running with nodejs:" && node -v && \
+    npm install --unsafe-perm && \
     pip3 install -e . && \
     node_modules/.bin/gulp build && \
     node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \

docker/docker-compose.yml

@@ -1,9 +1,12 @@
 version: '3'
 
+volumes:
+  pg_data: { }
+
 services:
   postgres:
-    image: "postgres:10"
+    image: "postgres:13.1-alpine"
-    restart: always
+    restart: on-failure
     volumes:
       - pg_data:/var/lib/postgresql/data
     env_file:
@@ -11,7 +14,9 @@ services:
 
   lemur:
     # image: "netlix-lemur:latest"
-    build: .
+    restart: on-failure
+    build:
+      context: .
     depends_on:
       - postgres
       - redis
@@ -19,11 +24,9 @@ services:
       - lemur-env
       - pgsql-env
     ports:
-      - 80:80
+      - 87:80
-      - 443:443
+      - 447:443
 
   redis:
-    image: "redis:alpine"
+    image: "redis:alpine3.12"
-
+    restart: on-failure
-volumes:
-    pg_data: {}

docker/entrypoint

@@ -14,10 +14,10 @@ export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}"
 export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
 
 
-PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'select 1;'
 
 echo " # Create Postgres trgm extension"
-PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 echo " # Done"
 
 if [ -z "${SKIP_SSL}" ]; then

docker/src/lemur.conf.py

@@ -1,11 +1,18 @@
-import os
+import os.path
 import random
 import string
+from celery.schedules import crontab
+
 import base64
-from ast import literal_eval
 
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
+# See the Lemur docs (https://lemur.readthedocs.org) for more information on configuration
+
+LOG_LEVEL = str(os.environ.get('LOG_LEVEL', 'DEBUG'))
+LOG_FILE = str(os.environ.get('LOG_FILE', '/home/lemur/.lemur/lemur.log'))
+LOG_JSON = True
+
 CORS = os.environ.get("CORS") == "True"
 debug = os.environ.get("DEBUG") == "True"
 
@@ -17,44 +24,214 @@ def get_random_secret(length):
     return secret_key + ''.join(random.choice(string.digits) for x in range(round(length / 4)))
 
 
+# This is the secret key used by Flask session management
 SECRET_KEY = repr(os.environ.get('SECRET_KEY', get_random_secret(32).encode('utf8')))
 
+# You should consider storing these separately from your config
 LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET',
                                          base64.b64encode(get_random_secret(32).encode('utf8'))))
+# This must match the key for whichever DB the container is using - this could be a dump of dev or test, or a unique key
 LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS',
-                                            base64.b64encode(get_random_secret(32).encode('utf8'))))
+                                            base64.b64encode(get_random_secret(32).encode('utf8')).decode('utf8')))
 
-LEMUR_ALLOWED_DOMAINS = []
+REDIS_HOST = 'redis'
+REDIS_PORT = 6379
+REDIS_DB = 0
+CELERY_RESULT_BACKEND = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_BROKER_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_IMPORTS = ('lemur.common.celery')
+CELERYBEAT_SCHEDULE = {
+    # All tasks are disabled by default. Enable any tasks you wish to run.
+    # 'fetch_all_pending_acme_certs': {
+    #     'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'remove_old_acme_certs': {
+    #     'task': 'lemur.common.celery.remove_old_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=8, minute=0, day_of_week=5),
+    # },
+    # 'clean_all_sources': {
+    #     'task': 'lemur.common.celery.clean_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=5, minute=0, day_of_week=5),
+    # },
+    # 'sync_all_sources': {
+    #     'task': 'lemur.common.celery.sync_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=0),
+    #     # this job is running 30min before endpoints_expire which deletes endpoints which were not updated
+    # },
+    # 'sync_source_destination': {
+    #     'task': 'lemur.common.celery.sync_source_destination',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=15),
+    # },
+    # 'report_celery_last_success_metrics': {
+    #     'task': 'lemur.common.celery.report_celery_last_success_metrics',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'certificate_reissue': {
+    #     'task': 'lemur.common.celery.certificate_reissue',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=9, minute=0),
+    # },
+    # 'certificate_rotate': {
+    #     'task': 'lemur.common.celery.certificate_rotate',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'endpoints_expire': {
+    #     'task': 'lemur.common.celery.endpoints_expire',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=30),
+    #     # this job is running 30min after sync_all_sources which updates endpoints
+    # },
+    # 'get_all_zones': {
+    #     'task': 'lemur.common.celery.get_all_zones',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*/30"),
+    # },
+    # 'check_revoked': {
+    #     'task': 'lemur.common.celery.check_revoked',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'enable_autorotate_for_certs_attached_to_endpoint': {
+    #     'task': 'lemur.common.celery.enable_autorotate_for_certs_attached_to_endpoint',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'notify_expirations': {
+    #     'task': 'lemur.common.celery.notify_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    #  },
+    # 'notify_authority_expirations': {
+    #     'task': 'lemur.common.celery.notify_authority_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'send_security_expiration_summary': {
+    #     'task': 'lemur.common.celery.send_security_expiration_summary',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0, day_of_week='mon-fri'),
+    # }
+}
+CELERY_TIMEZONE = 'UTC'
 
-LEMUR_EMAIL = ''
+SQLALCHEMY_ENABLE_FLASK_REPLICATED = False
-LEMUR_SECURITY_TEAM_EMAIL = []
+SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://lemur:lemur@localhost:5432/lemur')
 
-ALLOW_CERT_DELETION = os.environ.get('ALLOW_CERT_DELETION') == "True"
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+SQLALCHEMY_ECHO = True
+SQLALCHEMY_POOL_RECYCLE = 499
+SQLALCHEMY_POOL_TIMEOUT = 20
 
-LEMUR_DEFAULT_COUNTRY = str(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
+LEMUR_EMAIL = 'lemur@example.com'
-LEMUR_DEFAULT_STATE = str(os.environ.get('LEMUR_DEFAULT_STATE',''))
+LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
-LEMUR_DEFAULT_LOCATION = str(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
+LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
-LEMUR_DEFAULT_ORGANIZATION = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
+LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
-LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
+LEMUR_EMAIL_SENDER = 'smtp'
 
-LEMUR_DEFAULT_ISSUER_PLUGIN = str(os.environ.get('LEMUR_DEFAULT_ISSUER_PLUGIN',''))
+# mail configuration
-LEMUR_DEFAULT_AUTHORITY = str(os.environ.get('LEMUR_DEFAULT_AUTHORITY',''))
+# MAIL_SERVER = 'mail.example.com'
+
+PUBLIC_CA_MAX_VALIDITY_DAYS = 397
+DEFAULT_VALIDITY_DAYS = 365
+
+LEMUR_OWNER_EMAIL_IN_SUBJECT = False
+
+LEMUR_DEFAULT_COUNTRY = str(os.environ.get('LEMUR_DEFAULT_COUNTRY', 'US'))
+LEMUR_DEFAULT_STATE = str(os.environ.get('LEMUR_DEFAULT_STATE', 'California'))
+LEMUR_DEFAULT_LOCATION = str(os.environ.get('LEMUR_DEFAULT_LOCATION', 'Los Gatos'))
+LEMUR_DEFAULT_ORGANIZATION = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATION', 'Example, Inc.'))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT', ''))
+
+LEMUR_DEFAULT_AUTHORITY = str(os.environ.get('LEMUR_DEFAULT_AUTHORITY', 'ExampleCa'))
+
+LEMUR_DEFAULT_ROLE = 'operator'
 
 ACTIVE_PROVIDERS = []
-
 METRIC_PROVIDERS = []
 
-LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
+# Authority Settings - These will change depending on which authorities you are
-LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
+# using
+current_path = os.path.dirname(os.path.realpath(__file__))
 
-SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')
+# DNS Settings
 
-LDAP_DEBUG = os.environ.get('LDAP_DEBUG') == "True"
+# exclude logging missing SAN, since we can have certs from private CAs with only cn, prod parity
-LDAP_AUTH = os.environ.get('LDAP_AUTH') == "True"
+LOG_SSL_SUBJ_ALT_NAME_ERRORS = False
-LDAP_IS_ACTIVE_DIRECTORY = os.environ.get('LDAP_IS_ACTIVE_DIRECTORY') == "True"
+
-LDAP_BIND_URI = str(os.environ.get('LDAP_BIND_URI',''))
+ACME_DNS_PROVIDER_TYPES = {"items": [
-LDAP_BASE_DN = str(os.environ.get('LDAP_BASE_DN',''))
+    {
-LDAP_EMAIL_DOMAIN = str(os.environ.get('LDAP_EMAIL_DOMAIN',''))
+        'name': 'route53',
-LDAP_USE_TLS = str(os.environ.get('LDAP_USE_TLS',''))
+        'requirements': [
-LDAP_REQUIRED_GROUP = str(os.environ.get('LDAP_REQUIRED_GROUP',''))
+            {
-LDAP_GROUPS_TO_ROLES = literal_eval(os.environ.get('LDAP_GROUPS_TO_ROLES') or "{}")
+                'name': 'account_id',
+                'type': 'int',
+                'required': True,
+                'helpMessage': 'AWS Account number'
+            },
+        ]
+    },
+    {
+        'name': 'cloudflare',
+        'requirements': [
+            {
+                'name': 'email',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Email'
+            },
+            {
+                'name': 'key',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Key'
+            },
+        ]
+    },
+    {
+        'name': 'dyn',
+    },
+    {
+        'name': 'ultradns',
+    },
+]}
+
+# Authority plugins which support revocation
+SUPPORTED_REVOCATION_AUTHORITY_PLUGINS = ['acme-issuer']

docs/administration.rst

@@ -84,7 +84,7 @@ Basic Configuration
 
 
     .. warning::
-This is an optional setting but important to review and set for optimal database connection usage and for overall database performance.
+        This is an optional setting but important to review and set for optimal database connection usage and for overall database performance.
 
 .. data:: SQLALCHEMY_MAX_OVERFLOW
     :noindex:
@@ -99,7 +99,7 @@ This is an optional setting but important to review and set for optimal database
 
 
     .. note::
-Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size.
+        Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size.
 
 
 .. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION
@@ -151,6 +151,15 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
         to start. Multiple keys can be provided to facilitate key rotation. The first key in the list is used for
         encryption and all keys are tried for decryption until one works. Each key must be 32 URL safe base-64 encoded bytes.
 
+        Only fields of type ``Vault`` will be encrypted. At present, only the following fields are encrypted:
+
+        * ``certificates.private_key``
+        * ``pending_certificates.private_key``
+        * ``dns_providers.credentials``
+        * ``roles.password``
+
+        For implementation details, see ``Vault`` in ``utils.py``.
+
         Running lemur create_config will securely generate a key for your configuration file.
         If you would like to generate your own, we recommend the following method:
 
@@ -165,6 +174,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
 
 .. data:: PUBLIC_CA_MAX_VALIDITY_DAYS
     :noindex:
+
         Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities.
         The authorities with cab_compliant option set to true will use this config. The example below overrides the default validity
         of 397 days and sets it to 365 days.
@@ -176,6 +186,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
 
 .. data:: DEFAULT_VALIDITY_DAYS
     :noindex:
+
         Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which
         is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please
         note that this config is used for cert issuance only through Lemur UI. The example below overrides the default validity
@@ -198,6 +209,11 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
         in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed"
         and deleted certificates will always be visible in the UI. (default: `False`)
 
+.. data:: LEMUR_AWS_REGION
+    :noindex:
+
+        This is an optional config applicable for settings where Lemur is deployed in AWS. For accessing regionalized
+        STS endpoints, LEMUR_AWS_REGION defines the region where Lemur is deployed.
 
 Certificate Default Options
 ---------------------------
@@ -262,22 +278,123 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_AUTHORITY = "verisign"
 
 
+.. _NotificationOptions:
+
 Notification Options
 --------------------
 
-Lemur currently has very basic support for notifications. Currently only expiration notifications are supported. Actual notification
+Lemur supports a small variety of notification types through a set of notification plugins.
-is handled by the notification plugins that you have configured. Lemur ships with the 'Email' notification that allows expiration emails
+By default, Lemur configures a standard set of email notifications for all certificates.
-to be sent to subscribers.
 
-Templates for expiration emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
+**Plugin-capable notifications**
-Notifications are sent to the certificate creator, owner and security team as specified by the `LEMUR_SECURITY_TEAM_EMAIL` configuration parameter.
 
-Certificates marked as inactive will **not** be notified of upcoming expiration. This enables a user to essentially
+These notifications can be configured to use all available notification plugins.
-silence the expiration. If a certificate is active and is expiring the above will be notified according to the `LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS` or
-30, 15, 2 days before expiration if no intervals are set.
 
-Lemur supports sending certificate expiration notifications through SES and SMTP.
+Supported types:
 
+* Certificate expiration
+
+**Email-only notifications**
+
+These notifications can only be sent via email and cannot use other notification plugins.
+
+Supported types:
+
+* CA certificate expiration
+* Pending ACME certificate failure
+* Certificate rotation
+* Security certificate expiration summary
+
+**Default notifications**
+
+When a certificate is created, the following email notifications are created for it if they do not exist.
+If these notifications already exist, they will be associated with the new certificate.
+
+* ``DEFAULT_<OWNER>_X_DAY``, where X is the set of values specified in ``LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS`` and defaults to 30, 15, and 2 if not specified. The owner's username will replace ``<OWNER>``.
+* ``DEFAULT_SECURITY_X_DAY``, where X is the set of values specified in ``LEMUR_SECURITY_TEAM_EMAIL_INTERVALS`` and defaults to ``LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS`` if not specified (which also defaults to 30, 15, and 2 if not specified).
+
+These notifications can be disabled if desired. They can also be unassociated with a specific certificate.
+
+**Disabling notifications**
+
+Notifications can be disabled either for an individual certificate (which disables all notifications for that certificate)
+or for an individual notification object (which disables that notification for all associated certificates).
+At present, disabling a notification object will only disable certificate expiration notifications, and not other types,
+since other notification types don't use notification objects.
+
+**Certificate expiration**
+
+Certificate expiration notifications are sent when the scheduled task to send certificate expiration notifications runs
+(see :ref:`PeriodicTasks`). Specific patterns of certificate names may be excluded using ``--exclude`` (when using
+cron; you may specify this multiple times for multiple patterns) or via the config option ``EXCLUDE_CN_FROM_NOTIFICATION``
+(when using celery; this is a list configuration option, meaning you specify multiple values, such as
+``['exclude', 'also exclude']``). The specified exclude pattern will match if found anywhere in the certificate name.
+
+When the periodic task runs, Lemur checks for certificates meeting the following conditions:
+
+* Certificate has notifications enabled
+* Certificate is not expired
+* Certificate is not revoked
+* Certificate name does not match the `exclude` parameter
+* Certificate has at least one associated notification object
+* That notification is active
+* That notification's configured interval and unit match the certificate's remaining lifespan
+
+All eligible certificates are then grouped by owner and applicable notification. For each notification and certificate group,
+Lemur will send the expiration notification using whichever plugin was configured for that notification object.
+In addition, Lemur will send an email to the certificate owner and security team (as specified by the
+``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter).
+
+**CA certificate expiration**
+
+Certificate authority certificate expiration notifications are sent when the scheduled task to send authority certificate
+expiration notifications runs (see :ref:`PeriodicTasks`). Notifications are sent via the intervals configured in the
+configuration parameter ``LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS``, with a default of 365 and 180 days.
+
+When the periodic task runs, Lemur checks for certificates meeting the following conditions:
+
+* Certificate has notifications enabled
+* Certificate is not expired
+* Certificate is not revoked
+* Certificate is associated with a CA
+* Certificate's remaining lifespan matches one of the configured intervals
+
+All eligible certificates are then grouped by owner and expiration interval. For each interval and certificate group,
+Lemur will send the CA certificate expiration notification via email to the certificate owner and security team
+(as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter).
+
+**Pending ACME certificate failure**
+
+Whenever a pending ACME certificate fails to be issued, Lemur will send a notification via email to the certificate owner
+and security team (as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter). This email is not sent if
+the pending certificate had notifications disabled.
+
+Lemur will attempt 3x times to resolve a pending certificate.
+This can at times result into 3 duplicate certificates, if all certificate attempts get resolved.
+
+**Certificate rotation**
+
+Whenever a cert is rotated, Lemur will send a notification via email to the certificate owner. This notification is
+disabled by default; to enable it, you must set the option ``--notify`` (when using cron) or the configuration parameter
+``ENABLE_ROTATION_NOTIFICATION`` (when using celery).
+
+**Security certificate expiration summary**
+
+If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all
+certificates with upcoming expiration date that occurs within the number of days specified by the
+``LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS`` configuration parameter (with a fallback of 14 days).
+Note that certificates will be included in this summary even if they do not have any associated notifications.
+
+This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails.
+
+NOTE: At present, this summary email essentially duplicates the certificate expiration notifications, since all
+certificate expiration notifications are also sent to the security team. This issue will be fixed in the future.
+
+**Email notifications**
+
+Templates for emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
+
+The following configuration options are supported:
 
 .. data:: LEMUR_EMAIL_SENDER
     :noindex:
@@ -318,7 +435,7 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
 
         ::
 
-            LEMUR_EMAIL = 'lemur.example.com'
+            LEMUR_EMAIL = 'lemur@example.com'
 
 
 .. data:: LEMUR_SECURITY_TEAM_EMAIL
@@ -333,7 +450,7 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
 .. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS
     :noindex:
 
-        Lemur notification intervals
+        Lemur notification intervals. If unspecified, the value [30, 15, 2] is used.
 
         ::
 
@@ -348,6 +465,15 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
 
           LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
 
+.. data:: LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS
+    :noindex:
+
+       Notification interval set for CA certificate expiration notifications. If unspecified, the value [365, 180] is used (roughly one year and 6 months).
+
+       ::
+
+          LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS = [365, 180]
+
 
 Celery Options
 ---------------
@@ -593,6 +719,33 @@ For more information about how to use social logins, see: `Satellizer <https://g
 
             PING_AUTH_ENDPOINT = "https://<yourpingserver>/oauth2/authorize"
 
+.. data:: PING_USER_MEMBERSHIP_URL
+    :noindex:
+
+        An optional additional endpoint to learn membership details post the user validation.
+
+        ::
+
+            PING_USER_MEMBERSHIP_URL = "https://<yourmembershipendpoint>"
+
+.. data:: PING_USER_MEMBERSHIP_TLS_PROVIDER
+    :noindex:
+
+        A custom TLS session provider plugin name
+
+        ::
+
+            PING_USER_MEMBERSHIP_TLS_PROVIDER = "slug-name"
+
+.. data:: PING_USER_MEMBERSHIP_SERVICE
+    :noindex:
+
+        Membership service name used by PING_USER_MEMBERSHIP_TLS_PROVIDER to create a session
+
+        ::
+
+            PING_USER_MEMBERSHIP_SERVICE = "yourmembershipservice"
+
 .. data:: OAUTH2_SECRET
     :noindex:
 
@@ -704,6 +857,31 @@ ACME Plugin
         Enables delegated DNS domain validation using CNAMES.  When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges.
 
 
+The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
+account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
+
+
+.. data:: ACME_PRIVATE_KEY
+    :noindex:
+
+            This is the private key, the account was registered with (in JWK format)
+
+.. data:: ACME_REGR
+    :noindex:
+
+            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
+
+.. data:: ACME_PREFERRED_ISSUER
+    :noindex:
+
+            This is an optional parameter to indicate the preferred chain to retrieve from ACME when finalizing the order.
+            This is applicable to Let's Encrypts recent `migration <https://letsencrypt.org/certificates/>`_ to their
+            own root, where they provide two distinct certificate chains (fullchain_pem vs. alternative_fullchains_pem);
+            the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain
+            is rooted in X1 root CA.
+            Select "X1" to get the shorter chain (currently alternative), leave blank or "DST Root CA X3" for the longer chain.
+
+
 Active Directory Certificate Services Plugin
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -744,10 +922,12 @@ Active Directory Certificate Services Plugin
 
 .. data:: ADCS_START
     :noindex:
+
         Used in ADCS-Sourceplugin. Minimum id of the first certificate to be returned. ID is increased by one until ADCS_STOP. Missing cert-IDs are ignored
 
 .. data:: ADCS_STOP
     :noindex:
+
         Used for ADCS-Sourceplugin. Maximum id of the certificates returned. 
         
 
@@ -825,6 +1005,26 @@ The following parameters have to be set in the configuration files.
 
         If there is a config variable ENTRUST_PRODUCT_<upper(authority.name)> take the value as cert product name else default to "STANDARD_SSL". Refer to the API documentation for valid products names.
 
+
+.. data:: ENTRUST_CROSS_SIGNED_RSA_L1K
+    :noindex:
+
+        This is optional. Entrust provides support for cross-signed subCAS. One can set ENTRUST_CROSS_SIGNED_RSA_L1K to the respective cross-signed RSA-based subCA PEM and Lemur will replace the retrieved subCA with ENTRUST_CROSS_SIGNED_RSA_L1K.
+
+
+.. data:: ENTRUST_CROSS_SIGNED_ECC_L1F
+    :noindex:
+
+        This is optional. Entrust provides support for cross-signed subCAS. One can set ENTRUST_CROSS_SIGNED_ECC_L1F to the respective cross-signed EC-based subCA PEM and Lemur will replace the retrieved subCA with ENTRUST_CROSS_SIGNED_ECC_L1F.
+
+
+.. data:: ENTRUST_USE_DEFAULT_CLIENT_ID
+    :noindex:
+
+        If set to True, Entrust will use the primary client ID of 1, which applies to most use-case.
+        Otherwise, Entrust will first lookup the clientId before ordering the certificate.
+
+
 Verisign Issuer Plugin
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1206,23 +1406,6 @@ The following configuration properties are required to use the PowerDNS ACME Plu
 
             File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle.
 
-ACME Plugin
-~~~~~~~~~~~~
-
-The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
-account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
-
-
-.. data:: ACME_PRIVATE_KEY
-    :noindex:
-
-            This is the private key, the account was registered with (in JWK format)
-
-.. data:: ACME_REGR
-    :noindex:
-
-            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
-
 .. _CommandLineInterface:
 
 Command Line Interface
@@ -1477,7 +1660,7 @@ Slack
 
 
 AWS (Source)
-----
+------------
 
 :Authors:
     Kevin Glisson <kglisson@netflix.com>,
@@ -1490,7 +1673,7 @@ AWS (Source)
 
 
 AWS (Destination)
-----
+-----------------
 
 :Authors:
     Kevin Glisson <kglisson@netflix.com>,
@@ -1503,7 +1686,7 @@ AWS (Destination)
 
 
 AWS (SNS Notification)
------
+----------------------
 
 :Authors:
     Jasmine Schladen <jschladen@netflix.com>

docs/conf.py

@@ -32,6 +32,9 @@ if on_rtd:
     MOCK_MODULES = ["ldap"]
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
+autodoc_mock_imports = ["python-ldap", "acme", "certsrv", "dnspython3", "dyn", "factory-boy", "flask_replicated",
+                        "josepy", "logmatic", "pem"]
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -146,7 +149,7 @@ if not on_rtd:  # only import and set the theme if we're building docs locally
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
+# html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

docs/developer/index.rst

@@ -43,18 +43,35 @@ Building Documentation
 Inside the ``docs`` directory, you can run ``make`` to build the documentation.
 See ``make help`` for available options and the `Sphinx Documentation <http://sphinx-doc.org/contents.html>`_ for more information.
 
+Adding New Modules to Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When a new module is added, it will need to be added to the documentation.
+Ideally, we might rely on `sphinx-apidoc <https://www.sphinx-doc.org/en/master/man/sphinx-apidoc.html>`_ to autogenerate our documentation.
+Unfortunately, this causes some build problems.
+Instead, you'll need to add new modules by hand.
 
 Developing Against HEAD
 -----------------------
 
 We try to make it easy to get up and running in a development environment using a git checkout
-of Lemur. You'll want to make sure you have a few things on your local system first:
+of Lemur. There are two ways to run Lemur locally: directly on your development machine, or
+in a Docker container.
+
+**Running in a Docker container**
+
+Look at the `lemur-docker <https://github.com/Netflix/lemur-docker>`_ project.
+Usage instructions are self-contained in the README for that project.
+
+**Running directly on your development machine**
+
+You'll want to make sure you have a few things on your local system first:
 
 * python-dev (if you're on OS X, you already have this)
 * pip
 * virtualenv (ideally virtualenvwrapper)
 * node.js (for npm and building css/javascript)
-+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
 
 Once you've got all that, the rest is simple:
 
@@ -99,7 +116,9 @@ You'll likely want to make some changes to the default configuration (we recomme
 Running tests with Docker and docker-compose
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Alternatively you can use Docker and docker-compose for running the tests with ``docker-compose run test``.
+If you just want to run tests in a Docker container, you can use Docker and docker-compose for running the tests with ``docker-compose run test`` directly in the ``lemur`` project.
+
+(For running the Lemur service in Docker, see `lemur-docker <https://github.com/Netflix/lemur-docker>`_.)
 
 
 Coding Standards
@@ -152,7 +171,7 @@ You'll notice that the test suite is structured based on where the code lives, a
 Static Media
 ------------
 
-Lemur uses a library that compiles it's static media assets (LESS and JS files) automatically. If you're developing using
+Lemur uses a library that compiles its static media assets (LESS and JS files) automatically. If you're developing using
 runserver you'll see changes happen not only in the original files, but also the minified or processed versions of the file.
 
 If you've made changes and need to compile them by hand for any reason, you can do so by running:

docs/developer/internals/lemur.defaults.rst

@@ -0,0 +1,29 @@
+defaults Package
+================
+
+:mod:`defaults` Module
+----------------------------------------
+
+.. automodule:: lemur.defaults
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module	
+-----------------------------	
+
+.. automodule:: lemur.defaults.schemas	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module	
+---------------------------	
+
+.. automodule:: lemur.defaults.views	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.deployment.rst

@@ -0,0 +1,20 @@
+deployment Package
+===================
+
+:mod:`deployment` Module
+----------------------------------------
+
+.. automodule:: lemur.deployment
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module	
+------------------------------	
+
+.. automodule:: lemur.deployment.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.endpoints.rst

@@ -0,0 +1,56 @@
+endpoints Package
+===================
+
+:mod:`endpoints` Module
+----------------------------------------
+
+.. automodule:: lemur.endpoints
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module	
+--------------------------	
+
+.. automodule:: lemur.endpoints.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module	
+-----------------------------	
+
+.. automodule:: lemur.endpoints.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module	
+------------------------------	
+
+.. automodule:: lemur.endpoints.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module	
+------------------------------	
+
+.. automodule:: lemur.endpoints.service	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module	
+----------------------------	
+
+.. automodule:: lemur.endpoints.views	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.logs.rst

@@ -0,0 +1,47 @@
+logs Package
+===================
+
+:mod:`logs` Module
+--------------------
+
+.. automodule:: lemur.logs
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module
+------------------------------	
+
+.. automodule:: lemur.logs.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module
+------------------------------
+
+.. automodule:: lemur.logs.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.logs.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.logs.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_acme.rst

@@ -0,0 +1,83 @@
+lemur_acme package	
+=================================	
+
+:mod:`lemur_acme` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_acme
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`acme_handlers` Module
+-----------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.acme_handlers	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`challenge_types` Module
+-------------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.challenge_types	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cloudflare` Module
+-------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.cloudflare	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`dyn` Module
+------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.dyn	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+---------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.plugin	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`powerdns` Module
+-----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.powerdns	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`route53` Module
+----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.route53	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`ultradns` Module
+-----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.ultradns	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_atlas.rst

@@ -0,0 +1,20 @@
+lemur_atlas package
+==================================
+
+:mod:`lemur_atlas` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_atlas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_atlas.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_cryptography.rst

@@ -0,0 +1,20 @@
+lemur_cryptography package
+==================================
+
+:mod:`lemur_cryptography` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_cryptography
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_cryptography.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_digicert.rst

@@ -0,0 +1,20 @@
+lemur_digicert package
+==================================
+
+:mod:`lemur_digicert` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_digicert
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_digicert.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_jks.rst

@@ -0,0 +1,20 @@
+lemur_jks package
+==================================
+
+:mod:`lemur_jks` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_jks
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_jks.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_kubernetes.rst

@@ -0,0 +1,20 @@
+lemur_kubernetes package
+==================================
+
+:mod:`lemur_kubernetes` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_kubernetes
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_kubernetes.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_openssl.rst

@@ -0,0 +1,20 @@
+lemur_openssl package
+==================================
+
+:mod:`lemur_openssl` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_openssl
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_openssl.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_slack.rst

@@ -0,0 +1,20 @@
+lemur_slack package
+==================================
+
+:mod:`lemur_slack` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_slack
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_slack.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.reporting.rst

@@ -0,0 +1,38 @@
+reporting Package
+===================
+
+:mod:`reporting` Module
+----------------------------------------
+
+.. automodule:: lemur.reporting
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module
+------------------------------	
+
+.. automodule:: lemur.reporting.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.reporting.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.reporting.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.rst

@@ -28,15 +28,6 @@ lemur Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`decorators` Module
-------------------------
-
-.. automodule:: lemur.decorators
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`exceptions` Module
 ------------------------
 
@@ -108,7 +99,7 @@ Subpackages
     lemur.plugins.lemur_atlas
     lemur.plugins.lemur_cryptography
     lemur.plugins.lemur_digicert
-    lemur.plugins.lemur_java
+    lemur.plugins.lemur_jks
     lemur.plugins.lemur_kubernetes
     lemur.plugins.lemur_openssl
     lemur.plugins.lemur_slack

docs/developer/internals/lemur.sources.rst

@@ -0,0 +1,56 @@
+sources Package
+===================
+
+:mod:`sources` Module
+----------------------
+
+.. automodule:: lemur.sources
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module
+------------------------------	
+
+.. automodule:: lemur.sources.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module
+------------------------------
+
+.. automodule:: lemur.sources.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module
+------------------------------
+
+.. automodule:: lemur.sources.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.sources.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.sources.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.tests.rst

@@ -0,0 +1,11 @@
+tests Package
+=============
+
+:mod:`tests` Module
+--------------------
+
+.. automodule:: lemur.tests
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/plugins/index.rst

@@ -104,7 +104,7 @@ The `IssuerPlugin` exposes four functions functions::
 
     def create_certificate(self, csr, issuer_options):
         # requests.get('a third party')
-    def revoke_certificate(self, certificate, comments):
+    def revoke_certificate(self, certificate, reason):
         # requests.put('a third party')
     def get_ordered_certificate(self, order_id):
         # requests.get('already existing certificate')
@@ -145,8 +145,7 @@ The `IssuerPlugin` doesn't have any options like Destination, Source, and Notifi
 any fields you might need to submit a request to a third party. If there are additional options you need
 in your plugin feel free to open an issue, or look into adding additional options to issuers yourself.
 
-Asynchronous Certificates
+**Asynchronous Certificates**
-^^^^^^^^^^^^^^^^^^^^^^^^^
 An issuer may take some time to actually issue a certificate for an order.  In this case, a `PendingCertificate` is returned, which holds information to recreate a `Certificate` object at a later time.  Then, `get_ordered_certificate()` should be run periodically via `python manage.py pending_certs fetch -i all` to attempt to retrieve an ordered certificate::
 
     def get_ordered_ceriticate(self, order_id):
@@ -154,6 +153,7 @@ An issuer may take some time to actually issue a certificate for an order.  In t
         # retrieve an order, and check if there is an issued certificate attached to it
 
 `cancel_ordered_certificate()` should be implemented to allow an ordered certificate to be canceled before it is issued::
+
         def cancel_ordered_certificate(self, pending_cert, **kwargs):
             # pending_cert should contain the necessary information to match an order
             # kwargs can be given to provide information to the issuer for canceling
@@ -215,12 +215,13 @@ Notification
 ------------
 
 Lemur includes the ability to create Email notifications by **default**. These notifications
-currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and
+currently come in the form of expiration and rotation notices for all certificates, expiration notices for CA certificates,
+and ACME certificate creation failure notices. Lemur periodically checks certificate expiration dates and
 determines if a given certificate is eligible for notification. There are currently only two parameters used to
 determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
 of days the current date (UTC) is from that expiration date.
 
-Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable.
+Certificate expiration notifications can also be configured for Slack or AWS SNS. Other notifications are not configurable.
 Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email.
 
 There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for
@@ -284,6 +285,17 @@ The `ExportPlugin` object requires the implementation of one function::
     Support of various formats sometimes relies on external tools system calls. Always be mindful of sanitizing any input to these calls.
 
 
+Custom TLS Provider
+-------------------
+
+Managing TLS at the enterprise scale could be hard and often organizations offer custom wrapper implementations. It could
+be ideal to use those while making calls to internal services. The `TLSPlugin` would help to achieve this. It requires the
+implementation of one function which creates a TLS session::
+
+     def session(self, server_application):
+        # return active session
+
+
 Testing
 =======
 

docs/doing-a-release.rst

@@ -1,32 +1,65 @@
 Doing a release
 ===============
 
-Doing a release of ``lemur`` requires a few steps.
+Doing a release of ``lemur`` is now mostly automated and consists of the following steps:
 
-Bumping the version number
+* Raise a PR to add the release date and summary in the :doc:`/changelog`.
---------------------------
+* Merge above PR and create a new `Github release <https://github.com/Netflix/lemur/releaes>`_: set the tag starting with v, e.g., v0.9.0
+
+The `publish workflow <https://github.com/Netflix/lemur/actions/workflows/lemur-publish-release-pypi.yml>`_ uses the git
+tag to set the release version.
+
+The following describes the manual release steps, which is now obsolete:
+
+Manually Bumping the version number
+-----------------------------------
 
 The next step in doing a release is bumping the version number in the
 software.
 
 * Update the version number in ``lemur/__about__.py``.
 * Set the release date in the :doc:`/changelog`.
-* Do a commit indicating this.
+* Do a commit indicating this, and raise a pull request with this.
-* Send a pull request with this.
 * Wait for it to be merged.
 
-Performing the release
+Manually Performing the release
-----------------------
+-------------------------------
 
 The commit that merged the version number bump is now the official release
-commit for this release. You will need to have ``gpg`` installed and a ``gpg``
+commit for this release. You need an `API key <https://pypi.org/manage/account/#api-tokens>`_,
-key in order to do a release. Once this has happened:
+which requires permissions to maintain the Lemur `project  <https://pypi.org/project/lemur/>`_.
 
-* Run ``invoke release {version}``.
+For creating the release, follow these steps (more details `here <https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives>`_)
 
-The release should now be available on PyPI and a tag should be available in
+* Make sure you have the latest versions of setuptools and wheel installed:
+
+    ``python3 -m pip install --user --upgrade setuptools wheel``
+
+* Now run this command from the same directory where setup.py is located:
+
+    ``python3 setup.py sdist bdist_wheel``
+
+* Once completed it should generate two files in the dist directory:
+
+.. code-block:: pycon
+
+    $ ls dist/
+    lemur-0.8.0-py2.py3-none-any.whl	lemur-0.8.0.tar.gz
+
+
+* In this step, the distribution will be uploaded. You’ll need to install Twine:
+
+    ``python3 -m pip install --user --upgrade twine``
+
+* Once installed, run Twine to upload all of the archives under dist. Once installed, run Twine to upload all of the archives under dist:
+
+    ``python3 -m twine upload --repository pypi dist/*``
+
+The release should now be available on `PyPI Lemur <https://pypi.org/project/lemur/>`_ and a tag should be available in
 the repository.
 
+Make sure to also make a github `release <https://github.com/Netflix/lemur/releases>`_ which will pick up the latest version.
+
 Verifying the release
 ---------------------
 

docs/guide/index.rst

@@ -37,18 +37,20 @@ Create a New Certificate
 
 .. figure:: create_certificate.png
 
-    Enter an owner, short description and the authority you wish to issue this certificate.
+    Enter an owner, common name, short description and certificate authority you wish to issue this certificate.
-    Enter a common name into the certificate, if no validity range is selected two years is
+    Depending upon the selected CA, the UI displays default validity of the certificate. You can select different
-    the default.
+    validity by entering a custom date, if supported by the CA.
+
+    You can also add `Subject Alternate Names` or SAN for certificates that need to include more than one domains,
+    The first domain is the Common Name and all other domains are added here as DNSName entries.
 
     You can add notification options and upload the created certificate to a destination, both
     of these are editable features and can be changed after the certificate has been created.
 
 .. figure:: certificate_extensions.png
 
-    These options are typically for advanced users, the one exception is the `Subject Alternate Names` or SAN.
+    These options are typically for advanced users. Lemur creates ECC based certificate (ECCPRIME256V1 in particular)
-    For certificates that need to include more than one domains, the first domain is the Common Name and all
+    by default. One can change the key type using the dropdown option listed here.
-    other domains are added here as DNSName entries.
 
 
 Import an Existing Certificate
@@ -58,11 +60,12 @@ Import an Existing Certificate
 
     Enter an owner, short description and public certificate. If there are intermediates and private keys
     Lemur will track them just as it does if the certificate were created through Lemur. Lemur generates
-    a certificate name but you can override that by passing a value to the `Custom Name` field.
+    a certificate name but you can override that by passing a value to the `Custom Certificate Name` field.
 
     You can add notification options and upload the created certificate to a destination, both
     of these are editable features and can be changed after the certificate has been created.
 
+.. _CreateANewUser:
 
 Create a New User
 ~~~~~~~~~~~~~~~~~

docs/license/index.rst

@@ -18,3 +18,4 @@ Lemur License
 -------------
 
 .. include:: ../../LICENSE
+    :literal:

docs/production/index.rst

@@ -323,9 +323,9 @@ Periodic Tasks
 Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
 celery tasks or cron jobs that run these commands.
 
-There are currently three commands that could/should be run on a periodic basis:
+The following commands that could/should be run on a periodic basis:
 
-- `notify`
+- `notify expirations` `notify authority_expirations`, and `notify security_expiration_summary` (see :ref:`NotificationOptions` for configuration info)
 - `check_revoked`
 - `sync`
 
@@ -334,13 +334,16 @@ If you are using LetsEncrypt, you must also run the following:
 - `fetch_all_pending_acme_certs`
 - `remove_old_acme_certs`
 
-How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
+How often you run these commands is largely up to the user. `notify` should be run once a day (more often will result in
+duplicate notifications). `check_revoked` is typically run at least once a day.
 `sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
 `remove_old_acme_certs` can be ran more rarely, such as once every week.
 
 Example cron entries::
 
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify security_expiration_summary
     */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
 
@@ -382,6 +385,27 @@ Example Celery configuration (To be placed in your configuration file)::
                 'expires': 180
             },
             'schedule': crontab(hour="*"),
+        },
+        'notify_expirations': {
+            'task': 'lemur.common.celery.notify_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        },
+        'notify_authority_expirations': {
+            'task': 'lemur.common.celery.notify_authority_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        },
+        'send_security_expiration_summary': {
+            'task': 'lemur.common.celery.send_security_expiration_summary',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
         }
     }
 
@@ -415,8 +439,8 @@ And the worker can be started with desired options such as the following::
 
 supervisor or systemd configurations should be created for these in production environments as appropriate.
 
-Add support for LetsEncrypt
+Add support for LetsEncrypt/ACME
-===========================
+================================
 
 LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
 for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
@@ -424,7 +448,10 @@ LetsEncrypt requires that we prove ownership of a domain before we're able to is
 time we want a certificate.
 
 The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
-through the creation of DNS TXT records.
+through the creation of DNS TXT records as well as HTTP validation, reusing the destination concept.
+
+ACME DNS Challenge
+------------------
 
 In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
 token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
@@ -462,6 +489,24 @@ possible. To enable this functionality, periodically (or through Cron/Celery) ru
 This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
 Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.
 
+ACME HTTP Challenge
+-------------------
+
+The flow for requesting a certificate using the HTTP challenge is not that different from the one described for the DNS
+challenge. The only difference is, that instead of creating a DNS TXT record, a file is uploaded to a Webserver which
+serves the file at `http://<domain>/.well-known/acme-challenge/<token>`
+
+Currently the HTTP challenge also works without Celery, since it's done while creating the certificate, and doesn't
+rely on celery to create the DNS record. This will change when we implement mix & match of acme challenge types.
+
+To create a HTTP compatible Authority, you first need to create a new destination that will be used to deploy the
+challenge token. Visit `Admin` -> `Destination` and click `Create`. The path you provide for the destination needs to
+be the exact path that is called when the ACME providers calls `http://<domain>/.well-known/acme-challenge/`. The
+token part will be added dynamically by the acme_upload.
+Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge.
+
+Afterwards you can create a new certificate authority as described in the DNS challenge, but need to choose
+`Acme HTTP-01` as the plugin type, and then the destination you created beforehand.
 
 LetsEncrypt: pinning to cross-signed ICA
 ----------------------------------------
@@ -555,3 +600,118 @@ Using `python-jwt` converting an existing private key in PEM format is quite eas
     {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/<ACCOUNT_NUMBER>"}
 
 The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key.
+
+OpenSSH
+=======
+
+OpenSSH (also known as OpenBSD Secure Shell) is a suite of secure networking utilities based on the Secure Shell (SSH) protocol, which provides a secure channel over an unsecured network in a client–server architecture.
+
+Using a PKI with OpenSSH means you can sign a key for a user and it can log into any server that trust the CA.
+
+Using a CA avoids TOFU or synchronize a list of server public keys to `known_hosts` files.
+
+This is useful when you're managing large number of machines or for an immutable infrastructure.
+
+Add first OpenSSH authority
+---------------------------
+
+To start issuing OpenSSH, you need to create an OpenSSH authority. To do this, visit
+Authorities -> Create. Set the applicable attributes:
+
+- Name : OpenSSH
+- Common Name: example.net
+
+Then click "More Options" and change the plugin value to "OpenSSH".
+
+Just click to "Create" button to add this authority.
+
+.. note:: OpenSSH do not support sub CA feature.
+
+Add a server certificate
+-------------------------
+
+Now visit Certificates -> Create to add a server certificate. Set the applicable attributes:
+
+- Common Name: server.example.net
+
+Then click "More Options" and set the Certificate Template to "Server Certificate".
+
+This step is important, a certificat for a server and for a client is not exactly the same thing.
+In this case "Common Name" and all Subject Alternate Names with type DNSName will be added in the certificate.
+
+Finally click on "Create" button.
+
+Add a client certificate
+------------------------
+
+Now visit Certificates -> Create to add a client certificate. Set the applicable attributes:
+
+- Common Name: example.net
+
+Then click "More Options" and set the Certificate Template to "Client Certificate".
+
+In this case the name of the creator is used as principal (in this documentation we assume that this certificate is created by the user "lemur").
+
+Finally click on "Create" button.
+
+Configure OpenSSH server
+------------------------
+
+Connect to the server.example.net server to correctly configure the OpenSSH server with the CA created previously.
+
+First of all add the CA chain, private and public certificates:
+
+- Create file `/etc/ssh/ca.pub` and copy the "CHAIN" content of the *server certificate* (everything in one line).
+- Create file `/etc/ssh/ssh_host_key` and copy "PRIVATE KEY" content.
+- Create file `/etc/ssh/ssh_host_key.pub` and copy "PUBLIC CERTIFICATE" content (everything in one line).
+
+Set the appropriate right:
+
+.. code-block:: bash
+
+    chmod 600 /etc/ssh/ca.pub /etc/ssh/ssh_host_key
+    chmod 644 /etc/ssh/ssh_host_key.pub
+    chown root: /etc/ssh/ca.pub /etc/ssh/ssh_host_key /etc/ssh/ssh_host_key.pub
+
+Then change OpenSSH server configuration to use these files. Edit `/etc/ssh/sshd_config` and add::
+
+    TrustedUserCAKeys /etc/ssh/ca.pub
+    HostKey /etc/ssh/ssh_host_key
+    HostCertificate /etc/ssh/ssh_host_key.pub
+
+You can remove all other `HostKey` lines.
+
+Finally restart OpenSSH.
+
+Configure the OpenSSH client
+----------------------------
+
+Now you can configure the user's computer.
+
+First of all add private and public certificates:
+
+- Create file `~/.ssh/key` and copy "PRIVATE KEY" content.
+- Create file `~/.ssh/key.pub` and copy "PUBLIC CERTIFICATE" content of the *client certicate* (everything in one line).
+
+Set the appropriate right:
+
+.. code-block:: bash
+
+    chmod 600 ~/.ssh/key.pub ~/.ssh/key
+
+To avoid TOFU, edite the `~/.ssh/known_hosts` file and add a new line (all in one line):
+
+- @cert-authority \*example.net
+- the "CHAIN" content
+
+Now you can connect to server with (here 'lemur' is the principal name and must exists on the server):
+
+.. code-block:: bash
+
+    ssh lemur@server.example.net -i ~/.ssh/key
+
+With this configuration you don't have any line like::
+
+    Warning: Permanently added 'server.example.net,192.168.0.1' (RSA) to the list of known hosts.
+
+And you don't have to enter any password.

docs/quickstart/index.rst

@@ -1,9 +1,10 @@
 Quickstart
 **********
 
-This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.  This guide assumes a clean Ubuntu 14.04 instance, commands may differ based on the OS and configuration being used.
+This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.
+This guide assumes a clean Ubuntu 18.04/20.04 instance, commands may differ based on the OS and configuration being used.
 
-Pressed for time? See the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
+For a quicker alternative, see the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
 
 
 Dependencies
@@ -11,12 +12,14 @@ Dependencies
 
 Some basic prerequisites which you'll need in order to run Lemur:
 
-* A UNIX-based operating system (we test on Ubuntu, develop on OS X)
+* A UNIX-based operating system (we test on Ubuntu, develop on macOS)
 * Python 3.7 or greater
 * PostgreSQL 9.4 or greater
 * Nginx
+* Node v10.x (LTS)
 
-.. note:: Lemur was built with in AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
+.. note:: Ubuntu 18.04 supports by default Python 3.6.x and Node v8.x
+.. note:: Lemur was built with AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
 
 
 Installing Build Dependencies
@@ -27,7 +30,7 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack
 .. code-block:: bash
 
     sudo apt-get update
-    sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql
+    sudo apt-get install nodejs npm python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor postgresql
 
 .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver.  npm is needed if you're installing Lemur from the source (e.g., from git).
 
@@ -130,7 +133,7 @@ Once created, you will need to update the configuration file with information ab
     vi ~/.lemur/lemur.conf.py
 
 .. note:: If you are unfamiliar with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
-      ``postgresql://userame:password@<database-fqdn>:<database-port>/<database-name>``
+      ``postgresql://username:password@<database-fqdn>:<database-port>/<database-name>``
 
 Before Lemur will run you need to fill in a few required variables in the configuration file:
 
@@ -145,7 +148,7 @@ Before Lemur will run you need to fill in a few required variables in the config
     LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
 
 Set Up Postgres
---------------
+---------------
 
 For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
 
@@ -183,11 +186,12 @@ In addition to creating a new user, Lemur also creates a few default email notif
 Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user.
 
 .. code-block:: bash
+
     sudo -u postgres -i
     psql
     postgres=# ALTER USER lemur WITH SUPERUSER
 
-Additional notifications can be created through the UI or API.  See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+Additional notifications can be created through the UI or API.  See :ref:`Notification Options <NotificationOptions>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 **Make note of the password used as this will be used during first login to the Lemur UI.**
 
@@ -199,15 +203,16 @@ Additional notifications can be created through the UI or API.  See :ref:`Creati
 .. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now.
 
 .. code-block:: bash
+
     sudo -u postgres -i
     psql
     postgres=# ALTER USER lemur WITH NOSUPERUSER
 
 
-.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating a New User <CreateANewUser>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 Set Up a Reverse Proxy
----------------------
+----------------------
 
 By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
 
@@ -323,6 +328,12 @@ unlock
 Decrypts sensitive key material - used to decrypt the secrets stored in source during deployment.
 
 
+Automated celery tasks
+~~~~~~~~~~~~~~~~~~~~~~
+
+Please refer to :ref:`Periodic Tasks <PeriodicTasks>` to learn more about task scheduling in Lemur.
+
+
 What's Next?
 ------------
 

docs/security.rst

@@ -22,7 +22,7 @@ Supported Versions
 ------------------
 
 At any given time, we will provide security support for the `master`_ branch
-as well as the 2 most recent releases.
+as well as the most recent release.
 
 Disclosure Process
 ------------------
@@ -30,20 +30,15 @@ Disclosure Process
 Our process for taking a security issue from private discussion to public
 disclosure involves multiple steps.
 
-Approximately one week before full public disclosure, we will send advance
+Approximately one week before full public disclosure, we will provide advanced notification that a security issue exists. Depending on the severity of the issue, we may choose to either send a targeted email to known Lemur users and contributors or post an issue to the Lemur repository.  In either case, the notification should contain the following.
-notification of the issue to a list of people and organizations, primarily
-composed of operating-system vendors and other distributors of
-``lemur``.  This notification will consist of an email message
-containing:
 
-* A full description of the issue and the affected versions of
+* A description of the potential impact
-  ``lemur``.
+* The affected versions of ``lemur``.
 * The steps we will be taking to remedy the issue.
-* The patches, if any, that will be applied to ``lemur``.
 * The date on which the ``lemur`` team will apply these patches, issue
   new releases, and publicly disclose the issue.
 
-Simultaneously, the reporter of the issue will receive notification of the date
+If the issue was disclosed to us, the reporter will receive notification of the date
 on which we plan to make the issue public.
 
 On the day of disclosure, we will take the following steps:
@@ -52,7 +47,7 @@ On the day of disclosure, we will take the following steps:
   messages for these patches will indicate that they are for security issues,
   but will not describe the issue in any detail; instead, they will warn of
   upcoming disclosure.
-* Issue the relevant releases.
+* Issue an updated release.
 
 If a reported issue is believed to be particularly time-sensitive – due to a
 known exploit in the wild, for example – the time between advance notification

gulp/build.js

@@ -7,7 +7,7 @@ var gulp = require('gulp'),
   gulpif = require('gulp-if'),
   gutil = require('gulp-util'),
   foreach = require('gulp-foreach'),
-  path =require('path'),
+  path = require('path'),
   merge = require('merge-stream'),
   del = require('del'),
   size = require('gulp-size'),
@@ -21,7 +21,6 @@ var gulp = require('gulp'),
   useref = require('gulp-useref'),
   filter = require('gulp-filter'),
   rev = require('gulp-rev'),
-  revReplace = require('gulp-rev-replace'),
   imagemin = require('gulp-imagemin'),
   minifyHtml = require('gulp-minify-html'),
   bowerFiles = require('main-bower-files'),
@@ -29,52 +28,77 @@ var gulp = require('gulp'),
   replace = require('gulp-replace'),
   argv = require('yargs').argv;
 
-gulp.task('default', ['clean'], function () {
+// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
-  gulp.start('fonts', 'styles');
+function escapeRegExp(string) {
+  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
+}
+
+function replaceAll(string, find, replace) {
+  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
+}
+
+function stringSrc(filename, string) {
+  let src = require('stream').Readable({objectMode: true});
+  src._read = function () {
+    this.push(new gutil.File({cwd: '', base: '', path: filename, contents: Buffer.from(string)}));
+    this.push(null);
+  };
+  return src;
+}
+
+gulp.task('clean', function (done) {
+  del(['.tmp', 'lemur/static/dist'], done);
+  done();
 });
 
-gulp.task('clean', function (cb) {
+gulp.task('default', gulp.series(['clean'], function () {
-  del(['.tmp', 'lemur/static/dist'], cb);
+  gulp.start('fonts', 'styles');
-});
+}));
 
 gulp.task('test', function (done) {
   new karma.Server({
     configFile: __dirname + '/karma.conf.js',
     singleRun: true
-  }, function() {
+  }, function (err) {
+    if (err === 0) {
       done();
+    } else {
+      // if karma server failed to start raise error
+      done(new gutil.PluginError('karma', {
+        message: 'Karma Tests failed'
+      }));
+    }
   }).start();
 });
 
 gulp.task('dev:fonts', function () {
-  var fileList = [
+  let fileList = [
     'bower_components/bootstrap/dist/fonts/*',
     'bower_components/fontawesome/fonts/*'
   ];
 
   return gulp.src(fileList)
-    .pipe(gulp.dest('.tmp/fonts'));
+    .pipe(gulp.dest('.tmp/fonts')); // returns a stream making it async
 });
 
 gulp.task('dev:styles', function () {
-  var baseContent = '@import "bower_components/bootstrap/less/bootstrap.less";@import "bower_components/bootswatch/$theme$/variables.less";@import "bower_components/bootswatch/$theme$/bootswatch.less";@import "bower_components/bootstrap/less/utilities.less";';
+  let baseContent = '@import "bower_components/bootstrap/less/bootstrap.less";@import "bower_components/bootswatch/$theme$/variables.less";@import "bower_components/bootswatch/$theme$/bootswatch.less";@import "bower_components/bootstrap/less/utilities.less";';
-  var isBootswatchFile = function (file) {
+  let isBootswatchFile = function (file) {
 
-    var suffix = 'bootswatch.less';
+    let suffix = 'bootswatch.less';
     return file.path.indexOf(suffix, file.path.length - suffix.length) !== -1;
   };
 
-  var isBootstrapFile = function (file) {
+  let isBootstrapFile = function (file) {
-    var suffix = 'bootstrap-',
+    let suffix = 'bootstrap-',
       fileName = path.basename(file.path);
 
     return fileName.indexOf(suffix) === 0;
   };
 
-  var fileList = [
+  let fileList = [
     'bower_components/bootswatch/sandstone/bootswatch.less',
     'bower_components/fontawesome/css/font-awesome.css',
-    'bower_components/angular-spinkit/src/angular-spinkit.css',
     'bower_components/angular-chart.js/dist/angular-chart.css',
     'bower_components/angular-loading-bar/src/loading-bar.css',
     'bower_components/angular-ui-switch/angular-ui-switch.css',
@@ -87,20 +111,18 @@ gulp.task('dev:styles', function () {
 
   return gulp.src(fileList)
     .pipe(gulpif(isBootswatchFile, foreach(function (stream, file) {
-      var themeName = path.basename(path.dirname(file.path)),
+      let themeName = path.basename(path.dirname(file.path)),
-        content = replaceAll(baseContent, '$theme$', themeName),
+        content = replaceAll(baseContent, '$theme$', themeName);
-        file2 = string_src('bootstrap-' +  themeName + '.less', content);
+      return stringSrc('bootstrap-' + themeName + '.less', content);
-
-      return file2;
     })))
     .pipe(less())
     .pipe(gulpif(isBootstrapFile, foreach(function (stream, file) {
-      var fileName = path.basename(file.path),
+      let fileName = path.basename(file.path),
         themeName = fileName.substring(fileName.indexOf('-') + 1, fileName.indexOf('.'));
 
       // http://stackoverflow.com/questions/21719833/gulp-how-to-add-src-files-in-the-middle-of-a-pipe
       // https://github.com/gulpjs/gulp/blob/master/docs/recipes/using-multiple-sources-in-one-task.md
-      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css']))
+      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css'], {allowEmpty: true}))
         .pipe(concat('style-' + themeName + '.css'));
     })))
     .pipe(plumber())
@@ -111,24 +133,6 @@ gulp.task('dev:styles', function () {
     .pipe(size());
 });
 
-// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
-function escapeRegExp(string) {
-  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
-}
-
-function replaceAll(string, find, replace) {
-  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
-}
-
-function string_src(filename, string) {
-  var src = require('stream').Readable({ objectMode: true });
-  src._read = function () {
-    this.push(new gutil.File({ cwd: '', base: '', path: filename, contents: new Buffer(string) }));
-    this.push(null);
-  };
-  return src;
-}
-
 gulp.task('dev:scripts', function () {
   return gulp.src(['lemur/static/app/angular/**/*.js'])
     .pipe(jshint())
@@ -144,7 +148,7 @@ gulp.task('build:extras', function () {
 function injectHtml(isDev) {
   return gulp.src('lemur/static/app/index.html')
     .pipe(
-    inject(gulp.src(bowerFiles({ base: 'app' })), {
+      inject(gulp.src(bowerFiles({base: 'app'})), {
         starttag: '<!-- inject:bower:{{ext}} -->',
         addRootSlash: false,
         ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
@@ -162,7 +166,7 @@ function injectHtml(isDev) {
     }))
     .pipe(
       gulpif(!isDev,
-      inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js'), {
+        inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js', {allowEmpty: true}), {
           starttag: '<!-- inject:ngviews -->',
           addRootSlash: false
         })
@@ -170,13 +174,9 @@ function injectHtml(isDev) {
     ).pipe(gulp.dest('.tmp/'));
 }
 
-gulp.task('dev:inject', ['dev:styles', 'dev:scripts'], function () {
+gulp.task('dev:inject', gulp.series(['dev:styles', 'dev:scripts'], function () {
   return injectHtml(true);
-});
+}));
-
-gulp.task('build:inject', ['dev:styles', 'dev:scripts', 'build:ngviews'], function () {
-  return injectHtml(false);
-});
 
 gulp.task('build:ngviews', function () {
   return gulp.src(['lemur/static/app/angular/**/*.html'])
@@ -189,9 +189,13 @@ gulp.task('build:ngviews', function () {
     .pipe(size());
 });
 
-gulp.task('build:html', ['dev:styles', 'dev:scripts', 'build:ngviews', 'build:inject'], function () {
+gulp.task('build:inject', gulp.series(['dev:styles', 'dev:scripts', 'build:ngviews'], function () {
-  var jsFilter = filter(['**/*.js'], {'restore': true});
+  return injectHtml(false);
-  var cssFilter = filter(['**/*.css'], {'restore': true});
+}));
+
+gulp.task('build:html', gulp.series(['build:inject'], function () {
+  let jsFilter = filter(['**/*.js'], {'restore': true});
+  let cssFilter = filter(['**/*.css'], {'restore': true});
 
   return gulp.src('.tmp/index.html')
     .pipe(jsFilter)
@@ -203,12 +207,12 @@ gulp.task('build:html', ['dev:styles', 'dev:scripts', 'build:ngviews', 'build:in
     .pipe(useref())
     .pipe(gulp.dest('lemur/static/dist'))
     .pipe(size());
-});
+}));
 
-gulp.task('build:fonts', ['dev:fonts'], function () {
+gulp.task('build:fonts', gulp.series(['dev:fonts'], function () {
   return gulp.src('.tmp/fonts/**/*')
     .pipe(gulp.dest('lemur/static/dist/fonts'));
-});
+}));
 
 gulp.task('build:images', function () {
   return gulp.src('lemur/static/app/images/**/*')
@@ -230,35 +234,28 @@ gulp.task('package:strip', function () {
     .pipe(size());
 });
 
-gulp.task('addUrlContextPath',['addUrlContextPath:revreplace'], function(){
+gulp.task('addUrlContextPath:revision', function () {
-  var urlContextPathExists = argv.urlContextPath ? true : false;
+  return gulp.src(['lemur/static/dist/**/*.css', 'lemur/static/dist/**/*.js'])
-  ['lemur/static/dist/scripts/main*.js',
-  'lemur/static/dist/angular/**/*.html']
-  .forEach(function(file){
-    return gulp.src(file)
-      .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
-      .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/')))
-      .pipe(gulp.dest(function(file){
-        return file.base;
-      }))
-  })
-});
-
-gulp.task('addUrlContextPath:revision', function(){
-  return gulp.src(['lemur/static/dist/**/*.css','lemur/static/dist/**/*.js'])
     .pipe(rev())
     .pipe(gulp.dest('lemur/static/dist'))
     .pipe(rev.manifest())
-    .pipe(gulp.dest('lemur/static/dist'))
-})
-
-gulp.task('addUrlContextPath:revreplace', ['addUrlContextPath:revision'], function(){
-  var manifest = gulp.src("lemur/static/dist/rev-manifest.json");
-  var urlContextPathExists = argv.urlContextPath ? true : false;
-  return gulp.src( "lemur/static/dist/index.html")
     .pipe(gulp.dest('lemur/static/dist'));
-})
+});
 
+gulp.task('addUrlContextPath:revreplace', gulp.series(['addUrlContextPath:revision'], function () {
+  return gulp.src('lemur/static/dist/index.html')
+    .pipe(gulp.dest('lemur/static/dist'));
+}));
 
-gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']);
+gulp.task('addUrlContextPath', gulp.series(['addUrlContextPath:revreplace'], function () {
-gulp.task('package', ['addUrlContextPath', 'package:strip']);
+  let urlContextPathExists = !!argv.urlContextPath;
+  return gulp.src(['lemur/static/dist/scripts/main*.js', 'lemur/static/dist/angular/**/*.html'])
+    .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
+    .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/')))
+    .pipe(gulp.dest(function (file) {
+      return file.base;
+    }));
+}));
+
+gulp.task('build', gulp.series(['build:images', 'build:fonts', 'build:html', 'build:extras']));
+gulp.task('package', gulp.series(['addUrlContextPath', 'package:strip']));

gulp/karma.conf.js

@@ -1,12 +1,14 @@
 // Contents of: config/karma.conf.js
+'use strict';
+
 module.exports = function (config) {
   config.set({
-    basePath : '../',
+    basePath: '../',
 
     // Fix for "JASMINE is not supported anymore" warning
-    frameworks : ["jasmine"],
+    frameworks: ['jasmine'],
 
-    files : [
+    files: [
       'app/lib/angular/angular.js',
       'app/lib/angular/angular-*.js',
       'test/lib/angular/angular-mocks.js',
@@ -14,14 +16,22 @@ module.exports = function (config) {
       'test/unit/**/*.js'
     ],
 
-    autoWatch : true,
+    autoWatch: true,
 
-    browsers : ['Chrome'],
+    browsers: [process.env.TRAVIS ? 'Chrome_travis_ci' : 'Chrome'],
+    customLaunchers: {
+      'Chrome_travis_ci': {
+        base: 'Chrome',
+        flags: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu',],
+      },
+    },
 
-    junitReporter : {
+    junitReporter: {
-      outputFile : 'test_out/unit.xml',
+      outputFile: 'test_out/unit.xml',
-      suite      : 'unit'
+      suite: 'unit'
       //...
-    }
+    },
+
+    failOnEmptyTestSuite: false,
   });
 };

gulp/server.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var gulp = require('gulp');
+const watch = require('./watch')
 
 var browserSync = require('browser-sync');
 var httpProxy = require('http-proxy');
@@ -38,7 +39,7 @@ function browserSyncInit(baseDir, files, browser) {
 
 }
 
-gulp.task('serve', ['watch'], function () {
+gulp.task('serve', gulp.series(['watch'], function (done) {
   browserSyncInit([
     '.tmp',
     'lemur/static/app'
@@ -51,9 +52,12 @@ gulp.task('serve', ['watch'], function () {
     'lemur/static/app/angular/**/*',
     'lemur/static/app/index.html'
   ]);
-});
+
+  done();
+}));
 
 
-gulp.task('serve:dist', ['build'], function () {
+gulp.task('serve:dist', gulp.series(['build'], function (done) {
   browserSyncInit('lemur/static/dist');
-});
+  done();
+}));

gulp/watch.js

@@ -3,10 +3,13 @@
 var gulp = require('gulp');
 
 
-gulp.task('watch', ['dev:styles', 'dev:scripts', 'dev:inject', 'dev:fonts'] ,function () {
+const watch = gulp.task('watch', gulp.series(['dev:inject', 'dev:fonts'] ,function (done) {
-  gulp.watch('app/styles/**/*.less', ['dev:styles']);
+  gulp.watch('app/styles/**/*.less', gulp.series('dev:styles'));
-  gulp.watch('app/styles/**/*.css', ['dev:styles']);
+  gulp.watch('app/styles/**/*.css', gulp.series('dev:styles'));
-  gulp.watch('app/**/*.js', ['dev:scripts']);
+  gulp.watch('app/**/*.js', gulp.series('dev:scripts'));
-  gulp.watch('app/images/**/*', ['build:images']);
+  gulp.watch('app/images/**/*', gulp.series('build:images'));
-  gulp.watch('bower.json', ['dev:inject']);
+  gulp.watch('bower.json', gulp.series('dev:inject'));
-});
+  done();
+}));
+
+module.exports = {watch:watch}

lemur/__about__.py

@@ -15,7 +15,7 @@ __title__ = "lemur"
 __summary__ = "Certificate management and orchestration service"
 __uri__ = "https://github.com/Netflix/lemur"
 
-__version__ = "0.8.0"
+__version__ = "develop"
 
 __author__ = "The Lemur developers"
 __email__ = "security@netflix.com"

lemur/api_keys/service.py

@@ -7,6 +7,7 @@
 """
 from lemur import database
 from lemur.api_keys.models import ApiKey
+from lemur.logs import service as log_service
 
 
 def get(aid):
@@ -24,6 +25,7 @@ def delete(access_key):
     :param access_key:
     :return:
     """
+    log_service.audit_log("delete_api_key", access_key.name, "Deleting the API key")
     database.delete(access_key)
 
 
@@ -34,8 +36,9 @@ def revoke(aid):
     :return:
     """
     api_key = get(aid)
-    setattr(api_key, "revoked", False)
+    setattr(api_key, "revoked", True)
 
+    log_service.audit_log("revoke_api_key", api_key.name, "Revoking API key")
     return database.update(api_key)
 
 
@@ -55,6 +58,9 @@ def create(**kwargs):
     :return:
     """
     api_key = ApiKey(**kwargs)
+    # this logs only metadata about the api key
+    log_service.audit_log("create_api_key", api_key.name, f"Creating the API key {api_key}")
+
     database.create(api_key)
     return api_key
 
@@ -69,6 +75,7 @@ def update(api_key, **kwargs):
     for key, value in kwargs.items():
         setattr(api_key, key, value)
 
+    log_service.audit_log("update_api_key", api_key.name, f"Update summary - {kwargs}")
     return database.update(api_key)
 
 

lemur/api_keys/views.py

@@ -105,6 +105,7 @@ class ApiKeyList(AuthenticatedResource):
               POST /keys HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "name": "my custom name",
@@ -225,6 +226,7 @@ class ApiKeyUserList(AuthenticatedResource):
               POST /users/1/keys HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "name": "my custom name"
@@ -332,6 +334,7 @@ class ApiKeys(AuthenticatedResource):
               PUT /keys/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                   "name": "new_name",
@@ -474,6 +477,7 @@ class UserApiKeys(AuthenticatedResource):
               PUT /users/1/keys/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                   "name": "new_name",

lemur/auth/ldap.py

@@ -211,7 +211,7 @@ class LdapPrincipal:
             for group in lgroups:
                 (dn, values) = group
                 if type(values) == dict:
-                    self.ldap_groups.append(values["cn"][0].decode("ascii"))
+                    self.ldap_groups.append(values["cn"][0].decode("utf-8"))
         else:
             lgroups = self.ldap_client.search_s(
                 self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs

lemur/auth/service.py

@@ -75,9 +75,9 @@ def create_token(user, aid=None, ttl=None):
         if ttl == -1:
             del payload["exp"]
         else:
-            payload["exp"] = ttl
+            payload["exp"] = datetime.utcnow() + timedelta(days=ttl)
     token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"])
-    return token.decode("unicode_escape")
+    return token
 
 
 def login_required(f):
@@ -116,9 +116,8 @@ def login_required(f):
                 return dict(message="Token has been revoked"), 403
             if access_key.ttl != -1:
                 current_time = datetime.utcnow()
-                expired_time = datetime.fromtimestamp(
+                # API key uses days
-                    access_key.issued_at + access_key.ttl
+                expired_time = datetime.fromtimestamp(access_key.issued_at) + timedelta(days=access_key.ttl)
-                )
                 if current_time >= expired_time:
                     return dict(message="Token has expired"), 403
 

lemur/auth/views.py

@@ -5,6 +5,8 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import json
+
 import jwt
 import base64
 import requests
@@ -20,9 +22,10 @@ from lemur.common.utils import get_psuedo_random_string
 
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
+from lemur.logs import service as log_service
 from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key
 from lemur.auth import ldap
-
+from lemur.plugins.base import plugins
 
 mod = Blueprint("auth", __name__)
 api = Api(mod)
@@ -137,6 +140,47 @@ def retrieve_user(user_api_url, access_token):
     return user, profile
 
 
+def retrieve_user_memberships(user_api_url, user_membership_api_url, access_token):
+    user, profile = retrieve_user(user_api_url, access_token)
+
+    if user_membership_api_url is None:
+        return user, profile
+    """
+    Potentially, below code can be made more generic i.e., plugin driven. Unaware of the usage of this
+    code across the community, current implementation is config driven. Without user_membership_api_url
+    configured, it is backward compatible.
+    """
+    tls_provider = plugins.get(current_app.config.get("PING_USER_MEMBERSHIP_TLS_PROVIDER"))
+
+    # put user id in url
+    user_membership_api_url = user_membership_api_url.replace("%user_id%", profile["userId"])
+
+    session = tls_provider.session(current_app.config.get("PING_USER_MEMBERSHIP_SERVICE"))
+    headers = {"Content-Type": "application/json"}
+    data = {"relation": "DIRECT_ONLY", "groupFilter": {"type": "GOOGLE"}, "size": 500}
+    user_membership = {"email": profile["email"],
+                       "thumbnailPhotoUrl": profile["thumbnailPhotoUrl"],
+                       "googleGroups": []}
+    while True:
+        # retrieve information about the current user memberships
+        r = session.post(user_membership_api_url, data=json.dumps(data), headers=headers)
+
+        if r.status_code == 200:
+            response = r.json()
+            membership_details = response["data"]
+            for membership in membership_details:
+                user_membership["googleGroups"].append(membership["membership"]["name"])
+
+            if "nextPageToken" in response and response["nextPageToken"]:
+                data["nextPageToken"] = response["nextPageToken"]
+            else:
+                break
+        else:
+            current_app.logger.error(f"Response Code:{r.status_code} {r.text}")
+            break
+    return user, user_membership
+
+
 def create_user_roles(profile):
     """Creates new roles based on profile information.
 
@@ -155,7 +199,7 @@ def create_user_roles(profile):
                     description="This is a google group based role created by Lemur",
                     third_party=True,
                 )
-            if not role.third_party:
+            if (group != 'admin') and (not role.third_party):
                 role = role_service.set_third_party(role.id, third_party_status=True)
             roles.append(role)
     else:
@@ -198,7 +242,6 @@ def update_user(user, profile, roles):
     :param profile:
     :param roles:
     """
-
     # if we get an sso user create them an account
     if not user:
         user = user_service.create(
@@ -212,10 +255,16 @@ def update_user(user, profile, roles):
 
     else:
         # we add 'lemur' specific roles, so they do not get marked as removed
+        removed_roles = []
         for ur in user.roles:
             if not ur.third_party:
                 roles.append(ur)
+            elif ur not in roles:
+                # This is a role assigned in lemur, but not returned by sso during current login
+                removed_roles.append(ur.name)
 
+        if removed_roles:
+            log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
         # update any changes to the user
         user_service.update(
             user.id,
@@ -262,6 +311,7 @@ class Login(Resource):
               POST /auth/login HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "username": "test",
@@ -368,7 +418,6 @@ class Ping(Resource):
 
         # you can either discover these dynamically or simply configure them
         access_token_url = current_app.config.get("PING_ACCESS_TOKEN_URL")
-        user_api_url = current_app.config.get("PING_USER_API_URL")
 
         secret = current_app.config.get("PING_SECRET")
 
@@ -384,7 +433,12 @@ class Ping(Resource):
         error_code = validate_id_token(id_token, args["clientId"], jwks_url)
         if error_code:
             return error_code
-        user, profile = retrieve_user(user_api_url, access_token)
+
+        user, profile = retrieve_user_memberships(
+            current_app.config.get("PING_USER_API_URL"),
+            current_app.config.get("PING_USER_MEMBERSHIP_URL"),
+            access_token
+        )
         roles = create_user_roles(profile)
         update_user(user, profile, roles)
 

lemur/authorities/schemas.py

@@ -7,7 +7,7 @@
 """
 from flask import current_app
 
-from marshmallow import fields, validates_schema, pre_load
+from marshmallow import fields, validates_schema, pre_load, post_dump
 from marshmallow import validate
 from marshmallow.exceptions import ValidationError
 
@@ -24,6 +24,7 @@ from lemur.common import validators, missing
 
 from lemur.common.fields import ArrowDateTime
 from lemur.constants import CERTIFICATE_KEY_TYPES
+from lemur.plugins.base import plugins
 
 
 class AuthorityInputSchema(LemurInputSchema):
@@ -129,6 +130,12 @@ class AuthorityOutputSchema(LemurOutputSchema):
     default_validity_days = fields.Integer()
     authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema)
 
+    @post_dump
+    def handle_auth_certificate(self, cert):
+        # Plugins may need to modify the cert object before returning it to the user
+        plugin = plugins.get(cert['plugin']['slug'])
+        plugin.wrap_auth_certificate(cert['authority_certificate'])
+
 
 class AuthorityNestedOutputSchema(LemurOutputSchema):
     __envelope__ = False

lemur/authorities/service.py

@@ -117,6 +117,12 @@ def create(**kwargs):
     """
     Creates a new authority.
     """
+    ca_name = kwargs.get("name")
+    if get_by_name(ca_name):
+        raise Exception(f"Authority with name {ca_name} already exists")
+    if role_service.get_by_name(f"{ca_name}_admin") or role_service.get_by_name(f"{ca_name}_operator"):
+        raise Exception(f"Admin and/or operator roles for authority {ca_name} already exist")
+
     body, private_key, chain, roles = mint(**kwargs)
 
     kwargs["creator"].roles = list(set(list(kwargs["creator"].roles) + roles))

lemur/authorities/views.py

@@ -130,6 +130,7 @@ class AuthoritiesList(AuthenticatedResource):
               POST /authorities HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "country": "US",
@@ -217,8 +218,7 @@ class AuthoritiesList(AuthenticatedResource):
            :arg parent: the parent authority if this is to be a subca
            :arg signingAlgorithm: algorithm used to sign the authority
            :arg keyType: key type
-           :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored
+           :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored in an HSM
-           in an HSM
            :arg keyName: name of the key to store in the HSM (CloudCA)
            :arg serialNumber: serial number of the authority
            :arg firstSerial: specifies the starting serial number for certificates issued off of this authority
@@ -301,6 +301,7 @@ class Authorities(AuthenticatedResource):
               PUT /authorities/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "name": "TestAuthority5",
@@ -492,6 +493,26 @@ class CertificateAuthority(AuthenticatedResource):
 class AuthorityVisualizations(AuthenticatedResource):
     def get(self, authority_id):
         """
+        .. http:get:: /authorities/1/visualize
+
+           Authority visualization
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /certificates/1/visualize HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
                 {"name": "flare",
                     "children": [
                         {
@@ -508,7 +529,12 @@ class AuthorityVisualizations(AuthenticatedResource):
                                 }
                             ]
                         }
-        ]}
+                    ]
+                }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
         """
         authority = service.get(authority_id)
         return dict(

lemur/certificates/cli.py

@@ -5,13 +5,13 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import multiprocessing
 import sys
 from flask import current_app
 from flask_principal import Identity, identity_changed
 from flask_script import Manager
 from sqlalchemy import or_
 from tabulate import tabulate
+from time import sleep
 
 from lemur import database
 from lemur.authorities.models import Authority
@@ -26,9 +26,10 @@ from lemur.certificates.service import (
     get_all_valid_certs,
     get,
     get_all_certs_attached_to_endpoint_without_autorotate,
+    revoke as revoke_certificate,
 )
 from lemur.certificates.verify import verify_string
-from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS, CRLReason
 from lemur.deployment import service as deployment_service
 from lemur.domains.models import Domain
 from lemur.endpoints import service as endpoint_service
@@ -118,13 +119,20 @@ def request_rotation(endpoint, certificate, message, commit):
             status = SUCCESS_METRIC_STATUS
 
         except Exception as e:
+            sentry.captureException(extra={"certificate_name": str(certificate.name),
+                                           "endpoint": str(endpoint.dnsname)})
+            current_app.logger.exception(
+                f"Error rotating certificate: {certificate.name}", exc_info=True
+            )
             print(
                 "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
                     endpoint.name, certificate.name, e
                 )
             )
 
-    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status})
+    metrics.send("endpoint_rotation", "counter", 1, metric_tags={"status": status,
+                                                                 "certificate_name": str(certificate.name),
+                                                                 "endpoint": str(endpoint.dnsname)})
 
 
 def request_reissue(certificate, commit):
@@ -223,7 +231,7 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
             print(
                 f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
             )
-            log_data["message"] = "Rotating endpoint"
+            log_data["message"] = "Rotating one endpoint"
             log_data["endpoint"] = endpoint.dnsname
             log_data["certificate"] = new_cert.name
             request_rotation(endpoint, new_cert, message, commit)
@@ -231,8 +239,6 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
         elif old_cert and new_cert:
             print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
-
-            log_data["message"] = "Rotating all endpoints"
             log_data["certificate"] = new_cert.name
             log_data["certificate_old"] = old_cert.name
             log_data["message"] = "Rotating endpoint from old to new cert"
@@ -243,42 +249,24 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
                 current_app.logger.info(log_data)
 
         else:
+            # No certificate name or endpoint is provided. We will now fetch all endpoints,
+            # which are associated with a certificate that has been replaced
             print("[+] Rotating all endpoints that have new certificates available")
-            log_data["message"] = "Rotating all endpoints that have new certificates available"
             for endpoint in endpoint_service.get_all_pending_rotation():
+
+                log_data["message"] = "Rotating endpoint from old to new cert"
+                if len(endpoint.certificate.replaced) > 1:
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
+                                          f"{len(endpoint.certificate.replaced)}"
+
                 log_data["endpoint"] = endpoint.dnsname
-                if len(endpoint.certificate.replaced) == 1:
+                log_data["certificate"] = endpoint.certificate.replaced[0].name
                 print(
                     f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
                 )
-                    log_data["certificate"] = endpoint.certificate.replaced[0].name
+                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
-                    request_rotation(
-                        endpoint, endpoint.certificate.replaced[0], message, commit
-                    )
                 current_app.logger.info(log_data)
 
-                else:
-                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
-                    print(log_data)
-                    metrics.send(
-                        "endpoint_rotation",
-                        "counter",
-                        1,
-                        metric_tags={
-                            "status": FAILURE_METRIC_STATUS,
-                            "old_certificate_name": str(old_cert),
-                            "new_certificate_name": str(
-                                endpoint.certificate.replaced[0].name
-                            ),
-                            "endpoint_name": str(endpoint.name),
-                            "message": str(message),
-                        },
-                    )
-                    print(
-                        f"[!] Failed to rotate endpoint {endpoint.name} reason: "
-                        "Multiple replacement certificates found."
-                    )
-
         status = SUCCESS_METRIC_STATUS
         print("[+] Done!")
 
@@ -368,6 +356,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
     :param message: Send a rotation notification to the certificates owner.
     :param commit: Persist changes.
     :param region: Region in which to rotate the endpoint.
+    #todo: merge this method with rotate()
     """
     if commit:
         print("[!] Running in COMMIT mode.")
@@ -417,23 +406,19 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
                         1,
                         metric_tags={
                             "region": region,
-                            "old_certificate_name": str(old_cert),
                             "new_certificate_name": str(endpoint.certificate.replaced[0].name),
                             "endpoint_name": str(endpoint.dnsname),
                         },
                     )
+                    continue
 
-                if len(endpoint.certificate.replaced) == 1:
                 log_data["certificate"] = endpoint.certificate.replaced[0].name
                 log_data["message"] = "Rotating all endpoints in region"
-                    print(log_data)
+                if len(endpoint.certificate.replaced) > 1:
-                    current_app.logger.info(log_data)
+                    log_data["message"] = f"Multiple replacement certificates found, going with the first one out of " \
+                                          f"{len(endpoint.certificate.replaced)}"
+
                 request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
-                    status = SUCCESS_METRIC_STATUS
-                else:
-                    status = FAILURE_METRIC_STATUS
-                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
-                    print(log_data)
                 current_app.logger.info(log_data)
 
                 metrics.send(
@@ -442,8 +427,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
                     1,
                     metric_tags={
                         "status": FAILURE_METRIC_STATUS,
-                        "old_certificate_name": str(old_cert),
+                        "new_certificate_name": str(log_data["certificate"]),
-                        "new_certificate_name": str(endpoint.certificate.replaced[0].name),
                         "endpoint_name": str(endpoint.dnsname),
                         "message": str(message),
                         "region": str(region),
@@ -586,11 +570,10 @@ def worker(data, commit, reason):
     parts = [x for x in data.split(" ") if x]
     try:
         cert = get(int(parts[0].strip()))
-        plugin = plugins.get(cert.authority.plugin_name)
 
         print("[+] Revoking certificate. Id: {0} Name: {1}".format(cert.id, cert.name))
         if commit:
-            plugin.revoke_certificate(cert, reason)
+            revoke_certificate(cert, reason)
 
         metrics.send(
             "certificate_revoke",
@@ -620,10 +603,10 @@ def clear_pending():
     v.clear_pending_certificates()
 
 
-@manager.option(
+@manager.option("-p", "--path", dest="path", help="Absolute file path to a Lemur query csv.")
-    "-p", "--path", dest="path", help="Absolute file path to a Lemur query csv."
+@manager.option("-id", "--certid", dest="cert_id", help="ID of the certificate to be revoked")
-)
+@manager.option("-r", "--reason", dest="reason", default="unspecified", help="CRL Reason as per RFC 5280 section 5.3.1")
-@manager.option("-r", "--reason", dest="reason", help="Reason to revoke certificate.")
+@manager.option("-m", "--message", dest="message", help="Message explaining reason for revocation")
 @manager.option(
     "-c",
     "--commit",
@@ -632,20 +615,32 @@ def clear_pending():
     default=False,
     help="Persist changes.",
 )
-def revoke(path, reason, commit):
+def revoke(path, cert_id, reason, message, commit):
     """
     Revokes given certificate.
     """
+    if not path and not cert_id:
+        print("[!] No input certificates mentioned to revoke")
+        return
+    if path and cert_id:
+        print("[!] Please mention single certificate id (-id) or input file (-p)")
+        return
+
     if commit:
         print("[!] Running in COMMIT mode.")
 
     print("[+] Starting certificate revocation.")
 
-    with open(path, "r") as f:
+    if reason not in CRLReason.__members__:
-        args = [[x, commit, reason] for x in f.readlines()[2:]]
+        reason = CRLReason.unspecified.name
+    comments = {"comments": message, "crl_reason": reason}
 
-    with multiprocessing.Pool(processes=3) as pool:
+    if cert_id:
-        pool.starmap(worker, args)
+        worker(cert_id, commit, comments)
+    else:
+        with open(path, "r") as f:
+            for x in f.readlines()[2:]:
+                worker(x, commit, comments)
 
 
 @manager.command
@@ -750,7 +745,10 @@ def deactivate_entrust_certificates():
 
     certificates = get_all_valid_certs(['entrust-issuer'])
     entrust_plugin = plugins.get('entrust-issuer')
-    for cert in certificates:
+    for index, cert in enumerate(certificates):
+        if (index % 10) == 0:
+            # Entrust enforces a 10 request per 30s rate limit
+            sleep(30)
         try:
             response = entrust_plugin.deactivate_certificate(cert)
             if response == 200:

lemur/certificates/schemas.py

@@ -16,7 +16,7 @@ from lemur.certificates import utils as cert_utils
 from lemur.common import missing, utils, validators
 from lemur.common.fields import ArrowDateTime, Hex
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
-from lemur.constants import CERTIFICATE_KEY_TYPES
+from lemur.constants import CERTIFICATE_KEY_TYPES, CRLReason
 from lemur.destinations.schemas import DestinationNestedOutputSchema
 from lemur.dns_providers.schemas import DnsProvidersNestedOutputSchema
 from lemur.domains.schemas import DomainNestedOutputSchema
@@ -38,6 +38,7 @@ from lemur.schemas import (
     AssociatedRotationPolicySchema,
 )
 from lemur.users.schemas import UserNestedOutputSchema
+from lemur.plugins.base import plugins
 
 
 class CertificateSchema(LemurInputSchema):
@@ -89,7 +90,7 @@ class CertificateInputSchema(CertificateCreationSchema):
     csr = fields.String(allow_none=True, validate=validators.csr)
 
     key_type = fields.String(
-        validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="RSA2048"
+        validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="ECCPRIME256V1"
     )
 
     notify = fields.Boolean(default=True)
@@ -160,7 +161,7 @@ class CertificateInputSchema(CertificateCreationSchema):
             if data.get("body"):
                 data["key_type"] = utils.get_key_type_from_certificate(data["body"])
             else:
-                data["key_type"] = "RSA2048"  # default value
+                data["key_type"] = "ECCPRIME256V1"  # default value
 
         return missing.convert_validity_years(data)
 
@@ -324,6 +325,8 @@ class CertificateOutputSchema(LemurOutputSchema):
     notifications = fields.Nested(NotificationNestedOutputSchema, many=True)
     replaces = fields.Nested(CertificateNestedOutputSchema, many=True)
     authority = fields.Nested(AuthorityNestedOutputSchema)
+    # if this certificate is an authority, the authority informations are in root_authority
+    root_authority = fields.Nested(AuthorityNestedOutputSchema)
     dns_provider = fields.Nested(DnsProvidersNestedOutputSchema)
     roles = fields.Nested(RoleNestedOutputSchema, many=True)
     endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
@@ -357,6 +360,22 @@ class CertificateOutputSchema(LemurOutputSchema):
             if field in data and data[field] is None:
                 data.pop(field)
 
+    @post_dump
+    def handle_certificate(self, cert):
+        # Plugins may need to modify the cert object before returning it to the user
+        if cert['authority'] is None:
+            if cert['root_authority'] is None:
+                plugin = None
+            else:
+                # this certificate is an authority
+                plugin = plugins.get(cert['root_authority']['plugin']['slug'])
+        else:
+            plugin = plugins.get(cert['authority']['plugin']['slug'])
+        if plugin:
+            plugin.wrap_certificate(cert)
+        if 'root_authority' in cert:
+            del cert['root_authority']
+
 
 class CertificateShortOutputSchema(LemurOutputSchema):
     id = fields.Integer()
@@ -441,6 +460,7 @@ class CertificateExportInputSchema(LemurInputSchema):
 
 
 class CertificateNotificationOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
     description = fields.String()
     issuer = fields.String()
     name = fields.String()
@@ -455,6 +475,7 @@ class CertificateNotificationOutputSchema(LemurOutputSchema):
 
 class CertificateRevokeSchema(LemurInputSchema):
     comments = fields.String()
+    crl_reason = fields.String(validate=validate.OneOf(CRLReason.__members__), missing="unspecified")
 
 
 certificates_list_request_parser = RequestParser()

lemur/certificates/service.py

@@ -21,6 +21,7 @@ from lemur.certificates.schemas import CertificateOutputSchema, CertificateInput
 from lemur.common.utils import generate_private_key, truthiness
 from lemur.destinations.models import Destination
 from lemur.domains.models import Domain
+from lemur.endpoints import service as endpoint_service
 from lemur.extensions import metrics, sentry, signals
 from lemur.models import certificate_associations
 from lemur.notifications.models import Notification
@@ -87,6 +88,16 @@ def get_by_attributes(conditions):
     return database.find_all(query, Certificate, conditions).all()
 
 
+def get_by_root_authority(id):
+    """
+    Retrieves certificate by its root_authority's id.
+
+    :param id:
+    :return:
+    """
+    return database.get(Certificate, id, field="root_authority_id")
+
+
 def delete(cert_id):
     """
     Delete's a certificate.
@@ -152,6 +163,7 @@ def get_all_certs_attached_to_endpoint_without_autorotate():
     return (
         Certificate.query.filter(Certificate.endpoints.any())
         .filter(Certificate.rotation == false())
+        .filter(Certificate.revoked == false())
         .filter(Certificate.not_after >= arrow.now())
         .filter(not_(Certificate.replaced.any()))
         .all()  # noqa
@@ -387,6 +399,7 @@ def create(**kwargs):
         cert = Certificate(**kwargs)
         kwargs["creator"].certificates.append(cert)
     else:
+        # ACME path
         cert = PendingCertificate(**kwargs)
         kwargs["creator"].pending_certificates.append(cert)
 
@@ -562,10 +575,15 @@ def query_common_name(common_name, args):
     :return:
     """
     owner = args.pop("owner")
+    page = args.pop("page")
+    count = args.pop("count")
+
+    paginate = page and count
+    query = database.session_query(Certificate) if paginate else Certificate.query
+
     # only not expired certificates
     current_time = arrow.utcnow()
-
+    query = query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\
-    query = Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\
         .filter(not_(Certificate.revoked))\
         .filter(not_(Certificate.replaced.any()))  # ignore rotated certificates to avoid duplicates
 
@@ -576,6 +594,9 @@ def query_common_name(common_name, args):
         # if common_name is a wildcard ('%'), no need to include it in the query
         query = query.filter(Certificate.cn.ilike(common_name))
 
+    if paginate:
+        return database.paginate(query, page, count)
+
     return query.all()
 
 
@@ -668,7 +689,16 @@ def stats(**kwargs):
     :param kwargs:
     :return:
     """
-    if kwargs.get("metric") == "not_after":
+
+    # Verify requested metric
+    allow_list = ["bits", "issuer", "not_after", "signing_algorithm"]
+    req_metric = kwargs.get("metric")
+    if req_metric not in allow_list:
+        raise Exception(
+            f"Stats not available for requested metric: {req_metric}"
+        )
+
+    if req_metric == "not_after":
         start = arrow.utcnow()
         end = start.shift(weeks=+32)
         items = (
@@ -680,7 +710,7 @@ def stats(**kwargs):
         )
 
     else:
-        attr = getattr(Certificate, kwargs.get("metric"))
+        attr = getattr(Certificate, req_metric)
         query = database.db.session.query(attr, func.count(attr))
 
         items = query.group_by(attr).all()
@@ -794,6 +824,90 @@ def reissue_certificate(certificate, replace=None, user=None):
     else:
         primitives["description"] = f"{reissue_message_prefix}{certificate.id}"
 
+    # Rotate the certificate to ECCPRIME256V1 if cert owner is present in the configured list
+    # This is a temporary change intending to rotate certificates to ECC, if opted in by certificate owners
+    # Unless identified a use case, this will be removed in mid-Q2 2021
+    ecc_reissue_owner_list = current_app.config.get("ROTATE_TO_ECC_OWNER_LIST", [])
+    ecc_reissue_exclude_cn_list = current_app.config.get("ECC_NON_COMPATIBLE_COMMON_NAMES", [])
+
+    if (certificate.owner in ecc_reissue_owner_list) and (certificate.cn not in ecc_reissue_exclude_cn_list):
+        primitives["key_type"] = "ECCPRIME256V1"
+
     new_cert = create(**primitives)
 
     return new_cert
+
+
+def is_attached_to_endpoint(certificate_name, endpoint_name):
+    """
+    Find if given certificate is attached to the endpoint. Both, certificate and endpoint, are identified by name.
+    This method talks to elb and finds the real time information.
+    :param certificate_name:
+    :param endpoint_name:
+    :return: True if certificate is attached to the given endpoint, False otherwise
+    """
+    endpoint = endpoint_service.get_by_name(endpoint_name)
+    attached_certificates = endpoint.source.plugin.get_endpoint_certificate_names(endpoint)
+    return certificate_name in attached_certificates
+
+
+def remove_from_destination(certificate, destination):
+    """
+    Remove the certificate from given destination if clean() is implemented
+    :param certificate:
+    :param destination:
+    :return:
+    """
+    plugin = plugins.get(destination.plugin_name)
+    if not hasattr(plugin, "clean"):
+        info_text = f"Cannot clean certificate {certificate.name}, {destination.plugin_name} plugin does not implement 'clean()'"
+        current_app.logger.warning(info_text)
+    else:
+        plugin.clean(certificate=certificate, options=destination.options)
+
+
+def revoke(certificate, reason):
+    plugin = plugins.get(certificate.authority.plugin_name)
+    plugin.revoke_certificate(certificate, reason)
+
+    # Perform cleanup after revoke
+    return cleanup_after_revoke(certificate)
+
+
+def cleanup_after_revoke(certificate):
+    """
+    Perform the needed cleanup for a revoked certificate. This includes -
+    1. Disabling notification
+    2. Disabling auto-rotation
+    3. Update certificate status to 'revoked'
+    4. Remove from AWS
+    :param certificate: Certificate object to modify and update in DB
+    :return: None
+    """
+    certificate.notify = False
+    certificate.rotation = False
+    certificate.status = 'revoked'
+
+    error_message = ""
+
+    for destination in list(certificate.destinations):
+        try:
+            remove_from_destination(certificate, destination)
+            certificate.destinations.remove(destination)
+        except Exception as e:
+            # This cleanup is the best-effort since certificate is already revoked at this point.
+            # We will capture the exception and move on to the next destination
+            sentry.captureException()
+            error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
+
+    database.update(certificate)
+    return error_message
+
+
+def get_issued_cert_count_for_authority(authority):
+    """
+    Returns the count of certs issued by the specified authority.
+
+    :return:
+    """
+    return database.db.session.query(Certificate).filter(Certificate.authority_id == authority.id).count()

lemur/certificates/views.py

@@ -19,7 +19,7 @@ from lemur.auth.permissions import AuthorityPermission, CertificatePermission
 
 from lemur.certificates import service
 from lemur.certificates.models import Certificate
-from lemur.plugins.base import plugins
+from lemur.extensions import sentry
 from lemur.certificates.schemas import (
     certificate_input_schema,
     certificate_output_schema,
@@ -28,10 +28,12 @@ from lemur.certificates.schemas import (
     certificate_export_input_schema,
     certificate_edit_input_schema,
     certificates_list_output_schema_factory,
+    certificate_revoke_schema,
 )
 
 from lemur.roles import service as role_service
 from lemur.logs import service as log_service
+from lemur.plugins.base import plugins
 
 
 mod = Blueprint("certificates", __name__)
@@ -50,17 +52,20 @@ class CertificatesListValid(AuthenticatedResource):
         """
         .. http:get:: /certificates/valid/<query>
 
-           The current list of not-expired certificates for a given common name, and owner
+           The current list of not-expired certificates for a given common name, and owner. The API offers
+           optional pagination. One can send page number(>=1) and desired count per page. The returned data
+           contains total number of certificates which can help in determining the last page. Pagination
+           will not be offered if page or count info is not sent or if it is zero.
 
            **Example request**:
 
            .. sourcecode:: http
-              GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com
+
-              HTTP/1.1
+              GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com&page=1&count=20 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
 
-           **Example response**:
+           **Example response (with single cert to be concise)**:
 
            .. sourcecode:: http
 
@@ -127,10 +132,15 @@ class CertificatesListValid(AuthenticatedResource):
            :statuscode 403: unauthenticated
 
         """
-        parser = paginated_parser.copy()
+        # using non-paginated parser to ensure backward compatibility
-        args = parser.parse_args()
+        self.reqparse.add_argument("filter", type=str, location="args")
+        self.reqparse.add_argument("owner", type=str, location="args")
+        self.reqparse.add_argument("count", type=int, location="args")
+        self.reqparse.add_argument("page", type=int, location="args")
+
+        args = self.reqparse.parse_args()
         args["user"] = g.user
-        common_name = args["filter"].split(";")[1]
+        common_name = args.pop("filter").split(";")[1]
         return service.query_common_name(common_name, args)
 
 
@@ -368,6 +378,7 @@ class CertificatesList(AuthenticatedResource):
               POST /certificates HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                   "owner": "secure@example.net",
@@ -517,6 +528,7 @@ class CertificatesUpload(AuthenticatedResource):
               POST /certificates/upload HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "owner": "joe@example.com",
@@ -624,7 +636,12 @@ class CertificatesStats(AuthenticatedResource):
 
         args = self.reqparse.parse_args()
 
+        try:
             items = service.stats(**args)
+        except Exception as e:
+            sentry.captureException()
+            return dict(message=f"Failed to retrieve stats: {str(e)}"), 400
+
         return dict(items=items, total=len(items))
 
 
@@ -675,6 +692,16 @@ class CertificatePrivateKey(AuthenticatedResource):
                 return dict(message="You are not authorized to view this key"), 403
 
         log_service.create(g.current_user, "key_view", certificate=cert)
+
+        # Plugins may need to modify the cert object before returning it to the user
+        if cert.root_authority:
+            # this certificate is an authority
+            plugin_name = cert.root_authority.plugin_name
+        else:
+            plugin_name = cert.authority.plugin_name
+        plugin = plugins.get(plugin_name)
+        plugin.wrap_private_key(cert)
+
         response = make_response(jsonify(key=cert.private_key), 200)
         response.headers["cache-control"] = "private, max-age=0, no-cache, no-store"
         response.headers["pragma"] = "no-cache"
@@ -783,6 +810,7 @@ class Certificates(AuthenticatedResource):
               PUT /certificates/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "owner": "jimbob@example.com",
@@ -888,8 +916,24 @@ class Certificates(AuthenticatedResource):
         if cert.owner != data["owner"]:
             service.cleanup_owner_roles_notification(cert.owner, data)
 
+        error_message = ""
+        # if destination is removed, cleanup the certificate from AWS
+        for destination in cert.destinations:
+            if destination not in data["destinations"]:
+                try:
+                    service.remove_from_destination(cert, destination)
+                except Exception as e:
+                    sentry.captureException()
+                    # Add the removed destination back
+                    data["destinations"].append(destination)
+                    error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
+
+        # go ahead with DB update
         cert = service.update(certificate_id, **data)
         log_service.create(g.current_user, "update_cert", certificate=cert)
+
+        if error_message:
+            return dict(message=f"Edit Successful except -\n\n {error_message}"), 400
         return cert
 
     @validate_schema(certificate_edit_input_schema, certificate_output_schema)
@@ -906,6 +950,7 @@ class Certificates(AuthenticatedResource):
               POST /certificates/1/update/notify HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "notify": false
@@ -1274,6 +1319,7 @@ class CertificateExport(AuthenticatedResource):
               PUT /certificates/1/export HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "export": {
@@ -1381,7 +1427,7 @@ class CertificateRevoke(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(CertificateRevoke, self).__init__()
 
-    @validate_schema(None, None)
+    @validate_schema(certificate_revoke_schema, None)
     def put(self, certificate_id, data=None):
         """
         .. http:put:: /certificates/1/revoke
@@ -1395,6 +1441,12 @@ class CertificateRevoke(AuthenticatedResource):
               POST /certificates/1/revoke HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                "crlReason": "affiliationChanged",
+                "comments": "Additional details if any"
+              }
 
            **Example response**:
 
@@ -1405,12 +1457,13 @@ class CertificateRevoke(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                'id': 1
+                "id": 1
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
-           :statuscode 403: unauthenticated
+           :statuscode 403: unauthenticated or cert attached to LB
+           :statuscode 400: encountered error, more details in error message
 
         """
         cert = service.get(certificate_id)
@@ -1433,6 +1486,8 @@ class CertificateRevoke(AuthenticatedResource):
             return dict(message="Cannot revoke certificate. No external id found."), 400
 
         if cert.endpoints:
+            for endpoint in cert.endpoints:
+                if service.is_attached_to_endpoint(cert.name, endpoint.name):
                     return (
                         dict(
                             message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
@@ -1440,10 +1495,18 @@ class CertificateRevoke(AuthenticatedResource):
                         403,
                     )
 
-        plugin = plugins.get(cert.authority.plugin_name)
+        try:
-        plugin.revoke_certificate(cert, data)
+            error_message = service.revoke(cert, data)
             log_service.create(g.current_user, "revoke_cert", certificate=cert)
+
+            if error_message:
+                return dict(message=f"Certificate (id:{cert.id}) is revoked - {error_message}"), 400
             return dict(id=cert.id)
+        except NotImplementedError as ne:
+            return dict(message="Revoke is not implemented for issuer of this certificate"), 400
+        except Exception as e:
+            sentry.captureException()
+            return dict(message=f"Failed to revoke: {str(e)}"), 400
 
 
 api.add_resource(

lemur/common/celery.py

@@ -20,6 +20,7 @@ from flask import current_app
 from lemur.authorities.service import get as get_authority
 from lemur.certificates import cli as cli_certificate
 from lemur.common.redis import RedisHandler
+from lemur.constants import ACME_ADDITIONAL_ATTEMPTS
 from lemur.destinations import service as destinations_service
 from lemur.dns_providers import cli as cli_dns_providers
 from lemur.endpoints import cli as cli_endpoints
@@ -273,7 +274,8 @@ def fetch_acme_cert(id):
         real_cert = cert.get("cert")
         # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
         pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
-        if not pending_cert:
+        if not pending_cert or pending_cert.resolved:
+            # pending_cert is cleared or it was resolved by another process
             log_data[
                 "message"
             ] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
@@ -301,7 +303,7 @@ def fetch_acme_cert(id):
             error_log["last_error"] = cert.get("last_error")
             error_log["cn"] = pending_cert.cn
 
-            if pending_cert.number_attempts > 4:
+            if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
                 error_log["message"] = "Deleting pending certificate"
                 send_pending_failure_notification(
                     pending_cert, notify_owner=pending_cert.notify
@@ -656,11 +658,12 @@ def certificate_rotate(**kwargs):
 
     current_app.logger.debug(log_data)
     try:
+        notify = current_app.config.get("ENABLE_ROTATION_NOTIFICATION", None)
         if region:
             log_data["region"] = region
-            cli_certificate.rotate_region(None, None, None, None, True, region)
+            cli_certificate.rotate_region(None, None, None, notify, True, region)
         else:
-            cli_certificate.rotate(None, None, None, None, True)
+            cli_certificate.rotate(None, None, None, notify, True)
     except SoftTimeLimitExceeded:
         log_data["message"] = "Certificate rotate: Time limit exceeded."
         current_app.logger.error(log_data)
@@ -820,6 +823,78 @@ def notify_expirations():
     return log_data
 
 
+@celery.task(soft_time_limit=3600)
+def notify_authority_expirations():
+    """
+    This celery task notifies about expiring certificate authority certs
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "notify for certificate authority cert expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_notification.authority_expirations()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Notify expiring CA Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def send_security_expiration_summary():
+    """
+    This celery task sends a summary about expiring certificates to the security team.
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "send summary for certificate expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_notification.security_expiration_summary(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Send summary for expiring certs Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
 @celery.task(soft_time_limit=3600)
 def enable_autorotate_for_certs_attached_to_endpoint():
     """

lemur/common/utils.py

@@ -10,6 +10,7 @@ import random
 import re
 import string
 import pem
+import base64
 
 import sqlalchemy
 from cryptography import x509
@@ -34,6 +35,12 @@ paginated_parser.add_argument("filter", type=str, location="args")
 paginated_parser.add_argument("owner", type=str, location="args")
 
 
+def base64encode(string):
+    # Performs Base64 encoding of string to string using the base64.b64encode() function
+    # which encodes bytes to bytes.
+    return base64.b64encode(string.encode()).decode()
+
+
 def get_psuedo_random_string():
     """
     Create a random and strongish challenge.

lemur/constants.py

@@ -3,6 +3,8 @@
     :copyright: (c) 2018 by Netflix Inc.
     :license: Apache, see LICENSE for more details.
 """
+from enum import IntEnum
+
 SAN_NAMING_TEMPLATE = "SAN-{subject}-{issuer}-{not_before}-{not_after}"
 DEFAULT_NAMING_TEMPLATE = "{subject}-{issuer}-{not_before}-{not_after}"
 NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
@@ -10,6 +12,9 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
 SUCCESS_METRIC_STATUS = "success"
 FAILURE_METRIC_STATUS = "failure"
 
+# when ACME attempts to resolve a certificate try in total 3 times
+ACME_ADDITIONAL_ATTEMPTS = 2
+
 CERTIFICATE_KEY_TYPES = [
     "RSA2048",
     "RSA4096",
@@ -32,3 +37,17 @@ CERTIFICATE_KEY_TYPES = [
     "ECCSECT409R1",
     "ECCSECT571R2",
 ]
+
+
+# As per RFC 5280 section 5.3.1 (https://tools.ietf.org/html/rfc5280#section-5.3.1)
+class CRLReason(IntEnum):
+    unspecified = 0,
+    keyCompromise = 1,
+    cACompromise = 2,
+    affiliationChanged = 3,
+    superseded = 4,
+    cessationOfOperation = 5,
+    certificateHold = 6,
+    removeFromCRL = 8,
+    privilegeWithdrawn = 9,
+    aACompromise = 10

lemur/database.py

@@ -9,6 +9,7 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import math
 from inflection import underscore
 from sqlalchemy import exc, func, distinct
 from sqlalchemy.orm import make_transient, lazyload
@@ -219,13 +220,20 @@ def sort(query, model, field, direction):
 
 def paginate(query, page, count):
     """
-    Returns the items given the count and page specified
+    Returns the items given the count and page specified. The items would be an empty list
+    if page number exceeds max page number based on count per page and total number of records.
 
-    :param query:
+    :param query: search query
-    :param page:
+    :param page: current page number
-    :param count:
+    :param count: results per page
     """
-    return query.paginate(page, count)
+    total = get_count(query)
+    # Check if input page is higher than total number of pages based on count per page and total
+    # In such a case Flask-SQLAlchemy pagination call results in 404
+    if math.ceil(total / count) < page:
+        return dict(items=[], total=total)
+    items = query.paginate(page, count).items
+    return dict(items=items, total=total)
 
 
 def update_list(model, model_attr, item_model, items):

lemur/destinations/service.py

@@ -21,7 +21,7 @@ def create(label, plugin_name, options, description=None):
 
     :param label: Destination common name
     :param description:
-    :rtype : Destination
+    :rtype: Destination
     :return: New destination
     """
     # remove any sub-plugin objects before try to save the json options
@@ -50,7 +50,7 @@ def update(destination_id, label, plugin_name, options, description):
     :param plugin_name:
     :param options:
     :param description:
-    :rtype : Destination
+    :rtype: Destination
     :return:
     """
     destination = get(destination_id)
@@ -81,7 +81,7 @@ def get(destination_id):
     Retrieves an destination by its lemur assigned ID.
 
     :param destination_id: Lemur assigned ID
-    :rtype : Destination
+    :rtype: Destination
     :return:
     """
     return database.get(Destination, destination_id)

lemur/destinations/views.py

@@ -113,6 +113,7 @@ class DestinationsList(AuthenticatedResource):
               POST /destinations HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "description": "test33",
@@ -264,6 +265,7 @@ class Destinations(AuthenticatedResource):
               POST /destinations/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
 
               {
@@ -423,7 +425,7 @@ class CertificateDestinations(AuthenticatedResource):
 
 
 class DestinationsStats(AuthenticatedResource):
-    """ Defines the 'certificates' stats endpoint """
+    """ Defines the 'destinations' stats endpoint """
 
     def __init__(self):
         self.reqparse = reqparse.RequestParser()

lemur/dns_providers/schemas.py

@@ -10,9 +10,9 @@ class DnsProvidersNestedOutputSchema(LemurOutputSchema):
     name = fields.String()
     provider_type = fields.String()
     description = fields.String()
-    credentials = fields.String()
     api_endpoint = fields.String()
     date_created = ArrowDateTime()
+    # credentials are intentionally omitted (they are input-only)
 
 
 class DnsProvidersNestedInputSchema(LemurInputSchema):

lemur/dns_providers/service.py

@@ -36,7 +36,7 @@ def get_friendly(dns_provider_id):
     Retrieves a dns provider by its lemur assigned ID.
 
     :param dns_provider_id: Lemur assigned ID
-    :rtype : DnsProvider
+    :rtype: DnsProvider
     :return:
     """
     dns_provider = get(dns_provider_id)

lemur/dns_providers/views.py

@@ -86,9 +86,18 @@ class DnsProvidersList(AuthenticatedResource):
     @admin_permission.require(http_exception=403)
     def post(self, data=None):
         """
+        .. http:post:: /dns_providers
+
            Creates a DNS Provider
 
            **Example request**:
+
+           .. sourcecode:: http
+
+              POST /dns_providers HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
                 {
                   "providerType": {
                     "name": "route53",
@@ -112,7 +121,14 @@ class DnsProvidersList(AuthenticatedResource):
                   "description": "provider_description"
                 }
 
-        **Example request 2**
+        **Example request 2**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
                 {
                   "providerType": {
                     "name": "cloudflare",
@@ -142,6 +158,7 @@ class DnsProvidersList(AuthenticatedResource):
                   "name": "provider_name",
                   "description": "provider_description"
                 }
+
         :return:
         """
         return service.create(data)

lemur/domains/views.py

@@ -96,7 +96,7 @@ class DomainsList(AuthenticatedResource):
 
            .. sourcecode:: http
 
-              GET /domains HTTP/1.1
+              POST /domains HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
 

lemur/logs/service.py

@@ -7,7 +7,7 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from flask import current_app
+from flask import current_app, g
 
 from lemur import database
 from lemur.logs.models import Log
@@ -34,6 +34,20 @@ def create(user, type, certificate=None):
     database.commit()
 
 
+def audit_log(action, entity, message):
+    """
+    Logs given action
+    :param action: The action being logged e.g. assign_role, create_role etc
+    :param entity: The entity undergoing the action e.g. name of the role
+    :param message: Additional info e.g. Role being assigned to user X
+    :return:
+    """
+    user = g.current_user.email if hasattr(g, 'current_user') else "LEMUR"
+    current_app.logger.info(
+        f"[lemur-audit] action: {action}, user: {user}, entity: {entity}, details: {message}"
+    )
+
+
 def get_all():
     """
     Retrieve all logs from the database.

lemur/notifications/cli.py

@@ -10,6 +10,8 @@ from flask_script import Manager
 from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
 from lemur.extensions import sentry, metrics
 from lemur.notifications.messaging import send_expiration_notifications
+from lemur.notifications.messaging import send_authority_expiration_notifications
+from lemur.notifications.messaging import send_security_expiration_summary
 
 manager = Manager(usage="Handles notification related tasks.")
 
@@ -24,7 +26,7 @@ manager = Manager(usage="Handles notification related tasks.")
 )
 def expirations(exclude):
     """
-    Runs Lemur's notification engine, that looks for expired certificates and sends
+    Runs Lemur's notification engine, that looks for expiring certificates and sends
     notifications out to those that have subscribed to them.
 
     Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur
@@ -39,9 +41,7 @@ def expirations(exclude):
         print("Starting to notify subscribers about expiring certificates!")
         success, failed = send_expiration_notifications(exclude)
         print(
-            "Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}".format(
+            f"Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}"
-                success=success, failed=failed
-            )
         )
         status = SUCCESS_METRIC_STATUS
     except Exception as e:
@@ -50,3 +50,50 @@ def expirations(exclude):
     metrics.send(
         "expiration_notification_job", "counter", 1, metric_tags={"status": status}
     )
+
+
+def authority_expirations():
+    """
+    Runs Lemur's notification engine, that looks for expiring certificate authority certificates and sends
+    notifications out to the security team and owner.
+
+    :return:
+    """
+    status = FAILURE_METRIC_STATUS
+    try:
+        print("Starting to notify subscribers about expiring certificate authority certificates!")
+        success, failed = send_authority_expiration_notifications()
+        print(
+            "Finished notifying subscribers about expiring certificate authority certificates! "
+            f"Sent: {success} Failed: {failed}"
+        )
+        status = SUCCESS_METRIC_STATUS
+    except Exception as e:
+        sentry.captureException()
+
+    metrics.send(
+        "authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
+    )
+
+
+def security_expiration_summary(exclude):
+    """
+    Sends a summary email with info on all expiring certs (that match the configured expiry intervals).
+
+    :return:
+    """
+    status = FAILURE_METRIC_STATUS
+    try:
+        print("Starting to notify security team about expiring certificates!")
+        success = send_security_expiration_summary(exclude)
+        print(
+            f"Finished notifying security team about expiring certificates! Success: {success}"
+        )
+        if success:
+            status = SUCCESS_METRIC_STATUS
+    except Exception:
+        sentry.captureException()
+
+    metrics.send(
+        "security_expiration_notification_job", "counter", 1, metric_tags={"status": status}
+    )

lemur/notifications/messaging.py

@@ -19,9 +19,10 @@ from sqlalchemy import and_
 from sqlalchemy.sql.expression import false, true
 
 from lemur import database
+from lemur.certificates import service as certificates_service
 from lemur.certificates.models import Certificate
 from lemur.certificates.schemas import certificate_notification_output_schema
-from lemur.common.utils import windowed_query
+from lemur.common.utils import windowed_query, is_selfsigned
 from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS
 from lemur.extensions import metrics, sentry
 from lemur.pending_certificates.schemas import pending_certificate_output_schema
@@ -62,6 +63,67 @@ def get_certificates(exclude=None):
     return certs
 
 
+def get_certificates_for_security_summary_email(exclude=None):
+    """
+    Finds all certificates that are eligible for expiration notifications for the security expiration summary.
+    :param exclude:
+    :return:
+    """
+    now = arrow.utcnow()
+    threshold_days = current_app.config.get("LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS", 14)
+    max_not_after = now + timedelta(days=threshold_days + 1)
+
+    q = (
+        database.db.session.query(Certificate)
+        .filter(Certificate.not_after <= max_not_after)
+        .filter(Certificate.notify == true())
+        .filter(Certificate.expired == false())
+        .filter(Certificate.revoked == false())
+    )
+
+    exclude_conditions = []
+    if exclude:
+        for e in exclude:
+            exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e)))
+
+        q = q.filter(and_(*exclude_conditions))
+
+    certs = []
+    for c in windowed_query(q, Certificate.id, 10000):
+        days_remaining = (c.not_after - now).days
+        if days_remaining <= threshold_days:
+            certs.append(c)
+    return certs
+
+
+def get_expiring_authority_certificates():
+    """
+    Finds all certificate authority certificates that are eligible for expiration notifications.
+    :return:
+    """
+    now = arrow.utcnow()
+    authority_expiration_intervals = current_app.config.get("LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS",
+                                                            [365, 180])
+    max_not_after = now + timedelta(days=max(authority_expiration_intervals) + 1)
+
+    q = (
+        database.db.session.query(Certificate)
+        .filter(Certificate.not_after < max_not_after)
+        .filter(Certificate.notify == true())
+        .filter(Certificate.expired == false())
+        .filter(Certificate.revoked == false())
+        .filter(Certificate.root_authority_id.isnot(None))
+        .filter(Certificate.authority_id.is_(None))
+    )
+
+    certs = []
+    for c in windowed_query(q, Certificate.id, 10000):
+        days_remaining = (c.not_after - now).days
+        if days_remaining in authority_expiration_intervals:
+            certs.append(c)
+    return certs
+
+
 def get_eligible_certificates(exclude=None):
     """
     Finds all certificates that are eligible for certificate expiration notification.
@@ -90,6 +152,37 @@ def get_eligible_certificates(exclude=None):
     return certificates
 
 
+def get_eligible_security_summary_certs(exclude=None):
+    certificates = defaultdict(list)
+    all_certs = get_certificates_for_security_summary_email(exclude=exclude)
+    now = arrow.utcnow()
+
+    # group by expiration interval
+    for interval, interval_certs in groupby(all_certs, lambda x: (x.not_after - now).days):
+        certificates[interval] = list(interval_certs)
+
+    return certificates
+
+
+def get_eligible_authority_certificates():
+    """
+    Finds all certificate authority certificates that are eligible for certificate expiration notification.
+    Returns the set of all eligible CA certificates, grouped by owner and interval, with a list of applicable certs.
+    :return:
+    """
+    certificates = defaultdict(dict)
+    all_certs = get_expiring_authority_certificates()
+    now = arrow.utcnow()
+
+    # group by owner
+    for owner, owner_certs in groupby(all_certs, lambda x: x.owner):
+        # group by expiration interval
+        for interval, interval_certs in groupby(owner_certs, lambda x: (x.not_after - now).days):
+            certificates[owner][interval] = list(interval_certs)
+
+    return certificates
+
+
 def send_plugin_notification(event_type, data, recipients, notification):
     """
     Executes the plugin and handles failure.
@@ -103,9 +196,12 @@ def send_plugin_notification(event_type, data, recipients, notification):
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
     log_data = {
         "function": function,
-        "message": f"Sending expiration notification for to recipients {recipients}",
+        "message": f"Sending {event_type} notification for to recipients {recipients}",
-        "notification_type": "expiration",
+        "notification_type": event_type,
+        "notification_plugin": notification.plugin.slug,
         "certificate_targets": recipients,
+        "plugin": notification.plugin.slug,
+        "notification_id": notification.id,
     }
     status = FAILURE_METRIC_STATUS
     try:
@@ -121,7 +217,7 @@ def send_plugin_notification(event_type, data, recipients, notification):
         "notification",
         "counter",
         1,
-        metric_tags={"status": status, "event_type": event_type},
+        metric_tags={"status": status, "event_type": event_type, "plugin": notification.plugin.slug},
     )
 
     if status == SUCCESS_METRIC_STATUS:
@@ -142,7 +238,6 @@ def send_expiration_notifications(exclude):
 
         for notification_label, certificates in notification_group.items():
             notification_data = []
-            security_data = []
 
             notification = certificates[0][0]
 
@@ -152,33 +247,60 @@ def send_expiration_notifications(exclude):
                     certificate
                 ).data
                 notification_data.append(cert_data)
-                security_data.append(cert_data)
-
-            if send_default_notification(
-                    "expiration", notification_data, [owner], notification.options
-            ):
-                success += 1
-            else:
-                failure += 1
-
-            recipients = notification.plugin.filter_recipients(notification.options, security_email + [owner])
 
+            email_recipients = notification.plugin.get_recipients(notification.options, security_email + [owner])
+            # Plugin will ONLY use the provided recipients if it's email; any other notification plugin ignores them
             if send_plugin_notification(
-                "expiration",
+                    "expiration", notification_data, email_recipients, notification
-                notification_data,
-                recipients,
-                notification,
             ):
-                success += 1
+                success += len(email_recipients)
             else:
-                failure += 1
+                failure += len(email_recipients)
-
+            # If we're using an email plugin, we're done,
+            #   since "security_email + [owner]" were added as email_recipients.
+            # If we're not using an email plugin, we also need to send an email to the security team and owner,
+            #   since the plugin notification didn't send anything to them.
+            if notification.plugin.slug != "email-notification":
                 if send_default_notification(
-                "expiration", security_data, security_email, notification.options
+                        "expiration", notification_data, email_recipients, notification.options
                 ):
-                success += 1
+                    success = 1 + len(email_recipients)
                 else:
-                failure += 1
+                    failure = 1 + len(email_recipients)
+
+    return success, failure
+
+
+def send_authority_expiration_notifications():
+    """
+    This function will check for upcoming certificate authority certificate expiration,
+    and send out notification emails at configured intervals.
+    """
+    success = failure = 0
+
+    # security team gets all
+    security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
+
+    for owner, owner_cert_groups in get_eligible_authority_certificates().items():
+        for interval, certificates in owner_cert_groups.items():
+            notification_data = []
+
+            for certificate in certificates:
+                cert_data = certificate_notification_output_schema.dump(
+                    certificate
+                ).data
+                cert_data['self_signed'] = is_selfsigned(certificate.parsed_cert)
+                cert_data['issued_cert_count'] = certificates_service.get_issued_cert_count_for_authority(certificate.root_authority)
+                notification_data.append(cert_data)
+
+            email_recipients = security_email + [owner]
+            if send_default_notification(
+                    "authority_expiration", notification_data, email_recipients,
+                    notification_options=[{'name': 'interval', 'value': interval}]
+            ):
+                success = len(email_recipients)
+            else:
+                failure = len(email_recipients)
 
     return success, failure
 
@@ -195,15 +317,16 @@ def send_default_notification(notification_type, data, targets, notification_opt
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": f"Sending notification for certificate data {data}",
-        "notification_type": notification_type,
-    }
     status = FAILURE_METRIC_STATUS
     notification_plugin = plugins.get(
         current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
     )
+    log_data = {
+        "function": function,
+        "message": f"Sending {notification_type} notification for certificate data {data} to targets {targets}",
+        "notification_type": notification_type,
+        "notification_plugin": notification_plugin.slug,
+    }
 
     try:
         current_app.logger.debug(log_data)
@@ -212,7 +335,7 @@ def send_default_notification(notification_type, data, targets, notification_opt
         status = SUCCESS_METRIC_STATUS
     except Exception as e:
         log_data["message"] = f"Unable to send {notification_type} notification for certificate data {data} " \
-                              f"to target {targets}"
+                              f"to targets {targets}"
         current_app.logger.error(log_data, exc_info=True)
         sentry.captureException()
 
@@ -220,7 +343,7 @@ def send_default_notification(notification_type, data, targets, notification_opt
         "notification",
         "counter",
         1,
-        metric_tags={"status": status, "event_type": notification_type},
+        metric_tags={"status": status, "event_type": notification_type, "plugin": notification_plugin.slug},
     )
 
     if status == SUCCESS_METRIC_STATUS:
@@ -247,15 +370,14 @@ def send_pending_failure_notification(
     data = pending_certificate_output_schema.dump(pending_cert).data
     data["security_email"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
 
-    notify_owner_success = False
+    email_recipients = []
     if notify_owner:
-        notify_owner_success = send_default_notification("failed", data, [data["owner"]], pending_cert)
+        email_recipients = email_recipients + [data["owner"]]
 
-    notify_security_success = False
     if notify_security:
-        notify_security_success = send_default_notification("failed", data, data["security_email"], pending_cert)
+        email_recipients = email_recipients + data["security_email"]
 
-    return notify_owner_success or notify_security_success
+    return send_default_notification("failed", data, email_recipients, pending_cert)
 
 
 def needs_notification(certificate):
@@ -295,3 +417,59 @@ def needs_notification(certificate):
         if days == interval:
             notifications.append(notification)
     return notifications
+
+
+def send_security_expiration_summary(exclude=None):
+    """
+    Sends a report to the security team with a summary of all expiring certificates.
+    All expiring certificates are included here, regardless of notification configuration.
+    Certificates with notifications disabled are omitted.
+
+    :param exclude:
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    status = FAILURE_METRIC_STATUS
+    notification_plugin = plugins.get(
+        current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
+    )
+    notification_type = "expiration_summary"
+    log_data = {
+        "function": function,
+        "message": "Sending expiration summary notification for to security team",
+        "notification_type": notification_type,
+        "notification_plugin": notification_plugin.slug,
+    }
+
+    intervals_and_certs = get_eligible_security_summary_certs(exclude)
+    security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
+
+    try:
+        current_app.logger.debug(log_data)
+
+        message_data = []
+
+        for interval, certs in intervals_and_certs.items():
+            cert_data = []
+            for certificate in certs:
+                cert_data.append(certificate_notification_output_schema.dump(certificate).data)
+            interval_data = {"interval": interval, "certificates": cert_data}
+            message_data.append(interval_data)
+
+        notification_plugin.send(notification_type, message_data, security_email, None)
+        status = SUCCESS_METRIC_STATUS
+    except Exception:
+        log_data["message"] = f"Unable to send {notification_type} notification for certificates " \
+                              f"{intervals_and_certs} to targets {security_email}"
+        current_app.logger.error(log_data, exc_info=True)
+        sentry.captureException()
+
+    metrics.send(
+        "notification",
+        "counter",
+        1,
+        metric_tags={"status": status, "event_type": notification_type, "plugin": notification_plugin.slug},
+    )
+
+    if status == SUCCESS_METRIC_STATUS:
+        return True

lemur/notifications/schemas.py

@@ -21,6 +21,8 @@ class NotificationInputSchema(LemurInputSchema):
     active = fields.Boolean()
     plugin = fields.Nested(PluginInputSchema, required=True)
     certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[])
+    added_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[])
+    removed_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[])
 
 
 class NotificationOutputSchema(LemurOutputSchema):

lemur/notifications/service.py

@@ -94,7 +94,7 @@ def create(label, plugin_name, options, description, certificates):
     :param options:
     :param description:
     :param certificates:
-    :rtype : Notification
+    :rtype: Notification
     :return:
     """
     notification = Notification(
@@ -104,7 +104,7 @@ def create(label, plugin_name, options, description, certificates):
     return database.create(notification)
 
 
-def update(notification_id, label, plugin_name, options, description, active, certificates):
+def update(notification_id, label, plugin_name, options, description, active, added_certificates, removed_certificates):
     """
     Updates an existing notification.
 
@@ -114,8 +114,9 @@ def update(notification_id, label, plugin_name, options, description, active, ce
     :param options:
     :param description:
     :param active:
-    :param certificates:
+    :param added_certificates:
-    :rtype : Notification
+    :param removed_certificates:
+    :rtype: Notification
     :return:
     """
     notification = get(notification_id)
@@ -125,7 +126,8 @@ def update(notification_id, label, plugin_name, options, description, active, ce
     notification.options = options
     notification.description = description
     notification.active = active
-    notification.certificates = certificates
+    notification.certificates = notification.certificates + added_certificates
+    notification.certificates = [c for c in notification.certificates if c not in removed_certificates]
 
     return database.update(notification)
 
@@ -144,7 +146,7 @@ def get(notification_id):
     Retrieves an notification by its lemur assigned ID.
 
     :param notification_id: Lemur assigned ID
-    :rtype : Notification
+    :rtype: Notification
     :return:
     """
     return database.get(Notification, notification_id)

lemur/notifications/views.py

@@ -117,7 +117,7 @@ class NotificationsList(AuthenticatedResource):
         """
         .. http:post:: /notifications
 
-           Creates a new account
+           Creates a new notification
 
            **Example request**:
 
@@ -126,6 +126,7 @@ class NotificationsList(AuthenticatedResource):
               POST /notifications HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "description": "a test",
@@ -213,9 +214,12 @@ class NotificationsList(AuthenticatedResource):
                 "id": 2
               }
 
-           :arg accountNumber: aws account number
+           :label label: notification name
-           :arg label: human readable account label
+           :label slug: notification plugin slug
-           :arg comments: some description about the account
+           :label plugin_options: notification plugin options
+           :label description: notification description
+           :label active: whether or not the notification is active/enabled
+           :label certificates: certificates to attach to notification
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
@@ -238,7 +242,7 @@ class Notifications(AuthenticatedResource):
         """
         .. http:get:: /notifications/1
 
-           Get a specific account
+           Get a specific notification
 
            **Example request**:
 
@@ -305,15 +309,28 @@ class Notifications(AuthenticatedResource):
         """
         .. http:put:: /notifications/1
 
-           Updates an account
+           Updates a notification
 
            **Example request**:
 
            .. sourcecode:: http
 
-              POST /notifications/1 HTTP/1.1
+              PUT /notifications/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                "label": "labelChanged",
+                "plugin": {
+                    "slug": "email-notification",
+                    "plugin_options": "???"
+                  },
+                "description": "Sample notification",
+                "active": "true",
+                "added_certificates": "???",
+                "removed_certificates": "???"
+              }
 
 
            **Example response**:
@@ -326,14 +343,24 @@ class Notifications(AuthenticatedResource):
 
               {
                 "id": 1,
-                "accountNumber": 11111111111,
                 "label": "labelChanged",
-                "comments": "this is a thing"
+                "plugin": {
+                    "slug": "email-notification",
+                    "plugin_options": "???"
+                  },
+                "description": "Sample notification",
+                "active": "true",
+                "added_certificates": "???",
+                "removed_certificates": "???"
               }
 
-           :arg accountNumber: aws account number
+           :label label: notification name
-           :arg label: human readable account label
+           :label slug: notification plugin slug
-           :arg comments: some description about the account
+           :label plugin_options: notification plugin options
+           :label description: notification description
+           :label active: whether or not the notification is active/enabled
+           :label added_certificates: certificates to add
+           :label removed_certificates: certificates to remove
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
@@ -344,7 +371,8 @@ class Notifications(AuthenticatedResource):
             data["plugin"]["plugin_options"],
             data["description"],
             data["active"],
-            data["certificates"],
+            data["added_certificates"],
+            data["removed_certificates"],
         )
 
     def delete(self, notification_id):

lemur/pending_certificates/cli.py

@@ -12,10 +12,12 @@ from flask import current_app
 from flask_script import Manager
 
 from lemur.authorities.service import get as get_authority
+from lemur.constants import ACME_ADDITIONAL_ATTEMPTS
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 
+
 manager = Manager(usage="Handles pending certificate related tasks.")
 
 
@@ -107,7 +109,7 @@ def fetch_all_acme():
             error_log["last_error"] = cert.get("last_error")
             error_log["cn"] = pending_cert.cn
 
-            if pending_cert.number_attempts > 4:
+            if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
                 error_log["message"] = "Marking pending certificate as resolved"
                 send_pending_failure_notification(
                     pending_cert, notify_owner=pending_cert.notify

lemur/pending_certificates/service.py

@@ -93,11 +93,10 @@ def get_pending_certs(pending_ids):
 def create_certificate(pending_certificate, certificate, user):
     """
     Create and store a certificate with pending certificate's info
-    Args:
+
-        pending_certificate: PendingCertificate which will populate the certificate
+    :arg pending_certificate: PendingCertificate which will populate the certificate
-        certificate: dict from Authority, which contains the body, chain and external id
+    :arg certificate: dict from Authority, which contains the body, chain and external id
-        user: User that called this function, used as 'creator' of the certificate if it does
+    :arg user: User that called this function, used as 'creator' of the certificate if it does not have an owner
-              not have an owner
     """
     certificate["owner"] = pending_certificate.owner
     data, errors = CertificateUploadInputSchema().load(certificate)
@@ -158,9 +157,9 @@ def cancel(pending_certificate, **kwargs):
     """
     Cancel a pending certificate.  A check should be done prior to this function to decide to
     revoke the certificate or just abort cancelling.
-    Args:
+
-        pending_certificate: PendingCertificate to be cancelled
+    :arg pending_certificate: PendingCertificate to be cancelled
-    Returns: the pending certificate if successful, raises Exception if there was an issue
+    :return: the pending certificate if successful, raises Exception if there was an issue
     """
     plugin = plugins.get(pending_certificate.authority.plugin_name)
     plugin.cancel_ordered_certificate(pending_certificate, **kwargs)

lemur/pending_certificates/views.py

@@ -221,9 +221,10 @@ class PendingCertificates(AuthenticatedResource):
 
            .. sourcecode:: http
 
-              PUT /pending certificates/1 HTTP/1.1
+              PUT /pending_certificates/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "owner": "jimbob@example.com",
@@ -337,7 +338,7 @@ class PendingCertificates(AuthenticatedResource):
 
            .. sourcecode:: http
 
-              DELETE /pending certificates/1 HTTP/1.1
+              DELETE /pending_certificates/1 HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
 
@@ -465,6 +466,7 @@ class PendingCertificatesUpload(AuthenticatedResource):
               POST /certificates/1/upload HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                  "body": "-----BEGIN CERTIFICATE-----...",

lemur/plugins/bases/__init__.py

@@ -3,3 +3,4 @@ from .issuer import IssuerPlugin  # noqa
 from .source import SourcePlugin  # noqa
 from .notification import NotificationPlugin, ExpirationNotificationPlugin  # noqa
 from .export import ExportPlugin  # noqa
+from .tls import TLSPlugin  # noqa

lemur/plugins/bases/destination.py

@@ -31,6 +31,11 @@ class ExportDestinationPlugin(DestinationPlugin):
 
     @property
     def options(self):
+        """
+        Gets/sets options for the plugin.
+
+        :return:
+        """
         return self.default_options + self.additional_options
 
     def export(self, body, private_key, cert_chain, options):

lemur/plugins/bases/issuer.py

@@ -23,7 +23,7 @@ class IssuerPlugin(Plugin):
     def create_authority(self, options):
         raise NotImplementedError
 
-    def revoke_certificate(self, certificate, comments):
+    def revoke_certificate(self, certificate, reason):
         raise NotImplementedError
 
     def get_ordered_certificate(self, certificate):
@@ -31,3 +31,12 @@ class IssuerPlugin(Plugin):
 
     def cancel_ordered_certificate(self, pending_cert, **kwargs):
         raise NotImplementedError
+
+    def wrap_certificate(self, cert):
+        pass
+
+    def wrap_auth_certificate(self, cert):
+        pass
+
+    def wrap_private_key(self, cert):
+        pass

lemur/plugins/bases/notification.py

@@ -20,14 +20,14 @@ class NotificationPlugin(Plugin):
     def send(self, notification_type, message, targets, options, **kwargs):
         raise NotImplementedError
 
-    def filter_recipients(self, options, excluded_recipients):
+    def get_recipients(self, options, additional_recipients):
         """
-        Given a set of options (which should include configured recipient info), filters out recipients that
+        Given a set of options (which should include configured recipient info), returns the parsed list of recipients
-        we do NOT want to notify.
+        from those options plus the additional recipients specified. The returned value has no duplicates.
 
-        For any notification types where recipients can't be dynamically modified, this returns an empty list.
+        For any notification types where recipients can't be dynamically modified, this returns only the additional recipients.
         """
-        return []
+        return additional_recipients
 
 
 class ExpirationNotificationPlugin(NotificationPlugin):
@@ -57,6 +57,11 @@ class ExpirationNotificationPlugin(NotificationPlugin):
 
     @property
     def options(self):
+        """
+        Gets/sets options for the plugin.
+
+        :return:
+        """
         return self.default_options + self.additional_options
 
     def send(self, notification_type, message, excluded_targets, options, **kwargs):

lemur/plugins/bases/source.py

@@ -33,4 +33,9 @@ class SourcePlugin(Plugin):
 
     @property
     def options(self):
+        """
+        Gets/sets options for the plugin.
+
+        :return:
+        """
         return self.default_options + self.additional_options

lemur/plugins/bases/tls.py

@@ -0,0 +1,20 @@
+"""
+.. module: lemur.plugins.bases.tls
+    :platform: Unix
+    :copyright: (c) 2021 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Sayali Charhate <scharhate@netflix.com>
+"""
+from lemur.plugins.base import Plugin
+
+
+class TLSPlugin(Plugin):
+    """
+    This is the base class from which all supported
+    tls session providers will inherit from.
+    """
+    type = "tls"
+
+    def session(self, server_application):
+        raise NotImplementedError

lemur/plugins/lemur_acme/acme_handlers.py

@@ -23,6 +23,7 @@ from acme import challenges, errors, messages
 from acme.client import BackwardsCompatibleClientV2, ClientNetwork
 from acme.errors import TimeoutError
 from acme.messages import Error as AcmeError
+from certbot import crypto_util as acme_crypto_util
 from flask import current_app
 
 from lemur.common.utils import generate_private_key
@@ -36,12 +37,13 @@ from retrying import retry
 
 
 class AuthorizationRecord(object):
-    def __init__(self, domain, target_domain, authz, dns_challenge, change_id):
+    def __init__(self, domain, target_domain, authz, dns_challenge, change_id, cname_delegation):
         self.domain = domain
         self.target_domain = target_domain
         self.authz = authz
         self.dns_challenge = dns_challenge
         self.change_id = change_id
+        self.cname_delegation = cname_delegation
 
 
 class AcmeHandler(object):
@@ -70,7 +72,7 @@ class AcmeHandler(object):
             return False
 
     def strip_wildcard(self, host):
-        """Removes the leading *. and returns Host and whether it was removed or not (True/False)"""
+        """Removes the leading wildcard and returns Host and whether it was removed or not (True/False)"""
         prefix = "*."
         if host.startswith(prefix):
             return host[len(prefix):], True
@@ -91,7 +93,8 @@ class AcmeHandler(object):
         deadline = datetime.datetime.now() + datetime.timedelta(seconds=360)
 
         try:
-            orderr = acme_client.poll_and_finalize(order, deadline)
+            orderr = acme_client.poll_authorizations(order, deadline)
+            orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True)
 
         except (AcmeError, TimeoutError):
             sentry.captureException(extra={"order_url": str(order.uri)})
@@ -111,14 +114,23 @@ class AcmeHandler(object):
             f"Successfully resolved Acme order: {order.uri}", exc_info=True
         )
 
-        pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem)
+        pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem,
+                                                                             orderr.alternative_fullchains_pem)
 
         current_app.logger.debug(
             "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain))
         )
         return pem_certificate, pem_certificate_chain
 
-    def extract_cert_and_chain(self, fullchain_pem):
+    def extract_cert_and_chain(self, fullchain_pem, alternative_fullchains_pem, preferred_issuer=None):
+
+        if not preferred_issuer:
+            preferred_issuer = current_app.config.get("ACME_PREFERRED_ISSUER", None)
+        if preferred_issuer:
+            # returns first chain if not match
+            fullchain_pem = acme_crypto_util.find_chain_with_issuer([fullchain_pem] + alternative_fullchains_pem,
+                                                                    preferred_issuer)
+
         pem_certificate = OpenSSL.crypto.dump_certificate(
             OpenSSL.crypto.FILETYPE_PEM,
             OpenSSL.crypto.load_certificate(
@@ -126,11 +138,6 @@ class AcmeHandler(object):
             ),
         ).decode()
 
-        if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \
-                and datetime.datetime.now() < datetime.datetime.strptime(
-                current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'):
-            pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA")
-        else:
         pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip()
 
         return pem_certificate, pem_certificate_chain
@@ -221,17 +228,17 @@ class AcmeHandler(object):
         current_app.logger.debug("Got these domains: {0}".format(domains))
         return domains
 
-    def revoke_certificate(self, certificate):
+    def revoke_certificate(self, certificate, crl_reason=0):
         if not self.reuse_account(certificate.authority):
             raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.")
-        acme_client, _ = self.acme.setup_acme_client(certificate.authority)
+        acme_client, _ = self.setup_acme_client(certificate.authority)
 
         fullchain_com = jose.ComparableX509(
             OpenSSL.crypto.load_certificate(
                 OpenSSL.crypto.FILETYPE_PEM, certificate.body))
 
         try:
-            acme_client.revoke(fullchain_com, 0)  # revocation reason = 0
+            acme_client.revoke(fullchain_com, crl_reason)  # revocation reason as int (per RFC 5280 section 5.3.1)
         except (errors.ConflictError, errors.ClientError, errors.Error) as e:
             # Certificate already revoked.
             current_app.logger.error("Certificate revocation failed with message: " + e.detail)
@@ -305,6 +312,7 @@ class AcmeDnsHandler(AcmeHandler):
         current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.")
 
         change_ids = []
+        cname_delegation = domain != target_domain
         dns_challenges = self.get_dns_challenges(domain, order.authorizations)
         host_to_validate, _ = self.strip_wildcard(target_domain)
         host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
@@ -315,9 +323,7 @@ class AcmeDnsHandler(AcmeHandler):
             raise Exception("Unable to determine DNS challenges from authorizations")
 
         for dns_challenge in dns_challenges:
-
+            if not cname_delegation:
-            # Only prepend '_acme-challenge' if not using CNAME redirection
-            if domain == target_domain:
                 host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
 
             change_id = dns_provider.create_txt_record(
@@ -328,7 +334,7 @@ class AcmeDnsHandler(AcmeHandler):
             change_ids.append(change_id)
 
         return AuthorizationRecord(
-            domain, target_domain, order.authorizations, dns_challenges, change_ids
+            domain, target_domain, order.authorizations, dns_challenges, change_ids, cname_delegation
         )
 
     def complete_dns_challenge(self, acme_client, authz_record):
@@ -395,6 +401,9 @@ class AcmeDnsHandler(AcmeHandler):
                 if cname_result:
                     target_domain = cname_result
                     self.autodetect_dns_providers(target_domain)
+                    metrics.send(
+                        "get_authorizations_cname_delegation_for_domain", "counter", 1, metric_tags={"domain": domain}
+                    )
 
             if not self.dns_providers_for_domain.get(target_domain):
                 metrics.send(
@@ -455,7 +464,7 @@ class AcmeDnsHandler(AcmeHandler):
                     account_number = dns_provider_options.get("account_id")
                     host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
                     host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
-                    if authz_record.domain == authz_record.target_domain:
+                    if not authz_record.cname_delegation:
                         host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate)
                     dns_provider_plugin.delete_txt_record(
                         authz_record.change_id,
@@ -492,7 +501,7 @@ class AcmeDnsHandler(AcmeHandler):
 
                 dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
                 for dns_challenge in dns_challenges:
-                    if authz_record.domain == authz_record.target_domain:
+                    if not authz_record.cname_delegation:
                         host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
                     try:
                         dns_provider_plugin.delete_txt_record(

lemur/plugins/lemur_acme/challenge_types.py

@@ -119,8 +119,10 @@ class AcmeHttpChallenge(AcmeChallenge):
             current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order")
 
         try:
-            finalized_orderr = acme_client.poll_and_finalize(orderr,
+            deadline = datetime.datetime.now() + datetime.timedelta(seconds=90)
-                                                             datetime.datetime.now() + datetime.timedelta(seconds=90))
+            orderr = acme_client.poll_authorizations(orderr, deadline)
+            finalized_orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True)
+
         except errors.ValidationError as validationError:
             for authz in validationError.failed_authzrs:
                 for chall in authz.body.challenges:
@@ -130,7 +132,8 @@ class AcmeHttpChallenge(AcmeChallenge):
                                                                                          ERROR_CODES[chall.error.code]))
             raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.')
 
-        pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem)
+        pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem,
+                                                                                  finalized_orderr.alternative_fullchains_pem)
 
         if len(deployed_challenges) != 0:
             for token_path in deployed_challenges:

lemur/plugins/lemur_acme/plugin.py

@@ -17,6 +17,7 @@ from acme.messages import Error as AcmeError
 from botocore.exceptions import ClientError
 from flask import current_app
 from lemur.authorizations import service as authorization_service
+from lemur.constants import CRLReason
 from lemur.dns_providers import service as dns_provider_service
 from lemur.exceptions import InvalidConfiguration
 from lemur.extensions import metrics, sentry
@@ -267,9 +268,13 @@ class ACMEIssuerPlugin(IssuerPlugin):
         # Needed to override issuer function.
         pass
 
-    def revoke_certificate(self, certificate, comments):
+    def revoke_certificate(self, certificate, reason):
         self.acme = AcmeDnsHandler()
-        return self.acme.revoke_certificate(certificate)
+        crl_reason = CRLReason.unspecified
+        if "crl_reason" in reason:
+            crl_reason = CRLReason[reason["crl_reason"]]
+
+        return self.acme.revoke_certificate(certificate, crl_reason.value)
 
 
 class ACMEHttpIssuerPlugin(IssuerPlugin):
@@ -368,6 +373,11 @@ class ACMEHttpIssuerPlugin(IssuerPlugin):
         # Needed to override issuer function.
         pass
 
-    def revoke_certificate(self, certificate, comments):
+    def revoke_certificate(self, certificate, reason):
         self.acme = AcmeHandler()
-        return self.acme.revoke_certificate(certificate)
+
+        crl_reason = CRLReason.unspecified
+        if "crl_reason" in reason:
+            crl_reason = CRLReason[reason["crl_reason"]]
+
+        return self.acme.revoke_certificate(certificate, crl_reason.value)

lemur/plugins/lemur_acme/tests/test_acme_handler.py

@@ -5,6 +5,12 @@ from flask import Flask
 from cryptography.x509 import DNSName
 from lemur.plugins.lemur_acme import acme_handlers
 
+from lemur.tests.vectors import (
+    ACME_CHAIN_SHORT_STR,
+    ACME_CHAIN_LONG_STR,
+    SAN_CERT_STR,
+)
+
 
 class TestAcmeHandler(unittest.TestCase):
     def setUp(self):
@@ -30,7 +36,7 @@ class TestAcmeHandler(unittest.TestCase):
         self.assertEqual(expected, result)
 
     def test_authz_record(self):
-        a = acme_handlers.AuthorizationRecord("domain", "host", "authz", "challenge", "id")
+        a = acme_handlers.AuthorizationRecord("domain", "host", "authz", "challenge", "id", "cname_delegation")
         self.assertEqual(type(a), acme_handlers.AuthorizationRecord)
 
     def test_setup_acme_client_fail(self):
@@ -110,3 +116,18 @@ class TestAcmeHandler(unittest.TestCase):
         self.assertEqual(
             result, [options["common_name"], "test2.netflix.net"]
         )
+
+    def test_extract_cert_and_chain(self):
+        # expecting the short chain
+        leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR,
+                                                               [ACME_CHAIN_LONG_STR],
+                                                               "(STAGING) Artificial Apricot R3")
+        self.assertEqual(leaf_pem, SAN_CERT_STR)
+        self.assertEqual(chain_pem, ACME_CHAIN_SHORT_STR[len(leaf_pem):].lstrip())
+
+        # expecting the long chain
+        leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR,
+                                                               [ACME_CHAIN_LONG_STR],
+                                                               "(STAGING) Doctored Durian Root CA X3")
+        self.assertEqual(leaf_pem, SAN_CERT_STR)
+        self.assertEqual(chain_pem, ACME_CHAIN_LONG_STR[len(leaf_pem):].lstrip())

lemur/plugins/lemur_acme/tests/test_acme_http.py

@@ -1,8 +1,8 @@
 import unittest
 from unittest.mock import patch, Mock
 
-from flask import Flask
 from acme import challenges
+from flask import Flask
 from lemur.plugins.lemur_acme import plugin
 
 
@@ -51,7 +51,8 @@ class TestAcmeHttp(unittest.TestCase):
         mock_order_resource = Mock()
         mock_order_resource.authorizations = [Mock()]
         mock_order_resource.authorizations[0].body.challenges = [Mock()]
-        mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = (Mock(), "Anything-goes")
+        mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = (
+            Mock(), "Anything-goes")
         mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(
             token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f')
 
@@ -60,8 +61,93 @@ class TestAcmeHttp(unittest.TestCase):
         mock_client.answer_challenge.return_value = True
 
         mock_finalized_order = Mock()
-        mock_finalized_order.fullchain_pem = "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n"
+        mock_finalized_order.fullchain_pem = """
-        mock_client.poll_and_finalize.return_value = mock_finalized_order
+-----BEGIN CERTIFICATE-----
+MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw
+GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2
+MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0
+8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym
+oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0
+ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN
+xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56
+dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9
+AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
+HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0
+BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu
+b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu
+Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq
+hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF
+UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9
+AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp
+DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7
+IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf
+zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI
+PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w
+SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em
+2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0
+WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt
+n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw
+GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2
+MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0
+8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym
+oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0
+ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN
+xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56
+dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9
+AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
+HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0
+BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu
+b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu
+Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq
+hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF
+UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9
+AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp
+DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7
+IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf
+zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI
+PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w
+SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em
+2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0
+WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt
+n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFATCCAumgAwIBAgIRAKc9ZKBASymy5TLOEp57N98wDQYJKoZIhvcNAQELBQAw
+GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDMyMzIyNTM0NloXDTM2
+MDMyMzIyNTM0NlowGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMIICIjANBgkq
+hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+pYHvQw5iU3v2b3iNuYNKYgsWD6KU7aJ
+diddtZQxSWYzUI3U0I1UsRPTxnhTifs/M9NW4ZlV13ZfB7APwC8oqKOIiwo7IwlP
+xg0VKgyz+kT8RJfYr66PPIYP0fpTeu42LpMJ+CKo9sbpgVNDZN2z/qiXrRNX/VtG
+TkPV7a44fZ5bHHVruAxvDnylpQxJobtCBWlJSsbIRGFHMc2z88eUz9NmIOWUKGGj
+EmP76x8OfRHpIpuxRSCjn0+i9+hR2siIOpcMOGd+40uVJxbRRP5ZXnUFa2fF5FWd
+O0u0RPI8HON0ovhrwPJY+4eWKkQzyC611oLPYGQ4EbifRsTsCxUZqyUuStGyp8oa
+aoSKfF6X0+KzGgwwnrjRTUpIl19A92KR0Noo6h622OX+4sZiO/JQdkuX5w/HupK0
+A0M0WSMCvU6GOhjGotmh2VTEJwHHY4+TUk0iQYRtv1crONklyZoAQPD76hCrC8Cr
+IbgsZLfTMC8TWUoMbyUDgvgYkHKMoPm0VGVVuwpRKJxv7+2wXO+pivrrUl2Q9fPe
+Kk055nJLMV9yPUdig8othUKrRfSxli946AEV1eEOhxddfEwBE3Lt2xn0hhiIedbb
+Ftf/5kEWFZkXyUmMJK8Ra76Kus2ABueUVEcZ48hrRr1Hf1N9n59VbTUaXgeiZA50
+qXf2bymE6F8CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
+Af8wHQYDVR0OBBYEFMEmdKSKRKDm+iAo2FwjmkWIGHngMA0GCSqGSIb3DQEBCwUA
+A4ICAQBCPw74M9X/Xx04K1VAES3ypgQYH5bf9FXVDrwhRFSVckria/7dMzoF5wln
+uq9NGsjkkkDg17AohcQdr8alH4LvPdxpKr3BjpvEcmbqF8xH+MbbeUEnmbSfLI8H
+sefuhXF9AF/9iYvpVNC8FmJ0OhiVv13VgMQw0CRKkbtjZBf8xaEhq/YqxWVsgOjm
+dm5CAQ2X0aX7502x8wYRgMnZhA5goC1zVWBVAi8yhhmlhhoDUfg17cXkmaJC5pDd
+oenZ9NVhW8eDb03MFCrWNvIh89DDeCGWuWfDltDq0n3owyL0IeSn7RfpSclpxVmV
+/53jkYjwIgxIG7Gsv0LKMbsf6QdBcTjhvfZyMIpBRkTe3zuHd2feKzY9lEkbRvRQ
+zbh4Ps5YBnG6CKJPTbe2hfi3nhnw/MyEmF3zb0hzvLWNrR9XW3ibb2oL3424XOwc
+VjrTSCLzO9Rv6s5wi03qoWvKAQQAElqTYRHhynJ3w6wuvKYF5zcZF3MDnrVGLbh1
+Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4
+8iyIYUyQAbsvx8oD2M8kRvrIRSrRJSl6L957b4AFiLIQ/GgV2curs0jje7Edx34c
+idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng==
+-----END CERTIFICATE-----
+"""
+        mock_finalized_order.alternative_fullchains_pem = [mock_finalized_order.fullchain_pem]
+        mock_client.finalize_order.return_value = mock_finalized_order
 
         mock_acme.return_value = (mock_client, "")
 
@@ -84,8 +170,63 @@ class TestAcmeHttp(unittest.TestCase):
         pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options)
 
         self.assertEqual(pem_certificate, "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n")
-        self.assertEqual(pem_certificate_chain,
+        self.assertEqual(pem_certificate_chain, """-----BEGIN CERTIFICATE-----
-                         "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n")
+MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw
+GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2
+MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0
+8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym
+oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0
+ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN
+xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56
+dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9
+AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
+HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0
+BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu
+b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu
+Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq
+hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF
+UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9
+AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp
+DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7
+IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf
+zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI
+PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w
+SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em
+2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0
+WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt
+n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFATCCAumgAwIBAgIRAKc9ZKBASymy5TLOEp57N98wDQYJKoZIhvcNAQELBQAw
+GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDMyMzIyNTM0NloXDTM2
+MDMyMzIyNTM0NlowGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMIICIjANBgkq
+hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+pYHvQw5iU3v2b3iNuYNKYgsWD6KU7aJ
+diddtZQxSWYzUI3U0I1UsRPTxnhTifs/M9NW4ZlV13ZfB7APwC8oqKOIiwo7IwlP
+xg0VKgyz+kT8RJfYr66PPIYP0fpTeu42LpMJ+CKo9sbpgVNDZN2z/qiXrRNX/VtG
+TkPV7a44fZ5bHHVruAxvDnylpQxJobtCBWlJSsbIRGFHMc2z88eUz9NmIOWUKGGj
+EmP76x8OfRHpIpuxRSCjn0+i9+hR2siIOpcMOGd+40uVJxbRRP5ZXnUFa2fF5FWd
+O0u0RPI8HON0ovhrwPJY+4eWKkQzyC611oLPYGQ4EbifRsTsCxUZqyUuStGyp8oa
+aoSKfF6X0+KzGgwwnrjRTUpIl19A92KR0Noo6h622OX+4sZiO/JQdkuX5w/HupK0
+A0M0WSMCvU6GOhjGotmh2VTEJwHHY4+TUk0iQYRtv1crONklyZoAQPD76hCrC8Cr
+IbgsZLfTMC8TWUoMbyUDgvgYkHKMoPm0VGVVuwpRKJxv7+2wXO+pivrrUl2Q9fPe
+Kk055nJLMV9yPUdig8othUKrRfSxli946AEV1eEOhxddfEwBE3Lt2xn0hhiIedbb
+Ftf/5kEWFZkXyUmMJK8Ra76Kus2ABueUVEcZ48hrRr1Hf1N9n59VbTUaXgeiZA50
+qXf2bymE6F8CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
+Af8wHQYDVR0OBBYEFMEmdKSKRKDm+iAo2FwjmkWIGHngMA0GCSqGSIb3DQEBCwUA
+A4ICAQBCPw74M9X/Xx04K1VAES3ypgQYH5bf9FXVDrwhRFSVckria/7dMzoF5wln
+uq9NGsjkkkDg17AohcQdr8alH4LvPdxpKr3BjpvEcmbqF8xH+MbbeUEnmbSfLI8H
+sefuhXF9AF/9iYvpVNC8FmJ0OhiVv13VgMQw0CRKkbtjZBf8xaEhq/YqxWVsgOjm
+dm5CAQ2X0aX7502x8wYRgMnZhA5goC1zVWBVAi8yhhmlhhoDUfg17cXkmaJC5pDd
+oenZ9NVhW8eDb03MFCrWNvIh89DDeCGWuWfDltDq0n3owyL0IeSn7RfpSclpxVmV
+/53jkYjwIgxIG7Gsv0LKMbsf6QdBcTjhvfZyMIpBRkTe3zuHd2feKzY9lEkbRvRQ
+zbh4Ps5YBnG6CKJPTbe2hfi3nhnw/MyEmF3zb0hzvLWNrR9XW3ibb2oL3424XOwc
+VjrTSCLzO9Rv6s5wi03qoWvKAQQAElqTYRHhynJ3w6wuvKYF5zcZF3MDnrVGLbh1
+Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4
+8iyIYUyQAbsvx8oD2M8kRvrIRSrRJSl6L957b4AFiLIQ/GgV2curs0jje7Edx34c
+idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng==
+-----END CERTIFICATE-----
+""")
 
     @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
     @patch("lemur.plugins.base.manager.PluginManager.get")

lemur/plugins/lemur_adcs/plugin.py

@@ -59,8 +59,8 @@ class ADCSIssuerPlugin(IssuerPlugin):
         )
         return cert, chain, None
 
-    def revoke_certificate(self, certificate, comments):
+    def revoke_certificate(self, certificate, reason):
-        raise NotImplementedError("Not implemented\n", self, certificate, comments)
+        raise NotImplementedError("Not implemented\n", self, certificate, reason)
 
     def get_ordered_certificate(self, order_id):
         raise NotImplementedError("Not implemented\n", self, order_id)
@@ -77,15 +77,6 @@ class ADCSSourcePlugin(SourcePlugin):
 
     author = "sirferl"
     author_url = "https://github.com/sirferl/lemur"
-    options = [
-        {
-            "name": "dummy",
-            "type": "str",
-            "required": False,
-            "validation": "/^[0-9]{12,12}$/",
-            "helpMessage": "Just to prevent error",
-        }
-    ]
 
     def get_certificates(self, options, **kwargs):
         adcs_server = current_app.config.get("ADCS_SERVER")

lemur/plugins/lemur_aws/elb.py

@@ -149,6 +149,38 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
         raise
 
 
+@sts_client("elbv2")
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=5)
+def get_load_balancer_arn_from_endpoint(endpoint_name, **kwargs):
+    """
+    Get a load balancer ARN from an endpoint.
+    :param endpoint_name:
+    :return:
+    """
+    try:
+        client = kwargs.pop("client")
+        elbs = client.describe_load_balancers(Names=[endpoint_name])
+        if "LoadBalancers" in elbs and elbs["LoadBalancers"]:
+            return elbs["LoadBalancers"][0]["LoadBalancerArn"]
+
+    except Exception as e:  # noqa
+        metrics.send(
+            "get_load_balancer_arn_from_endpoint",
+            "counter",
+            1,
+            metric_tags={
+                "error": str(e),
+                "endpoint_name": endpoint_name,
+            },
+        )
+        sentry.captureException(
+            extra={
+                "endpoint_name": str(endpoint_name),
+            }
+        )
+        raise
+
+
 @sts_client("elb")
 @retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
 def get_elbs(**kwargs):

lemur/plugins/lemur_aws/plugin.py

@@ -300,6 +300,41 @@ class AWSSourcePlugin(SourcePlugin):
             )
         return None
 
+    def get_endpoint_certificate_names(self, endpoint):
+        options = endpoint.source.options
+        account_number = self.get_option("accountNumber", options)
+        region = get_region_from_dns(endpoint.dnsname)
+        certificate_names = []
+
+        if endpoint.type == "elb":
+            elb_details = elb.get_elbs(account_number=account_number,
+                                    region=region,
+                                    LoadBalancerNames=[endpoint.name],)
+
+            for lb_description in elb_details["LoadBalancerDescriptions"]:
+                for listener_description in lb_description["ListenerDescriptions"]:
+                    listener = listener_description.get("Listener")
+                    if not listener.get("SSLCertificateId"):
+                        continue
+
+                    certificate_names.append(iam.get_name_from_arn(listener.get("SSLCertificateId")))
+        elif endpoint.type == "elbv2":
+            listeners = elb.describe_listeners_v2(
+                account_number=account_number,
+                region=region,
+                LoadBalancerArn=elb.get_load_balancer_arn_from_endpoint(endpoint.name,
+                                                                        account_number=account_number,
+                                                                        region=region),
+            )
+            for listener in listeners["Listeners"]:
+                if not listener.get("Certificates"):
+                    continue
+
+                for certificate in listener["Certificates"]:
+                    certificate_names.append(iam.get_name_from_arn(certificate["CertificateArn"]))
+
+        return certificate_names
+
 
 class AWSDestinationPlugin(DestinationPlugin):
     title = "AWS"
@@ -344,6 +379,10 @@ class AWSDestinationPlugin(DestinationPlugin):
     def deploy(self, elb_name, account, region, certificate):
         pass
 
+    def clean(self, certificate, options, **kwargs):
+        account_number = self.get_option("accountNumber", options)
+        iam.delete_cert(certificate.name, account_number=account_number)
+
 
 class S3DestinationPlugin(ExportDestinationPlugin):
     title = "AWS-S3"
@@ -412,6 +451,7 @@ class S3DestinationPlugin(ExportDestinationPlugin):
     def upload_acme_token(self, token_path, token, options, **kwargs):
         """
         This is called from the acme http challenge
+
         :param self:
         :param token_path:
         :param token:
@@ -524,4 +564,4 @@ class SNSNotificationPlugin(ExpirationNotificationPlugin):
                     f"{self.get_option('topicName', options)}"
 
         current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}")
-        sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options))
+        sns.publish(topic_arn, message, notification_type, options, region_name=self.get_option("region", options))

lemur/plugins/lemur_aws/sns.py

@@ -11,21 +11,24 @@ import arrow
 import boto3
 from flask import current_app
 
+from lemur.plugins.bases import ExpirationNotificationPlugin
 
-def publish(topic_arn, certificates, notification_type, **kwargs):
+
+def publish(topic_arn, certificates, notification_type, options, **kwargs):
     sns_client = boto3.client("sns", **kwargs)
     message_ids = {}
     subject = "Lemur: {0} Notification".format(notification_type.capitalize())
     for certificate in certificates:
-        message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject)
+        message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type,
+                                                          subject, options)
 
     return message_ids
 
 
-def publish_single(sns_client, topic_arn, certificate, notification_type, subject):
+def publish_single(sns_client, topic_arn, certificate, notification_type, subject, options):
     response = sns_client.publish(
         TopicArn=topic_arn,
-        Message=format_message(certificate, notification_type),
+        Message=format_message(certificate, notification_type, options),
         Subject=subject,
     )
 
@@ -46,13 +49,30 @@ def create_certificate_url(name):
     )
 
 
-def format_message(certificate, notification_type):
+def format_message(certificate, notification_type, options):
     json_message = {
         "notification_type": notification_type,
         "certificate_name": certificate["name"],
+        "issuer": certificate["issuer"],
+        "id": certificate["id"],
         "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),  # 2047-12-31T22:00:00
         "endpoints_detected": len(certificate["endpoints"]),
         "owner": certificate["owner"],
         "details": create_certificate_url(certificate["name"])
     }
+    if notification_type == "expiration":
+        json_message["notification_interval_days"] = calculate_expiration_days(options)
     return json.dumps(json_message)
+
+
+def calculate_expiration_days(options):
+    unit = ExpirationNotificationPlugin.get_option("unit", options)
+    interval = ExpirationNotificationPlugin.get_option("interval", options)
+    if unit == "weeks":
+        return interval * 7
+
+    elif unit == "months":
+        return interval * 30
+
+    elif unit == "days":
+        return interval

lemur/plugins/lemur_aws/sts.py

@@ -20,6 +20,12 @@ def sts_client(service, service_type="client"):
     def decorator(f):
         @wraps(f)
         def decorated_function(*args, **kwargs):
+            if current_app.config.get("LEMUR_AWS_REGION"):
+                deployment_region = current_app.config.get("LEMUR_AWS_REGION")
+                sts = boto3.client('sts', region_name=deployment_region,
+                                   endpoint_url=f"https://sts.{deployment_region}.amazonaws.com/",
+                                   config=config)
+            else:
                 sts = boto3.client("sts", config=config)
             arn = "arn:aws:iam::{0}:role/{1}".format(
                 kwargs.pop("account_number"),

lemur/plugins/lemur_aws/tests/test_elb.py

@@ -1,5 +1,5 @@
 import boto3
-from moto import mock_sts, mock_elb
+from moto import mock_sts, mock_ec2, mock_elb, mock_elbv2, mock_iam
 
 
 @mock_sts()
@@ -27,3 +27,107 @@ def test_get_all_elbs(app, aws_credentials):
 
     elbs = get_all_elbs(account_number="123456789012", region="us-east-1")
     assert elbs
+
+
+@mock_sts()
+@mock_ec2
+@mock_elbv2()
+@mock_iam
+def test_create_elb_with_https_listener_miscellaneous(app, aws_credentials):
+    from lemur.plugins.lemur_aws import iam, elb
+    endpoint_name = "example-lbv2"
+    account_number = "123456789012"
+    region_ue1 = "us-east-1"
+
+    client = boto3.client("elbv2", region_name="us-east-1")
+    ec2 = boto3.resource("ec2", region_name="us-east-1")
+
+    # Create VPC
+    vpc = ec2.create_vpc(CidrBlock="172.28.7.0/24")
+
+    # Create LB (elbv2) in above VPC
+    assert create_load_balancer(client, ec2, vpc.id, endpoint_name)
+    # Create target group
+    target_group_arn = create_target_group(client, vpc.id)
+    assert target_group_arn
+
+    # Test get_load_balancer_arn_from_endpoint
+    lb_arn = elb.get_load_balancer_arn_from_endpoint(endpoint_name,
+                                                     account_number=account_number,
+                                                     region=region_ue1)
+    assert lb_arn
+
+    # Test describe_listeners_v2
+    listeners = elb.describe_listeners_v2(account_number=account_number,
+                                          region=region_ue1,
+                                          LoadBalancerArn=lb_arn)
+    assert listeners
+    assert not listeners["Listeners"]
+
+    # Upload cert
+    response = iam.upload_cert("LemurTestCert", "testCert", "cert1", "cert2",
+                               account_number=account_number)
+    assert response
+    cert_arn = response["ServerCertificateMetadata"]["Arn"]
+    assert cert_arn
+
+    # Create https listener using above cert
+    listeners = client.create_listener(
+        LoadBalancerArn=lb_arn,
+        Protocol="HTTPS",
+        Port=443,
+        Certificates=[{"CertificateArn": cert_arn}],
+        DefaultActions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
+    )
+    assert listeners
+    listener_arn = listeners["Listeners"][0]["ListenerArn"]
+    assert listener_arn
+
+    assert listeners["Listeners"]
+    for listener in listeners["Listeners"]:
+        if listener["Port"] == 443:
+            assert listener["Certificates"]
+            assert cert_arn == listener["Certificates"][0]["CertificateArn"]
+
+    # Test get_listener_arn_from_endpoint
+    assert listener_arn == elb.get_listener_arn_from_endpoint(
+        endpoint_name,
+        443,
+        account_number=account_number,
+        region=region_ue1,
+    )
+
+
+@mock_sts()
+@mock_elb()
+def test_get_all_elbs_v2():
+    from lemur.plugins.lemur_aws.elb import get_all_elbs_v2
+
+    elbs = get_all_elbs_v2(account_number="123456789012",
+                           region="us-east-1")
+    assert elbs
+
+
+def create_load_balancer(client, ec2, vpc_id, endpoint_name):
+    subnet1 = ec2.create_subnet(
+        VpcId=vpc_id,
+        CidrBlock="172.28.7.192/26",
+        AvailabilityZone="us-east-1a"
+    )
+
+    return client.create_load_balancer(
+        Name=endpoint_name,
+        Subnets=[
+            subnet1.id,
+        ],
+    )
+
+
+def create_target_group(client, vpc_id):
+    response = client.create_target_group(
+        Name="a-target",
+        Protocol="HTTPS",
+        Port=443,
+        VpcId=vpc_id,
+    )
+    return response.get("TargetGroups")[0]["TargetGroupArn"]

lemur/plugins/lemur_aws/tests/test_sns.py

@@ -13,19 +13,44 @@ from lemur.tests.test_messaging import verify_sender_email
 
 
 @mock_sns()
-def test_format(certificate, endpoint):
+def test_format_nonexpiration(certificate, endpoint):
     data = [certificate_notification_output_schema.dump(certificate).data]
 
+    for certificate in data:
+        expected_message = {
+            "notification_type": "not-expiration",
+            "certificate_name": certificate["name"],
+            "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
+            "issuer": certificate["issuer"],
+            "id": certificate["id"],
+            "endpoints_detected": 0,
+            "owner": certificate["owner"],
+            "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
+        }
+        # We don't currently support any SNS notifications besides expiration;
+        # when we do, this test will probably need to be refactored.
+        # For now, this is a placeholder proving empty options works as long as it's not "expiration" type
+        assert expected_message == json.loads(format_message(certificate, "not-expiration", None))
+
+
+@mock_sns()
+def test_format_expiration(certificate, endpoint):
+    data = [certificate_notification_output_schema.dump(certificate).data]
+    options = get_options()
+
     for certificate in data:
         expected_message = {
             "notification_type": "expiration",
             "certificate_name": certificate["name"],
             "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
+            "issuer": certificate["issuer"],
+            "id": certificate["id"],
             "endpoints_detected": 0,
             "owner": certificate["owner"],
-            "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
+            "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]),
+            "notification_interval_days": 10  # 10 days specified in options
         }
-        assert expected_message == json.loads(format_message(certificate, "expiration"))
+        assert expected_message == json.loads(format_message(certificate, "expiration", options))
 
 
 @mock_sns()
@@ -50,7 +75,7 @@ def test_publish(certificate, endpoint):
 
     topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic()
 
-    message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1")
+    message_ids = publish(topic_arn, data, "expiration", get_options(), region_name="us-east-1")
     assert len(message_ids) == len(data)
     received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
 
@@ -59,7 +84,7 @@ def test_publish(certificate, endpoint):
         actual_message = next(
             (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None)
         actual_json = json.loads(actual_message["Body"])
-        assert actual_json["Message"] == format_message(certificate, "expiration")
+        assert actual_json["Message"] == format_message(certificate, "expiration", get_options())
         assert actual_json["Subject"] == "Lemur: Expiration Notification"
 
 
@@ -96,7 +121,8 @@ def test_send_expiration_notification():
 
     received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
     assert len(received_messages) == 1
-    expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration")
+    expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration",
+                                      notification.options)
     actual_message = json.loads(received_messages[0]["Body"])["Message"]
     assert actual_message == expected_message