add OpenSSH plugin

release 0.9.0 changelog

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+include = lemur/*.py
+omit = lemur/migrations/*
+       lemur/tests/*
+

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# These owners will be the default owners for everything in the repo.
+*                            @hosseinsh @csine-nflx @charhate @jtschladen

.github/dependabot.yml

@@ -0,0 +1,15 @@
+  version: 2
+  updates:
+    - directory: "/"
+      schedule:
+        interval: "weekly"
+        day: "monday"
+        time: "08:00"
+        timezone: "America/Los_Angeles"
+      package-ecosystem: "pip"
+      reviewers:
+        - "hosseinsh"
+        - "csine-nflx"
+        - "charhate"
+        - "jtschladen"
+      versioning-strategy: lockfile-only

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '15 16 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    
+    # Install prerequisites for python-ldap. See: https://www.python-ldap.org/en/python-ldap-3.3.0/installing.html#build-prerequisites
+    - name: Install python-ldap prerequisites
+      run: sudo apt-get install libldap2-dev libsasl2-dev
+    
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dependabot-auto-merge.yml

@@ -0,0 +1,14 @@
+name: dependabot-auto-merge
+
+on:
+  pull_request:
+
+jobs:
+  auto-merge:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ahmadnassri/action-dependabot-auto-merge@v2
+        with:
+          target: minor
+          github-token: ${{ secrets.DEPENDABOT_GITHUB_TOKEN }}

.github/workflows/lemur-publish-release-pypi.yml

@@ -0,0 +1,41 @@
+# This workflow will upload a Python Package using Twine when a Lemur release is created via github
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Publish Lemur's latest package to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Autobump version
+      run: |
+        # from refs/tags/v0.8.1 get 0.8.1
+        VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+        PLACEHOLDER='__version__ = "develop"'
+        VERSION_FILE='lemur/__about__.py'
+        # in case placeholder is missing, exists with code 1 and github actions aborts the build
+        grep "$PLACEHOLDER" "$VERSION_FILE"
+        sed -i "s/$PLACEHOLDER/__version__ = \"${VERSION}\"/g" "$VERSION_FILE"
+      shell: bash
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.LEMUR_PYPI_API_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.LEMUR_PYPI_API_TOKEN }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.gitignore

@@ -1,3 +1,4 @@
+/.cache
 .coverage
 .tox
 .DS_Store
@@ -9,9 +10,11 @@
 *.db
 *.pid
 *.enc
+*.env
 MANIFEST
 test.conf
 pip-log.txt
+package-lock.json
 /htmlcov
 /cover
 /build
@@ -23,8 +26,17 @@ pip-log.txt
 /lemur/static/dist/
 /lemur/static/app/vendor/
 /wheelhouse
+/lemur/lib
+/lemur/bin
+/lemur/lib64
+/lemur/include
+
 docs/_build
 .editorconfig
 .idea
-test.conf
-lemur/tests/tmp
+.pytest_cache
+lemur/tests/tmp
+
+/lemur/plugins/lemur_email/tests/expiration-rendered.html
+/lemur/plugins/lemur_email/tests/rotation-rendered.html
+.celerybeat-schedule

.jshintrc

@@ -8,7 +8,7 @@
   "eqeqeq": true,
   "immed": true,
   "indent": 2,
-  "latedef": true,
+  "latedef": false,
   "newcap": false,
   "noarg": true,
   "quotmark": "single",
@@ -22,6 +22,8 @@
     "angular": false,
     "moment": false,
     "toaster": false,
+    "d3": false,
+    "self": false,
     "_": false
   }
 }

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: v0.9.1
+    hooks:
+    -   id: trailing-whitespace
+    -   id: flake8
+    -   id: check-merge-conflict
+-   repo: git://github.com/pre-commit/mirrors-jshint
+    sha: v2.9.5
+    hooks:
+    -   id: jshint
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.7
+
+-   repo: local
+    hooks:
+    -   id: python-bandit-vulnerability-check
+        name: bandit
+        entry: bandit
+        args: ['--ini', 'tox.ini', '-r', 'consoleme']
+        language: system
+        pass_filenames: false

.readthedocs.yml

@@ -0,0 +1,23 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/conf.py
+   fail_on_warning: true
+
+# Build docs in all formats (html, pdf, epub)
+formats: all
+
+# Set the version of Python and requirements required to build the docs
+python:
+   version: 3.7
+   install:
+   - requirements: requirements-docs.txt
+   - method: setuptools
+     path: .
+   system_packages: true

.travis.yml

@@ -1,18 +1,50 @@
-sudo: false
+node_js:
+  - "10"
 
-language: python
-
-addons:
-  postgresql: "9.4"
-
-matrix:
+jobs:
   include:
-    - python: "2.7"
-      env: TOXENV=py27
-    - python: "3.3"
-      env: TOXENV=py33
-    - python: "3.4"
-      env: TOXENV=py34
+    - name: "python3.7-postgresql-9.4-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
+      env: TOXENV=py37
+      addons:
+        postgresql: "9.4"
+        chrome: stable
+      services:
+        - xvfb
+    - name: "python3.7-postgresql-10-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
+      env: TOXENV=py37
+      addons:
+        postgresql: '10'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-10
+            - postgresql-client-10
+            - postgresql-server-dev-10
+      services:
+        - postgresql
+        - xvfb
+    - name: "python3.8-postgresql-12-focal"
+      dist: focal
+      language: python
+      python: "3.8"
+      env: TOXENV=py38
+      addons:
+        postgresql: '12'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-12
+            - postgresql-client-12
+            - postgresql-server-dev-12
+      services:
+        - postgresql
+        - xvfb
 
 cache:
   directories:
@@ -22,15 +54,46 @@ cache:
 env:
   global:
     - PIP_DOWNLOAD_CACHE=".pip_download_cache"
+    # The following line is a temporary workaround for this issue: https://github.com/pypa/setuptools/issues/2230
+    - SETUPTOOLS_USE_DISTUTILS=stdlib
+    # do not load /etc/boto.cfg with Python 3 incompatible plugin
+    # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
+    - BOTO_CONFIG=/doesnotexist
+
+before_install:
+  - export CHROME_BIN=/usr/bin/google-chrome
 
 before_script:
+  - sudo systemctl stop postgresql
+  # the port may have been auto-configured to use 5433 if it thought 5422 was already in use,
+  # for some reason it happens very often
+  # https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_setup_postgresql.bash#L52
+  - sudo sed -i -e 's/5433/5432/' /etc/postgresql/*/main/postgresql.conf
+  - sudo systemctl restart postgresql
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
-  - npm install -g bower
+  - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
+  - npm config set registry https://registry.npmjs.org
+  - npm install -g npm@latest bower
+  - pip install --upgrade setuptools
+  - export DISPLAY=:99.0
+
+install:
+  - pip install coveralls
+  - pip install bandit
 
 script:
   - make test
+  - bandit -r . -ll -ii -x lemur/tests/,docs
+  - make test-js
+
+after_success:
+  - coveralls
 
 notifications:
   email:
-    kglisson@netflix.com
+    recipients:
+      - lemur@netflix.com
+    on_success: never
+    on_failure: always
+    on_cancel: never # Dependbot cancels Travis before rebase and triggers too many emails

CHANGELOG.rst

@@ -1,15 +1,331 @@
 Changelog
 =========
 
+0.9.0 - `2021-03-17`
+~~~~~~~~~~~~~~~~~~~~
 
-0.2.1 - `master` _
+This release fixes three critical vulnerabilities where an authenticated user could retrieve/access
+unauthorized information. (Issue `#3463 <https://github.com/Netflix/lemur/issues/3463>`_)
+
+
+0.8.1 - `2021-03-12`
+~~~~~~~~~~~~~~~~~~~~
+
+This release includes improvements on many fronts, such as:
+
+- Notifications:
+    - Enhanced SNS flow
+    - Expiration Summary
+    - CA expiration email
+- EC algorithm as the default
+- Improved revocation flow
+- Localized AWS STS option
+- Improved Lemur doc building
+- ACME:
+    - reduced failed attempts to 3x trials
+    - support for selecting the chain (Let's Encrypt X1 transition)
+    - revocation
+    - http01 documentation
+- Entrust:
+    - Support for cross-signed intermediate CA
+- Revised disclosure process
+- Dependency updates and conflict resolutions
+
+Special thanks to all who contributed to this release, notably:
+
+- `peschmae  <https://github.com/peschmae>`_
+- `atugushev  <https://github.com/atugushev>`_
+- `sirferl   <https://github.com/sirferl>`_
+
+
+
+0.8.0 - `2020-11-13`
+~~~~~~~~~~~~~~~~~~~~
+
+This release comes after more than two years and contains many interesting new features and improvements.
+In addition to multiple new plugins, such as ACME-http01, ADCS, PowerDNS, UltraDNS, Entrust, SNS, many of Lemur's existing
+flows have improved.
+
+In the future, we plan to do frequent releases.
+
+
+Summary of notable changes:
+
+- AWS S3 plugin: added delete, get methods, and support for uploading/deleting acme tokens
+- ACME plugin:
+    - revamp of the plugin
+    - support for http01 domain validation, via S3 and SFTP as destination for the acme token
+    - support for CNAME delegated domain validation
+    - store-acme-account-details
+- PowerDNS plugin
+- UltraDNS plugin
+- ADCS plugin
+- SNS plugin
+- Entrust plugin
+- Rotation:
+    - respecting keyType and extensions
+    - region-by-region rotation option
+    - default to auto-rotate when cert attached to endpoint
+    - default to 1y validity during rotation for multi-year browser-trusted certs
+- Certificate: search_by_name, and important performance improvements
+- UI
+    - reducing the EC curve options to the relevant ones
+    - edit option for notifications, destinations and sources
+    - showing 13 month validity as default
+    - option to hide certs expired since 3month
+    - faster Permalink (no search involved)
+    - commonName Auto Added as DNS in the UI
+    - improved search and cert lookup
+- celery tasks instead of crone, for better logging and monitoring
+- countless bugfixes
+    - group-lookup-fix-referral
+    - url_context_path
+    - duplicate notification
+    - digicert-time-bug-fix
+    - improved-csr-support
+    - fix-cryptography-intermediate-ca
+    - enhanced logging
+    - vault-k8s-auth
+    - cfssl-key-fix
+    - cert-sync-endpoint-find-by-hash
+    - nlb-naming-bug
+    - fix_vault_api_v2_append
+    - aid_openid_roles_provider_integration
+    - rewrite-java-keystore-use-pyjks
+    - vault_kv2
+
+
+To see the full list of changes, you can run
+
+    $ git log --merges --first-parent master         --pretty=format:"%h %<(10,trunc)%aN %C(white)%<(15)%ar%Creset %C(red bold)%<(15)%D%Creset %s" | grep -v "depend"
+
+
+Special thanks to all who contributed to this release, notably:
+
+- `peschmae  <https://github.com/peschmae>`_
+- `sirferl   <https://github.com/sirferl>`_
+- `lukasmrtvy  <https://github.com/lukasmrtvy>`_
+- `intgr  <https://github.com/intgr>`_
+- `kush-bavishi  <https://github.com/kush-bavishi>`_
+- `alwaysjolley  <https://github.com/alwaysjolley>`_
+- `jplana <https://github.com/jplana>`_
+- `explody <https://github.com/explody>`_
+- `titouanc <https://github.com/titouanc>`_
+- `jramosf <https://github.com/jramosf>`_
+
+
+Upgrading
+---------
+
+.. note:: This release will need a migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+
+0.7 - `2018-05-07`
 ~~~~~~~~~~~~~~~~~~
 
-.. note:: This version not yet released and is under active development
+This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality.
+The linux_dst plugin will also be deprecated and removed.
+
+The pending_dns_authorizations and dns_providers tables were created. New columns
+were added to the certificates and pending_certificates tables, (For the DNS provider ID), and authorities (For options).
+Please run a database migration when upgrading.
+
+The Let's Encrypt flow will run asynchronously. When a certificate is requested through the acme-issuer, a pending certificate
+will be created. A cron needs to be defined to run `lemur pending_certs fetch_all_acme`. This command will iterate through all of the pending
+certificates, request a DNS challenge token from Let's Encrypt, and set the appropriate _acme-challenge TXT entry. It will
+then iterate through and resolve the challenges before requesting a certificate for each pending certificate. If a certificate
+is successfully obtained, the pending_certificate will be moved to the certificates table with the appropriate properties.
+
+Special thanks to all who helped with this release, notably:
+
+- The folks at Cloudflare
+- dmitryzykov
+- jchuong
+- seils
+- titouanc
+
+
+Upgrading
+---------
+
+.. note:: This release will need a migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+0.6 - `2018-01-02`
+~~~~~~~~~~~~~~~~~~
+
+Happy Holidays! This is a big release with lots of bug fixes and features. Below are the highlights and are not exhaustive.
+
+
+Features:
+
+* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used.
+* Adds per-user API Keys, allows users to issue multiple long-lived API tokens with the same permission as the user creating them.
+* Adds the ability to revoke certificates from the Lemur UI/API, this is currently only supported for the digicert CIS and cfssl plugins.
+* Allow destinations to support an export function. Useful for file system destinations e.g. S3 to specify the export plugin you wish to run before being sent to the destination.
+* Adds support for uploading certificates to Cloudfront.
+* Re-worked certificate metadata pane for improved readability.
+* Adds support for LDAP user authentication
+
+Bugs:
+
+* Closed `#767 <https://github.com/Netflix/lemur/issues/767>`_ - Fixed issue with login redirect loop.
+* Closed `#792 <https://github.com/Netflix/lemur/issues/792>`_ - Fixed an issue with a unique constraint was violated when replacing certificates.
+* Closed `#752 <https://github.com/Netflix/lemur/issues/752>`_ - Fixed an internal server error when validating notification units.
+* Closed `#684 <https://github.com/Netflix/lemur/issues/684>`_ - Fixed migration failure when null values encountered.
+* Closes `#661 <https://github.com/Netflix/lemur/issues/661>`_ - Fixed an issue where default values were missing during clone operations.
+
+
+Special thanks to all who helped with this release, notably:
+
+- intgr
+- SecurityInsanity
+- johanneslange
+- RickB17
+- pr8kerl
+- bunjiboys
+
+See the full list of issues closed in `0.6 <https://github.com/Netflix/lemur/milestone/5>`_.
+
+Upgrading
+---------
+
+.. note:: This release will need a migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+
+0.5 - `2016-04-08`
+~~~~~~~~~~~~~~~~~~
+
+This release is most notable for dropping support for python2.7. All Lemur versions >0.4 will now support python3.5 only.
+
+Big thanks to neilschelly for quite a lot of improvements to the `lemur-cryptography` plugin.
+
+Other Highlights:
+
+* Closed `#501 <https://github.com/Netflix/lemur/issues/501>`_ - Endpoint resource as now kept in sync via an expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never removed from Lemur.
+* Closed `#551 <https://github.com/Netflix/lemur/pull/551>`_ - Added the ability to create a 4096 bit key during certificate creation. Closed `#528 <https://github.com/Netflix/lemur/pull/528>`_ to ensure that issuer plugins supported the new 4096 bit keys.
+* Closed `#566 <https://github.com/Netflix/lemur/issues/566>`_ - Fixed an issue changing the notification status for  certificates without private keys.
+* Closed `#594 <https://github.com/Netflix/lemur/issues/594>`_ - Added `replaced` field indicating if a certificate has been superseded.
+* Closed `#602 <https://github.com/Netflix/lemur/issues/602>`_ - AWS plugin added support for ALBs for endpoint tracking.
+
+
+Special thanks to all who helped with this release, notably:
+
+- RcRonco
+- harmw
+- jeremyguarini
+
+See the full list of issues closed in `0.5 <https://github.com/Netflix/lemur/milestone/4>`_.
+
+Upgrading
+---------
+
+.. note:: This release will need a slight migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+0.4 - `2016-11-17`
+~~~~~~~~~~~~~~~~~~
+
+There have been quite a few issues closed in this release. Some notables:
+
+* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for future enhancements of Lemur's certificate 'deployment' capabilities.
+* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability to restrict certificate expiration dates to weekdays.
+
+Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!)
+
+This will most likely be the last release to support python2.7 moving Lemur to target python3 exclusively. Please comment
+on issue #340 if this negatively affects your usage of Lemur.
+
+See the full list of issues closed in `0.4 <https://github.com/Netflix/lemur/milestone/3>`_.
+
+Upgrading
+---------
+
+.. note:: This release will need a slight migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+0.3.0 - `2016-06-06`
+~~~~~~~~~~~~~~~~~~~~
+
+This is quite a large upgrade, it is highly advised you backup your database before attempting to upgrade as this release
+requires the migration of database structure as well as data.
+
+
+Upgrading
+---------
+
+Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+Source Plugin Owners
+--------------------
+
+The dictionary returned from a source plugin has changed keys from `public_certificate` to `body` and `intermediate_certificate` to chain.
+
+
+Issuer Plugin Owners
+--------------------
+
+This release may break your plugins, the keys in `issuer_options` have been changed from `camelCase` to `under_score`.
+This change was made to break an undue reliance on downstream options maintains a more pythonic naming convention. Renaming
+these keys should be fairly trivial, additionally pull requests have been submitted to affected plugins to help ease the transition.
+
+.. note:: This change only affects issuer plugins and does not affect any other types of plugins.
+
+
+* Closed `#63 <https://github.com/Netflix/lemur/issues/63>`_ - Validates all endpoints with Marshmallow schemas, this allows for
+    stricter input validation and better error messages when validation fails.
+* Closed `#146 <https://github.com/Netflix/lemur/issues/146>`_ - Moved authority type to first pane of authority creation wizard.
+* Closed `#147 <https://github.com/Netflix/lemur/issues/147>`_ - Added and refactored the relationship between authorities and their
+    root certificates. Displays the certificates (and chains) next to the authority in question.
+* Closed `#199 <https://github.com/Netflix/lemur/issues/199>`_ - Ensures that the dates submitted to Lemur during authority and
+    certificate creation are actually dates.
+* Closed `#230 <https://github.com/Netflix/lemur/issues/230>`_ - Migrated authority dropdown to an ui-select based dropdown, this
+    should be easier to determine what authorities are available and when an authority has actually been selected.
+* Closed `#254 <https://github.com/Netflix/lemur/issues/254>`_ - Forces certificate names to be generally unique. If a certificate name
+    (generated or otherwise) is found to be a duplicate we increment by appending a counter.
+* Closed `#275 <https://github.com/Netflix/lemur/issues/275>`_ - Switched to using Fernet generated passphrases for exported items.
+    These are more sounds that pseudo random passphrases generated before and have the nice property of being in base64.
+* Closed `#278 <https://github.com/Netflix/lemur/issues/278>`_ - Added ability to specify a custom name to certificate creation, previously
+    this was only available in the certificate import wizard.
+* Closed `#281 <https://github.com/Netflix/lemur/issues/281>`_ - Fixed an issue where notifications could not be removed from a certificate
+    via the UI.
+* Closed `#289 <https://github.com/Netflix/lemur/issues/289>`_ - Fixed and issue where intermediates were not being properly exported.
+* Closed `#315 <https://github.com/Netflix/lemur/issues/315>`_ - Made how roles are associated with certificates and authorities much more
+    explicit, including adding the ability to add roles directly to certificates and authorities on creation.
+
+
+
+0.2.2 - 2016-02-05
+~~~~~~~~~~~~~~~~~~
+
+* Closed `#234 <https://github.com/Netflix/lemur/issues/234>`_ - Allows export plugins to define whether they need
+    private key material (default is True)
+* Closed `#231 <https://github.com/Netflix/lemur/issues/231>`_ - Authorities were not respecting 'owning' roles and their
+    users
+* Closed `#228 <https://github.com/Netflix/lemur/issues/228>`_ - Fixed documentation with correct filter values
+* Closed `#226 <https://github.com/Netflix/lemur/issues/226>`_ - Fixes issue were `import_certificate` was requiring
+    replacement certificates to be specified
+* Closed `#224 <https://github.com/Netflix/lemur/issues/224>`_ - Fixed an issue where NPM might not be globally available (thanks AlexClineBB!)
+* Closed `#221 <https://github.com/Netflix/lemur/issues/234>`_ - Fixes several reported issues where older migration scripts were
+    missing tables, this change removes pre 0.2 migration scripts
+* Closed `#218 <https://github.com/Netflix/lemur/issues/234>`_ - Fixed an issue where export passphrases would not validate
+
+
+0.2.1 - 2015-12-14
+~~~~~~~~~~~~~~~~~~
+
+* Fixed bug with search not refreshing values
+* Cleaned up documentation, including working supervisor example (thanks rpicard!)
+* Closed #165 - Fixed an issue with email templates
+* Closed #188 - Added ability to submit third party CSR
+* Closed #176 - Java-export should allow user to specify truststore/keystore
+* Closed #176 - Extended support for exporting certificate in P12 format
 
 
 0.2.0 - 2015-12-02
-~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 * Closed #120 - Error messages not displaying long enough
 * Closed #121 - Certificate create form should not be valid until a Certificate Authority object is available
@@ -25,8 +341,8 @@ Changelog
 
 
 0.1.5 - 2015-10-26
-~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
-* **SECURITY ISSUE**: Switched from use a AES static key to Fernet encryption.
+* **SECURITY ISSUE**: Switched from use an AES static key to Fernet encryption.
   Affects all versions prior to 0.1.5. If upgrading this will require a data migration.
-  see: `Upgrading Lemur <https://lemur.readthedocs.com/adminstration#UpgradingLemur>`_
+  see: `Upgrading Lemur <https://lemur.readthedocs.io/administration#UpgradingLemur>`_

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.7
+RUN apt-get update
+RUN apt-get install -y make software-properties-common curl
+RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
+RUN apt-get update
+RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN pip install pip==20.0.2
+RUN pip install -U setuptools
+RUN pip install coveralls bandit
+WORKDIR /app
+COPY . /app/
+RUN pip install -e .
+RUN pip install "file://`pwd`#egg=lemur[dev]"
+RUN pip install "file://`pwd`#egg=lemur[tests]"

LICENSE

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -187,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2014 Netflix, Inc.
+   Copyright 2018 Netflix, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

MANIFEST.in

@@ -1,4 +1,4 @@
-include setup.py version.py package.json bower.json gulpfile.js README.rst MANIFEST.in LICENSE AUTHORS
+include setup.py version.py package.json bower.json gulpfile.js README.rst MANIFEST.in LICENSE AUTHORS requirements*.txt
 recursive-include lemur/plugins/lemur_email/templates *
 recursive-include lemur/static *
 global-exclude *~

Makefile

@@ -1,33 +1,59 @@
 NPM_ROOT = ./node_modules
 STATIC_DIR = src/lemur/static/app
+SHELL=/bin/bash
+USER := $(shell whoami)
 
 develop: update-submodules setup-git
 	@echo "--> Installing dependencies"
+ifeq ($(USER), root)
+	@echo "WARNING: It looks like you are installing Lemur as root. This is not generally advised."
+	npm install --unsafe-perm
+else
 	npm install
+endif
 	pip install "setuptools>=0.9.8"
 	# order matters here, base package must install first
 	pip install -e .
 	pip install "file://`pwd`#egg=lemur[dev]"
 	pip install "file://`pwd`#egg=lemur[tests]"
 	node_modules/.bin/gulp build
-	node_modules/.bin/gulp package
+	node_modules/.bin/gulp package --urlContextPath=$(urlContextPath)
+	@echo ""
+
+release:
+	@echo "--> Installing dependencies"
+ifeq ($(USER), root)
+	@echo "WARNING: It looks like you are installing Lemur as root. This is not generally advised."
+	npm install --unsafe-perm
+else
+	npm install
+endif
+	pip install "setuptools>=0.9.8"
+	# order matters here, base package must install first
+	pip install -e .
+	node_modules/.bin/gulp build
+	node_modules/.bin/gulp package --urlContextPath=$(urlContextPath)
 	@echo ""
 
 dev-docs:
-	pip install -r docs/requirements.txt
+	pip install -r requirements-docs.txt
 
 reset-db:
 	@echo "--> Dropping existing 'lemur' database"
 	dropdb lemur || true
 	@echo "--> Creating 'lemur' database"
 	createdb -E utf-8 lemur
+	@echo "--> Enabling pg_trgm extension"
+	psql lemur -c "create extension IF NOT EXISTS pg_trgm;"
 	@echo "--> Applying migrations"
-	lemur db upgrade
+	cd lemur && lemur db upgrade
 
 setup-git:
 	@echo "--> Installing git hooks"
-	git config branch.autosetuprebase always
-	cd .git/hooks && ln -sf ../../hooks/* ./
+	if [ -d .git/hooks ]; then \
+		git config branch.autosetuprebase always; \
+		cd .git/hooks && ln -sf ../../hooks/* ./; \
+	fi
 	@echo ""
 
 clean:
@@ -41,7 +67,7 @@ test: develop lint test-python
 
 testloop: develop
 	pip install pytest-xdist
-	py.test tests -f
+	coverage run --source lemur -m py.test
 
 test-cli:
 	@echo "--> Testing CLI"
@@ -60,7 +86,7 @@ test-js:
 
 test-python:
 	@echo "--> Running Python tests"
-	py.test lemur/tests || exit 1
+	coverage run --source lemur -m py.test
 	@echo ""
 
 lint: lint-python lint-js
@@ -82,4 +108,28 @@ coverage: develop
 publish:
 	python setup.py sdist bdist_wheel upload
 
-.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish
+up-reqs:
+ifndef VIRTUAL_ENV
+    $(error Please activate virtualenv first)
+endif
+	@echo "--> Updating Python requirements"
+	pip install --upgrade pip
+	pip install --upgrade pip-tools
+	pip-compile --output-file requirements.txt requirements.in -U --no-emit-index-url
+	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-emit-index-url
+	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-emit-index-url
+	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-emit-index-url
+	@echo "--> Done updating Python requirements"
+	@echo "--> Removing python-ldap from requirements-docs.txt"
+	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt
+	@echo "--> Installing new dependencies"
+	pip install -e .
+	@echo "--> Done installing new dependencies"
+	@echo ""
+
+# Execute with make checkout-pr pr=<pr number>
+checkout-pr:
+	git fetch upstream pull/$(pr)/head:pr-$(pr)
+
+
+.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish release

OSSMETADATA

@@ -0,0 +1 @@
+osslifecycle=active

README.rst

@@ -5,36 +5,31 @@ Lemur
    :alt: Join the chat at https://gitter.im/Netflix/lemur
    :target: https://gitter.im/Netflix/lemur?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: https://img.shields.io/pypi/v/lemur.svg
-    :target: https://pypi.python.org/pypi/lemur/
-    :alt: Latest Version
-
 .. image:: https://readthedocs.org/projects/lemur/badge/?version=latest
-    :target: https://lemur.readthedocs.org
+    :target: https://lemur.readthedocs.io
     :alt: Latest Docs
 
+.. image:: https://img.shields.io/badge/NetflixOSS-active-brightgreen.svg
+
 .. image:: https://travis-ci.org/Netflix/lemur.svg
     :target: https://travis-ci.org/Netflix/lemur
 
-.. image:: https://requires.io/github/Netflix/lemur/requirements.svg?branch=master
-    :target: https://requires.io/github/Netflix/lemur/requirements/?branch=master
-    :alt: Requirements Status
+.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=master
+    :target: https://coveralls.io/github/Netflix/lemur?branch=master
+
 
-.. image:: https://badge.waffle.io/Netflix/lemur.png?label=ready&title=Ready
-    :target: https://waffle.io/Netflix/lemur
-    :alt: 'Stories in Ready'
 
 Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs
 and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults.
 
-It works on CPython 2.7, 3.3, 3.4. We deploy on Ubuntu and develop on OS X.
+It works on Python 3.7. We deploy on Ubuntu and develop on OS X.
 
 
 Project resources
 =================
 
 - `Lemur Blog Post <http://techblog.netflix.com/2015/09/introducing-lemur.html>`_
-- `Documentation <http://lemur.readthedocs.org/>`_
+- `Documentation <http://lemur.readthedocs.io/>`_
 - `Source code <https://github.com/netflix/lemur>`_
 - `Issue tracker <https://github.com/netflix/lemur/issues>`_
 - `Docker <https://github.com/Netflix/lemur-docker>`_

bower.json

@@ -6,43 +6,45 @@
   },
   "private": true,
   "dependencies": {
-    "angular": "1.3",
-    "json3": "~3.3",
-    "es5-shim": "~4.0",
-    "jquery": "~2.1",
-    "angular-resource": "1.2.15",
-    "angular-cookies": "1.2.15",
-    "angular-sanitize": "1.2.15",
-    "angular-route": "1.2.15",
-    "angular-strap": "~2.0.2",
-    "restangular": "~1.4.0",
-    "ng-table": "~0.5.4",
-    "ngAnimate": "*",
-    "moment": "~2.6.0",
-    "angular-animate": "~1.4.0",
-    "angular-loading-bar": "~0.6.0",
-    "fontawesome": "~4.2.0",
+    "jquery": "~2.2.0",
     "angular-wizard": "~0.4.0",
-    "bootswatch": "3.3.1+2",
-    "angular-spinkit": "~0.3.3",
-    "angular-bootstrap": "~0.12.0",
-    "angular-ui-switch": "~0.1.0",
-    "angular-chart.js": "~0.7.1",
-    "satellizer": "~0.9.4",
-    "angularjs-toaster": "~0.4.14",
-    "ngletteravatar": "~3.0.1",
+    "angular": "1.4.9",
+    "json3": "~3.3",
+    "es5-shim": "~4.5.0",
+    "angular-bootstrap": "~1.1.1",
+    "angular-animate": "~1.4.9",
+    "restangular": "~1.5.1",
+    "ng-table": "~0.8.3",
+    "moment": "~2.11.1",
+    "bootstrap": "~3.4.1",
+    "angular-loading-bar": "~0.8.0",
+    "angular-moment": "~0.10.3",
+    "moment-range": "~2.1.0",
+    "angular-clipboard": "~1.3.0",
+    "angularjs-toaster": "~1.0.0",
+    "angular-chart.js": "~0.8.8",
+    "ngletteravatar": "~4.0.0",
+    "bootswatch": "3.4.1+1",
+    "fontawesome": "~4.5.0",
+    "satellizer": "~0.13.4",
     "angular-ui-router": "~0.2.15",
-    "angular-clipboard": "~1.1.1",
-    "angular-file-saver": "~1.0.1"
-  },
-  "devDependencies": {
-    "angular-mocks": "~1.3",
-    "angular-scenario": "~1.3",
-    "ngletteravatar": "~3.0.1"
+    "font-awesome": "~4.5.0",
+    "lodash": "~4.17.20",
+    "underscore": "~1.8.3",
+    "angular-smart-table": "2.1.8",
+    "angular-strap": ">= 2.2.2",
+    "angular-underscore": "^0.5.0",
+    "angular-translate": "^2.9.0",
+    "angular-ui-switch": "~0.1.0",
+    "angular-sanitize": "~1.5.0",
+    "angular-file-saver": "~1.0.1",
+    "angular-ui-select": "~0.17.1",
+    "d3": "^3.5.17"
   },
   "resolutions": {
-    "bootstrap": "~3.3.1",
-    "angular": "1.3"
+    "moment": ">=2.8.0 <2.11.0",
+    "lodash": ">=1.3.0 <2.5.0",
+    "angular": "1.4.9"
   },
   "ignore": [
     "**/.*",

docker-compose.yml

@@ -0,0 +1,27 @@
+---
+version: '2.0'
+services:
+  test:
+    build: .
+    volumes:
+      - ".:/app"
+    links:
+      - postgres
+    command: make test
+    environment:
+      SQLALCHEMY_DATABASE_URI: postgresql://lemur:lemur@postgres:5432/lemur
+      VIRTUAL_ENV: 'true'
+
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: lemur
+      POSTGRES_PASSWORD: lemur
+    ports:
+      - "5432:5432"
+
+  redis:
+    image: "redis:alpine"
+    ports:
+      - "6379:6379"

docker/.dockerignore

@@ -0,0 +1,3 @@
+*-env
+docker-compose.yml
+Dockerfile

docker/Dockerfile

@@ -0,0 +1,69 @@
+FROM python:3.7.9-alpine3.12
+
+ARG VERSION
+ENV VERSION master
+
+ARG URLCONTEXT
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk add --no-cache --update python3 py-pip libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                libtool \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    mkdir -p /opt/lemur /home/lemur/.lemur/ && \
+    curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
+    chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
+
+WORKDIR /opt/lemur
+
+RUN echo "Running with python:" && python -c 'import platform; print(platform.python_version())' && \
+    echo "Running with nodejs:" && node -v && \
+    npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
+    apk del build-dependencies
+
+COPY entrypoint /
+COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY supervisor.conf /
+COPY nginx/default.conf /etc/nginx/conf.d/
+COPY nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/Dockerfile-src

@@ -0,0 +1,67 @@
+FROM alpine:3.8
+
+ARG VERSION
+ENV VERSION master
+
+ARG URLCONTEXT
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                libtool \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    mkdir -p /home/lemur/.lemur/ && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/
+
+COPY ./ /opt/lemur    
+WORKDIR /opt/lemur
+
+RUN chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \
+    npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
+    apk del build-dependencies
+
+COPY docker/entrypoint /
+COPY docker/src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY docker/supervisor.conf /
+COPY docker/nginx/default.conf /etc/nginx/conf.d/
+COPY docker/nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '3'
+
+volumes:
+  pg_data: { }
+
+services:
+  postgres:
+    image: "postgres:13.1-alpine"
+    restart: on-failure
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    env_file:
+      - pgsql-env
+
+  lemur:
+    # image: "netlix-lemur:latest"
+    restart: on-failure
+    build:
+      context: .
+    depends_on:
+      - postgres
+      - redis
+    env_file:
+      - lemur-env
+      - pgsql-env
+    ports:
+      - 87:80
+      - 447:443
+
+  redis:
+    image: "redis:alpine3.12"
+    restart: on-failure

docker/entrypoint

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then
+  echo "Database vars not set"
+  exit 1
+fi
+
+export POSTGRES_PORT="${POSTGRES_PORT:-5432}"
+
+export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}"
+
+export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
+
+
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'select 1;'
+
+echo " # Create Postgres trgm extension"
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
+echo " # Done"
+
+if [ -z "${SKIP_SSL}" ]; then
+  if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then
+    openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE"
+  fi
+  [ -f "/etc/nginx/conf.d/default-ssl.conf.a" ] && mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf
+  [ -f "/etc/nginx/conf.d/default.conf" ] && mv -f /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a
+fi
+
+# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then
+# echo "Creating config"
+# https://github.com/Netflix/lemur/issues/2257
+# python3 /opt/lemur/lemur/manage.py create_config
+# echo "Done"
+# fi
+
+echo " # Running init"
+su lemur -s /bin/bash -c "cd /opt/lemur/lemur; lemur init -p ${LEMUR_ADMIN_PASSWORD}"
+echo " # Done"
+
+# echo "Creating user"
+# https://github.com/Netflix/lemur/issues/
+# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell
+# echo "Done"
+
+cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
+cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
+cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
+cron_reissue="${CRON_REISSUE:-"0 23 * * *"}"
+
+echo " # Populating crontab"
+echo "${cron_notify} lemur notify expirations" > /etc/crontabs/lemur
+echo "${cron_sync} lemur source sync -s all" >> /etc/crontabs/lemur
+echo "${cron_revoked} lemur certificate check_revoked" >> /etc/crontabs/lemur
+echo "${cron_reissue} lemur certificate reissue -c" >> /etc/crontabs/lemur
+echo " # Done"
+
+exec "$@"

docker/lemur-env

@@ -0,0 +1,25 @@
+# SKIP_SSL=1
+# LEMUR_TOKEN_SECRET=
+# LEMUR_DEFAULT_COUNTRY=
+# LEMUR_DEFAULT_STATE=
+# LEMUR_DEFAULT_LOCATION=
+# LEMUR_DEFAULT_ORGANIZATION=
+# LEMUR_DEFAULT_ORGANIZATIONAL_UNIT=
+# LEMUR_DEFAULT_ISSUER_PLUGIN=cryptography-issuer
+# LEMUR_DEFAULT_AUTHORITY=cryptography
+# MAIL_SERVER=mail.example.com
+# MAIL_PORT=25
+# LEMUR_EMAIL=lemur@example.com
+# LEMUR_SECURITY_TEAM_EMAIL=['team@example.com']
+# LEMUR_TOKEN_SECRET=
+# LEMUR_ENCRYPTION_KEYS=['']
+# DEBUG=True
+# LDAP_DEBUG=True
+# LDAP_AUTH=True
+# LDAP_BIND_URI=ldap://example.com
+# LDAP_BASE_DN=DC=example,DC=com
+# LDAP_EMAIL_DOMAIN=example.com
+# LDAP_USE_TLS=False
+# LDAP_REQUIRED_GROUP=certificate-management-admins
+# LDAP_GROUPS_TO_ROLES={'certificate-management-admins': 'admin', 'Team': 'team@example.com'}
+# LDAP_IS_ACTIVE_DIRECTORY=False

docker/nginx/default-ssl.conf

@@ -0,0 +1,37 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+    listen 80;
+    server_name  _;
+    return 301 https://$host$request_uri;
+}
+
+server {
+   listen 443 ssl;
+   server_name  _;
+   access_log /dev/stdout;
+   error_log /dev/stderr;
+   ssl_certificate /etc/nginx/ssl/server.crt;
+   ssl_certificate_key /etc/nginx/ssl/server.key;
+   ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+   ssl_ciphers HIGH:!aNULL:!MD5;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/nginx/default.conf

@@ -0,0 +1,26 @@
+add_header X-Frame-Options DENY;
+add_header X-Content-Type-Options nosniff;
+add_header X-XSS-Protection "1; mode=block";
+
+server {
+   listen      80;
+   access_log  /dev/stdout;
+   error_log   /dev/stderr;
+
+   location /api {
+        proxy_pass  http://127.0.0.1:8000;
+        proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_set_header        Host            $host;
+        proxy_set_header        X-Real-IP       $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location / {
+        root /opt/lemur/lemur/static/dist;
+        include mime.types;
+        index index.html;
+    }
+
+}

docker/pgsql-env

@@ -0,0 +1,4 @@
+POSTGRES_USER=lemur
+POSTGRES_PASSWORD=12345
+POSTGRES_DB=lemur
+POSTGRES_HOST=postgres

docker/src/lemur.conf.py

@@ -0,0 +1,237 @@
+import os.path
+import random
+import string
+from celery.schedules import crontab
+
+import base64
+
+_basedir = os.path.abspath(os.path.dirname(__file__))
+
+# See the Lemur docs (https://lemur.readthedocs.org) for more information on configuration
+
+LOG_LEVEL = str(os.environ.get('LOG_LEVEL', 'DEBUG'))
+LOG_FILE = str(os.environ.get('LOG_FILE', '/home/lemur/.lemur/lemur.log'))
+LOG_JSON = True
+
+CORS = os.environ.get("CORS") == "True"
+debug = os.environ.get("DEBUG") == "True"
+
+
+def get_random_secret(length):
+    secret_key = ''.join(random.choice(string.ascii_uppercase) for x in range(round(length / 4)))
+    secret_key = secret_key + ''.join(random.choice("~!@#$%^&*()_+") for x in range(round(length / 4)))
+    secret_key = secret_key + ''.join(random.choice(string.ascii_lowercase) for x in range(round(length / 4)))
+    return secret_key + ''.join(random.choice(string.digits) for x in range(round(length / 4)))
+
+
+# This is the secret key used by Flask session management
+SECRET_KEY = repr(os.environ.get('SECRET_KEY', get_random_secret(32).encode('utf8')))
+
+# You should consider storing these separately from your config
+LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET',
+                                         base64.b64encode(get_random_secret(32).encode('utf8'))))
+# This must match the key for whichever DB the container is using - this could be a dump of dev or test, or a unique key
+LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS',
+                                            base64.b64encode(get_random_secret(32).encode('utf8')).decode('utf8')))
+
+REDIS_HOST = 'redis'
+REDIS_PORT = 6379
+REDIS_DB = 0
+CELERY_RESULT_BACKEND = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_BROKER_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_IMPORTS = ('lemur.common.celery')
+CELERYBEAT_SCHEDULE = {
+    # All tasks are disabled by default. Enable any tasks you wish to run.
+    # 'fetch_all_pending_acme_certs': {
+    #     'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'remove_old_acme_certs': {
+    #     'task': 'lemur.common.celery.remove_old_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=8, minute=0, day_of_week=5),
+    # },
+    # 'clean_all_sources': {
+    #     'task': 'lemur.common.celery.clean_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=5, minute=0, day_of_week=5),
+    # },
+    # 'sync_all_sources': {
+    #     'task': 'lemur.common.celery.sync_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=0),
+    #     # this job is running 30min before endpoints_expire which deletes endpoints which were not updated
+    # },
+    # 'sync_source_destination': {
+    #     'task': 'lemur.common.celery.sync_source_destination',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=15),
+    # },
+    # 'report_celery_last_success_metrics': {
+    #     'task': 'lemur.common.celery.report_celery_last_success_metrics',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'certificate_reissue': {
+    #     'task': 'lemur.common.celery.certificate_reissue',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=9, minute=0),
+    # },
+    # 'certificate_rotate': {
+    #     'task': 'lemur.common.celery.certificate_rotate',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'endpoints_expire': {
+    #     'task': 'lemur.common.celery.endpoints_expire',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=30),
+    #     # this job is running 30min after sync_all_sources which updates endpoints
+    # },
+    # 'get_all_zones': {
+    #     'task': 'lemur.common.celery.get_all_zones',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*/30"),
+    # },
+    # 'check_revoked': {
+    #     'task': 'lemur.common.celery.check_revoked',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'enable_autorotate_for_certs_attached_to_endpoint': {
+    #     'task': 'lemur.common.celery.enable_autorotate_for_certs_attached_to_endpoint',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'notify_expirations': {
+    #     'task': 'lemur.common.celery.notify_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    #  },
+    # 'notify_authority_expirations': {
+    #     'task': 'lemur.common.celery.notify_authority_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'send_security_expiration_summary': {
+    #     'task': 'lemur.common.celery.send_security_expiration_summary',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0, day_of_week='mon-fri'),
+    # }
+}
+CELERY_TIMEZONE = 'UTC'
+
+SQLALCHEMY_ENABLE_FLASK_REPLICATED = False
+SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://lemur:lemur@localhost:5432/lemur')
+
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+SQLALCHEMY_ECHO = True
+SQLALCHEMY_POOL_RECYCLE = 499
+SQLALCHEMY_POOL_TIMEOUT = 20
+
+LEMUR_EMAIL = 'lemur@example.com'
+LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
+LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
+LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
+LEMUR_EMAIL_SENDER = 'smtp'
+
+# mail configuration
+# MAIL_SERVER = 'mail.example.com'
+
+PUBLIC_CA_MAX_VALIDITY_DAYS = 397
+DEFAULT_VALIDITY_DAYS = 365
+
+LEMUR_OWNER_EMAIL_IN_SUBJECT = False
+
+LEMUR_DEFAULT_COUNTRY = str(os.environ.get('LEMUR_DEFAULT_COUNTRY', 'US'))
+LEMUR_DEFAULT_STATE = str(os.environ.get('LEMUR_DEFAULT_STATE', 'California'))
+LEMUR_DEFAULT_LOCATION = str(os.environ.get('LEMUR_DEFAULT_LOCATION', 'Los Gatos'))
+LEMUR_DEFAULT_ORGANIZATION = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATION', 'Example, Inc.'))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT', ''))
+
+LEMUR_DEFAULT_AUTHORITY = str(os.environ.get('LEMUR_DEFAULT_AUTHORITY', 'ExampleCa'))
+
+LEMUR_DEFAULT_ROLE = 'operator'
+
+ACTIVE_PROVIDERS = []
+METRIC_PROVIDERS = []
+
+# Authority Settings - These will change depending on which authorities you are
+# using
+current_path = os.path.dirname(os.path.realpath(__file__))
+
+# DNS Settings
+
+# exclude logging missing SAN, since we can have certs from private CAs with only cn, prod parity
+LOG_SSL_SUBJ_ALT_NAME_ERRORS = False
+
+ACME_DNS_PROVIDER_TYPES = {"items": [
+    {
+        'name': 'route53',
+        'requirements': [
+            {
+                'name': 'account_id',
+                'type': 'int',
+                'required': True,
+                'helpMessage': 'AWS Account number'
+            },
+        ]
+    },
+    {
+        'name': 'cloudflare',
+        'requirements': [
+            {
+                'name': 'email',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Email'
+            },
+            {
+                'name': 'key',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Key'
+            },
+        ]
+    },
+    {
+        'name': 'dyn',
+    },
+    {
+        'name': 'ultradns',
+    },
+]}
+
+# Authority plugins which support revocation
+SUPPORTED_REVOCATION_AUTHORITY_PLUGINS = ['acme-issuer']

docker/supervisor.conf

@@ -0,0 +1,33 @@
+[supervisord]
+nodaemon=true
+user=root
+logfile=/dev/stdout
+logfile_maxbytes=0
+pidfile = /tmp/supervisord.pid
+
+[program:lemur]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
+command=lemur start -b 0.0.0.0:8000
+user=lemur
+directory=/opt/lemur/lemur
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:cron]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
+command=/usr/sbin/crond -f
+user=root
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docs/administration.rst

@@ -0,0 +1,1839 @@
+Configuration
+=============
+
+.. warning::
+    There are many secrets that Lemur uses that must be protected. All of these options are set via the Lemur configuration
+    file. It is highly advised that you do not store your secrets in this file! Lemur provides functions
+    that allow you to encrypt files at rest and decrypt them when it's time for deployment. See :ref:`Credential Management <CredentialManagement>`
+    for more information.
+
+.. note::
+    All configuration values are python strings unless otherwise noted.
+
+
+Basic Configuration
+-------------------
+
+.. data:: LOG_LEVEL
+    :noindex:
+
+    ::
+
+        LOG_LEVEL = "DEBUG"
+
+.. data:: LOG_FILE
+    :noindex:
+
+    ::
+
+        LOG_FILE = "/logs/lemur/lemur-test.log"
+
+.. data:: LOG_UPGRADE_FILE
+    :noindex:
+
+    ::
+
+        LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log"
+
+.. data:: DEBUG
+    :noindex:
+
+    Sets the flask debug flag to true (if supported by the webserver)
+
+    ::
+
+        DEBUG = False
+
+    .. warning::
+        This should never be used in a production environment as it exposes Lemur to
+        remote code execution through the debug console.
+
+
+.. data:: CORS
+    :noindex:
+
+    Allows for cross domain requests, this is most commonly used for development but could
+    be use in production if you decided to host the webUI on a different domain than the server.
+
+    Use this cautiously, if you're not sure. Set it to `False`
+
+    ::
+
+        CORS = False
+
+
+.. data:: SQLALCHEMY_DATABASE_URI
+    :noindex:
+
+        If you have ever used sqlalchemy before this is the standard connection string used. Lemur uses a postgres database and the connection string would look something like:
+
+    ::
+
+        SQLALCHEMY_DATABASE_URI = 'postgresql://<user>:<password>@<hostname>:5432/lemur'
+
+
+.. data:: SQLALCHEMY_POOL_SIZE
+    :noindex:
+
+            The default connection pool size is 5 for sqlalchemy managed connections.   Depending on the number of Lemur instances,
+            please specify per instance connection pool size.  Below is an example to set connection pool size to 10.
+
+    ::
+
+        SQLALCHEMY_POOL_SIZE = 10
+
+
+    .. warning::
+        This is an optional setting but important to review and set for optimal database connection usage and for overall database performance.
+
+.. data:: SQLALCHEMY_MAX_OVERFLOW
+    :noindex:
+
+        This setting allows to create connections in addition to specified number of connections in pool size.   By default, sqlalchemy
+        allows 10 connections to create in addition to the pool size.  This is also an optional setting.  If `SQLALCHEMY_POOL_SIZE` and
+        `SQLALCHEMY_MAX_OVERFLOW` are not speficied then each Lemur instance may create maximum of 15 connections.
+
+    ::
+
+        SQLALCHECK_MAX_OVERFLOW = 0
+
+
+    .. note::
+        Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size.
+
+
+.. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION
+    :noindex:
+
+        Specifies whether to allow certificates created by Lemur to expire on weekends. Default is True.
+
+.. data:: LEMUR_ALLOWED_DOMAINS
+    :noindex:
+
+        List of regular expressions for domain restrictions; if the list is not empty, normal users can only issue
+        certificates for domain names matching at least one pattern on this list. Administrators are exempt from this
+        restriction.
+
+        Cerificate common name is matched against these rules *if* it does not contain a space. SubjectAltName DNS names
+        are always matched against these rules.
+
+        Take care to write patterns in such way to not allow the `*` wildcard character inadvertently. To match a `.`
+        character, it must be escaped (as `\.`).
+
+.. data:: LEMUR_OWNER_EMAIL_IN_SUBJECT
+    :noindex:
+
+        By default, Lemur will add the certificate owner's email address to certificate subject (for CAs that allow it).
+        Set this to `False` to disable this.
+
+.. data:: LEMUR_TOKEN_SECRET
+    :noindex:
+
+        The TOKEN_SECRET is the secret used to create JWT tokens that are given out to users. This should be securely generated and kept private.
+
+    ::
+
+        LEMUR_TOKEN_SECRET = 'supersecret'
+
+    An example of how you might generate a random string:
+
+        >>> import random
+        >>> secret_key = ''.join(random.choice(string.ascii_uppercase) for x in range(6))
+        >>> secret_key = secret_key + ''.join(random.choice("~!@#$%^&*()_+") for x in range(6))
+        >>> secret_key = secret_key + ''.join(random.choice(string.ascii_lowercase) for x in range(6))
+        >>> secret_key = secret_key + ''.join(random.choice(string.digits) for x in range(6))
+
+
+.. data:: LEMUR_ENCRYPTION_KEYS
+    :noindex:
+
+        The LEMUR_ENCRYPTION_KEYS is used to encrypt data at rest within Lemur's database. Without a key Lemur will refuse
+        to start. Multiple keys can be provided to facilitate key rotation. The first key in the list is used for
+        encryption and all keys are tried for decryption until one works. Each key must be 32 URL safe base-64 encoded bytes.
+
+        Only fields of type ``Vault`` will be encrypted. At present, only the following fields are encrypted:
+
+        * ``certificates.private_key``
+        * ``pending_certificates.private_key``
+        * ``dns_providers.credentials``
+        * ``roles.password``
+
+        For implementation details, see ``Vault`` in ``utils.py``.
+
+        Running lemur create_config will securely generate a key for your configuration file.
+        If you would like to generate your own, we recommend the following method:
+
+            >>> import os
+            >>> import base64
+            >>> base64.urlsafe_b64encode(os.urandom(32))
+
+    ::
+
+        LEMUR_ENCRYPTION_KEYS = ['1YeftooSbxCiX2zo8m1lXtpvQjy27smZcUUaGmffhMY=', 'LAfQt6yrkLqOK5lwpvQcT4jf2zdeTQJV1uYeh9coT5s=']
+
+
+.. data:: PUBLIC_CA_MAX_VALIDITY_DAYS
+    :noindex:
+
+        Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities.
+        The authorities with cab_compliant option set to true will use this config. The example below overrides the default validity
+        of 397 days and sets it to 365 days.
+
+    ::
+
+        PUBLIC_CA_MAX_VALIDITY_DAYS = 365
+
+
+.. data:: DEFAULT_VALIDITY_DAYS
+    :noindex:
+
+        Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which
+        is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please
+        note that this config is used for cert issuance only through Lemur UI. The example below overrides the default validity
+        of 365 days and sets it to 1095 days (3 years).
+
+    ::
+
+        DEFAULT_VALIDITY_DAYS = 1095
+
+
+.. data:: DEBUG_DUMP
+    :noindex:
+
+        Dump all imported or generated CSR and certificate details to stdout using OpenSSL. (default: `False`)
+
+.. data:: ALLOW_CERT_DELETION
+    :noindex:
+
+        When set to True, certificates can be marked as deleted via the API and deleted certificates will not be displayed
+        in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed"
+        and deleted certificates will always be visible in the UI. (default: `False`)
+
+.. data:: LEMUR_AWS_REGION
+    :noindex:
+
+        This is an optional config applicable for settings where Lemur is deployed in AWS. For accessing regionalized
+        STS endpoints, LEMUR_AWS_REGION defines the region where Lemur is deployed.
+
+Certificate Default Options
+---------------------------
+
+Lemur allows you to fine tune your certificates to your organization. The following defaults are presented in the UI
+and are used when Lemur creates the CSR for your certificates.
+
+
+.. data:: LEMUR_DEFAULT_COUNTRY
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_COUNTRY = "US"
+
+
+.. data:: LEMUR_DEFAULT_STATE
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_STATE = "California"
+
+
+.. data:: LEMUR_DEFAULT_LOCATION
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_LOCATION = "Los Gatos"
+
+
+.. data:: LEMUR_DEFAULT_ORGANIZATION
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_ORGANIZATION = "Netflix"
+
+
+.. data:: LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = ""
+
+
+.. data:: LEMUR_DEFAULT_ISSUER_PLUGIN
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_ISSUER_PLUGIN = "verisign-issuer"
+
+
+.. data:: LEMUR_DEFAULT_AUTHORITY
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_AUTHORITY = "verisign"
+
+
+.. _NotificationOptions:
+
+Notification Options
+--------------------
+
+Lemur supports a small variety of notification types through a set of notification plugins.
+By default, Lemur configures a standard set of email notifications for all certificates.
+
+**Plugin-capable notifications**
+
+These notifications can be configured to use all available notification plugins.
+
+Supported types:
+
+* Certificate expiration
+
+**Email-only notifications**
+
+These notifications can only be sent via email and cannot use other notification plugins.
+
+Supported types:
+
+* CA certificate expiration
+* Pending ACME certificate failure
+* Certificate rotation
+* Security certificate expiration summary
+
+**Default notifications**
+
+When a certificate is created, the following email notifications are created for it if they do not exist.
+If these notifications already exist, they will be associated with the new certificate.
+
+* ``DEFAULT_<OWNER>_X_DAY``, where X is the set of values specified in ``LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS`` and defaults to 30, 15, and 2 if not specified. The owner's username will replace ``<OWNER>``.
+* ``DEFAULT_SECURITY_X_DAY``, where X is the set of values specified in ``LEMUR_SECURITY_TEAM_EMAIL_INTERVALS`` and defaults to ``LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS`` if not specified (which also defaults to 30, 15, and 2 if not specified).
+
+These notifications can be disabled if desired. They can also be unassociated with a specific certificate.
+
+**Disabling notifications**
+
+Notifications can be disabled either for an individual certificate (which disables all notifications for that certificate)
+or for an individual notification object (which disables that notification for all associated certificates).
+At present, disabling a notification object will only disable certificate expiration notifications, and not other types,
+since other notification types don't use notification objects.
+
+**Certificate expiration**
+
+Certificate expiration notifications are sent when the scheduled task to send certificate expiration notifications runs
+(see :ref:`PeriodicTasks`). Specific patterns of certificate names may be excluded using ``--exclude`` (when using
+cron; you may specify this multiple times for multiple patterns) or via the config option ``EXCLUDE_CN_FROM_NOTIFICATION``
+(when using celery; this is a list configuration option, meaning you specify multiple values, such as
+``['exclude', 'also exclude']``). The specified exclude pattern will match if found anywhere in the certificate name.
+
+When the periodic task runs, Lemur checks for certificates meeting the following conditions:
+
+* Certificate has notifications enabled
+* Certificate is not expired
+* Certificate is not revoked
+* Certificate name does not match the `exclude` parameter
+* Certificate has at least one associated notification object
+* That notification is active
+* That notification's configured interval and unit match the certificate's remaining lifespan
+
+All eligible certificates are then grouped by owner and applicable notification. For each notification and certificate group,
+Lemur will send the expiration notification using whichever plugin was configured for that notification object.
+In addition, Lemur will send an email to the certificate owner and security team (as specified by the
+``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter).
+
+**CA certificate expiration**
+
+Certificate authority certificate expiration notifications are sent when the scheduled task to send authority certificate
+expiration notifications runs (see :ref:`PeriodicTasks`). Notifications are sent via the intervals configured in the
+configuration parameter ``LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS``, with a default of 365 and 180 days.
+
+When the periodic task runs, Lemur checks for certificates meeting the following conditions:
+
+* Certificate has notifications enabled
+* Certificate is not expired
+* Certificate is not revoked
+* Certificate is associated with a CA
+* Certificate's remaining lifespan matches one of the configured intervals
+
+All eligible certificates are then grouped by owner and expiration interval. For each interval and certificate group,
+Lemur will send the CA certificate expiration notification via email to the certificate owner and security team
+(as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter).
+
+**Pending ACME certificate failure**
+
+Whenever a pending ACME certificate fails to be issued, Lemur will send a notification via email to the certificate owner
+and security team (as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter). This email is not sent if
+the pending certificate had notifications disabled.
+
+Lemur will attempt 3x times to resolve a pending certificate.
+This can at times result into 3 duplicate certificates, if all certificate attempts get resolved.
+
+**Certificate rotation**
+
+Whenever a cert is rotated, Lemur will send a notification via email to the certificate owner. This notification is
+disabled by default; to enable it, you must set the option ``--notify`` (when using cron) or the configuration parameter
+``ENABLE_ROTATION_NOTIFICATION`` (when using celery).
+
+**Security certificate expiration summary**
+
+If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all
+certificates with upcoming expiration date that occurs within the number of days specified by the
+``LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS`` configuration parameter (with a fallback of 14 days).
+Note that certificates will be included in this summary even if they do not have any associated notifications.
+
+This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails.
+
+NOTE: At present, this summary email essentially duplicates the certificate expiration notifications, since all
+certificate expiration notifications are also sent to the security team. This issue will be fixed in the future.
+
+**Email notifications**
+
+Templates for emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
+
+The following configuration options are supported:
+
+.. data:: LEMUR_EMAIL_SENDER
+    :noindex:
+
+    Specifies which service will be delivering notification emails. Valid values are `SMTP` or `SES`
+
+    .. note::
+        If using SMTP as your provider you will need to define additional configuration options as specified by Flask-Mail.
+        See: `Flask-Mail <https://pythonhosted.org/Flask-Mail>`_
+
+        If you are using SES the email specified by the `LEMUR_MAIL` configuration will need to be verified by AWS before
+        you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
+
+
+.. data:: LEMUR_SES_SOURCE_ARN
+    :noindex:
+
+    Specifies an ARN to use as the SourceArn when sending emails via SES.
+
+    .. note::
+        This parameter is only required if you're using a sending authorization with SES.
+        See: `Using sending authorization with Amazon SES <https://docs.aws.amazon.com/ses/latest/DeveloperGuide/sending-authorization.html>`_
+
+
+.. data:: LEMUR_SES_REGION
+    :noindex:
+
+    Specifies a region for sending emails via SES.
+
+    .. note::
+        This parameter defaults to us-east-1 and is only required if you wish to use a different region.
+
+
+.. data:: LEMUR_EMAIL
+    :noindex:
+
+        Lemur sender's email
+
+        ::
+
+            LEMUR_EMAIL = 'lemur@example.com'
+
+
+.. data:: LEMUR_SECURITY_TEAM_EMAIL
+    :noindex:
+
+        This is an email or list of emails that should be notified when a certificate is expiring. It is also the contact email address for any discovered certificate.
+
+        ::
+
+            LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
+
+.. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS
+    :noindex:
+
+        Lemur notification intervals. If unspecified, the value [30, 15, 2] is used.
+
+        ::
+
+            LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
+
+.. data:: LEMUR_SECURITY_TEAM_EMAIL_INTERVALS
+    :noindex:
+
+       Alternate notification interval set for security team notifications. Use this if you would like the default security team notification interval for new certificates to differ from the global default as specified in LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS. If unspecified, the value of LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS is used. Security team default notifications for new certificates can effectively be disabled by setting this value to an empty array.
+
+       ::
+
+          LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
+
+.. data:: LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS
+    :noindex:
+
+       Notification interval set for CA certificate expiration notifications. If unspecified, the value [365, 180] is used (roughly one year and 6 months).
+
+       ::
+
+          LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS = [365, 180]
+
+
+Celery Options
+---------------
+To make use of automated tasks within lemur (e.g. syncing source/destinations, or reissuing ACME certificates), you
+need to configure celery. See :ref:`Periodic Tasks <PeriodicTasks>` for more in depth documentation.
+
+.. data:: CELERY_RESULT_BACKEND
+    :noindex:
+
+        The url to your redis backend (needs to be in the format `redis://<host>:<port>/<database>`)
+
+.. data:: CELERY_BROKER_URL
+    :noindex:
+
+        The url to your redis broker (needs to be in the format `redis://<host>:<port>/<database>`)
+
+.. data:: CELERY_IMPORTS
+    :noindex:
+
+        The module that celery needs to import, in our case thats `lemur.common.celery`
+
+.. data:: CELERY_TIMEZONE
+    :noindex:
+
+        The timezone for celery to work with
+
+
+.. data:: CELERYBEAT_SCHEDULE
+    :noindex:
+
+        This defines the schedule, with which the celery beat makes the worker run the specified tasks.
+
+Since the celery module, relies on the RedisHandler, the following options also need to be set.
+
+.. data:: REDIS_HOST
+    :noindex:
+
+        Hostname of your redis instance
+
+.. data:: REDIS_PORT
+    :noindex:
+
+        Port on which redis is running (default: 6379)
+
+.. data:: REDIS_DB
+    :noindex:
+
+        Which redis database to be used, by default redis offers databases 0-15 (default: 0)
+
+Authentication Options
+----------------------
+Lemur currently supports Basic Authentication, LDAP Authentication, Ping OAuth2, and Google out of the box. Additional flows can be added relatively easily.
+
+LDAP Options
+~~~~~~~~~~~~
+
+Lemur supports the use of an LDAP server in conjunction with Basic Authentication. Lemur local users can still be defined and take precedence over LDAP users. If a local user does not exist, LDAP will be queried for authentication. Only simple ldap binding with or without TLS is supported.
+
+LDAP support requires the pyldap python library, which also depends on the following openldap packages.
+
+.. code-block:: bash
+
+      $ sudo apt-get update
+      $ sudo apt-get install libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+
+
+To configure the use of an LDAP server, a number of settings need to be configured in `lemur.conf.py`.
+
+Here is an example LDAP configuration stanza you can add to your config. Adjust to suit your environment of course.
+
+.. code-block:: python
+
+        LDAP_AUTH = True
+        LDAP_BIND_URI='ldaps://secure.evilcorp.net'
+        LDAP_BASE_DN='DC=users,DC=evilcorp,DC=net'
+        LDAP_EMAIL_DOMAIN='evilcorp.net'
+        LDAP_USE_TLS = True
+        LDAP_CACERT_FILE = '/opt/lemur/trusted.pem'
+        LDAP_REQUIRED_GROUP = 'certificate-management-access'
+        LDAP_GROUPS_TO_ROLES = {'certificate-management-admin': 'admin', 'certificate-management-read-only': 'read-only'}
+        LDAP_IS_ACTIVE_DIRECTORY = True
+
+
+The lemur ldap module uses the `user principal name` (upn) of the authenticating user to bind. This is done once for each user at login time. The UPN is effectively the email address in AD/LDAP of the user. If the user doesn't provide the email address, it constructs one based on the username supplied (which should normally match the samAccountName) and the value provided by the config LDAP_EMAIL_DOMAIN.
+The config LDAP_BASE_DN tells lemur where to search within the AD/LDAP tree for the given UPN (user). If the bind with those credentials is successful - there is a valid user in AD with correct password.
+
+Each of the LDAP options are described below.
+
+.. data:: LDAP_AUTH
+    :noindex:
+
+        This enables the use of LDAP
+
+        ::
+
+            LDAP_AUTH = True
+
+.. data:: LDAP_BIND_URI
+    :noindex:
+
+        Specifies the LDAP server connection string
+
+        ::
+
+            LDAP_BIND_URI = 'ldaps://hostname'
+
+.. data:: LDAP_BASE_DN
+    :noindex:
+
+        Specifies the LDAP distinguished name location to search for users
+
+        ::
+
+            LDAP_BASE_DN = 'DC=Users,DC=Evilcorp,DC=com'
+
+.. data:: LDAP_EMAIL_DOMAIN
+    :noindex:
+
+        The email domain used by users in your directory. This is used to build the userPrincipalName to search with.
+
+        ::
+
+            LDAP_EMAIL_DOMAIN = 'evilcorp.com'
+
+The following LDAP options are not required, however TLS is always recommended.
+
+.. data:: LDAP_USE_TLS
+    :noindex:
+
+        Enables the use of TLS when connecting to the LDAP server. Ensure the LDAP_BIND_URI is using ldaps scheme.
+
+        ::
+
+            LDAP_USE_TLS = True
+
+.. data:: LDAP_CACERT_FILE
+    :noindex:
+
+        Specify a Certificate Authority file containing PEM encoded trusted issuer certificates. This can be used if your LDAP server is using certificates issued by a private CA.
+
+        ::
+
+            LDAP_CACERT_FILE = '/path/to/cacert/file'
+
+.. data:: LDAP_REQUIRED_GROUP
+    :noindex:
+
+        Lemur has pretty open permissions. You can define an LDAP group to specify who can access Lemur. Only members of this group will be able to login.
+
+        ::
+
+            LDAP_REQUIRED_GROUP = 'Lemur LDAP Group Name'
+
+.. data:: LDAP_GROUPS_TO_ROLES
+    :noindex:
+
+        You can also define a dictionary of ldap groups mapped to lemur roles. This allows you to use ldap groups to manage access to owner/creator roles in Lemur
+
+        ::
+
+            LDAP_GROUPS_TO_ROLES = {'lemur_admins': 'admin', 'Lemur Team DL Group': 'team@example.com'}
+
+
+.. data:: LDAP_IS_ACTIVE_DIRECTORY
+    :noindex:
+
+        When set to True, nested group memberships are supported, by searching for groups with the member:1.2.840.113556.1.4.1941 attribute set to the user DN.
+        When set to False, the list of groups will be determined by the 'memberof' attribute of the LDAP user logging in.
+
+        ::
+
+            LDAP_IS_ACTIVE_DIRECTORY = False
+
+
+Authentication Providers
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you are not using an authentication provider you do not need to configure any of these options.
+
+For more information about how to use social logins, see: `Satellizer <https://github.com/sahat/satellizer>`_
+
+.. data:: ACTIVE_PROVIDERS
+    :noindex:
+
+        ::
+
+            ACTIVE_PROVIDERS = ["ping", "google", "oauth2"]
+
+.. data:: PING_SECRET
+    :noindex:
+
+        ::
+
+            PING_SECRET = 'somethingsecret'
+
+.. data:: PING_ACCESS_TOKEN_URL
+    :noindex:
+
+        ::
+
+            PING_ACCESS_TOKEN_URL = "https://<yourpingserver>/as/token.oauth2"
+
+
+.. data:: PING_USER_API_URL
+    :noindex:
+
+        ::
+
+            PING_USER_API_URL = "https://<yourpingserver>/idp/userinfo.openid"
+
+.. data:: PING_JWKS_URL
+    :noindex:
+
+        ::
+
+            PING_JWKS_URL = "https://<yourpingserver>/pf/JWKS"
+
+.. data:: PING_NAME
+    :noindex:
+
+        ::
+
+            PING_NAME = "Example Oauth2 Provider"
+
+.. data:: PING_CLIENT_ID
+    :noindex:
+
+        ::
+
+            PING_CLIENT_ID = "client-id"
+
+.. data:: PING_REDIRECT_URI
+    :noindex:
+
+        ::
+
+            PING_REDIRECT_URI = "https://<yourlemurserver>/api/1/auth/ping"
+
+.. data:: PING_AUTH_ENDPOINT
+    :noindex:
+
+        ::
+
+            PING_AUTH_ENDPOINT = "https://<yourpingserver>/oauth2/authorize"
+
+.. data:: PING_USER_MEMBERSHIP_URL
+    :noindex:
+
+        An optional additional endpoint to learn membership details post the user validation.
+
+        ::
+
+            PING_USER_MEMBERSHIP_URL = "https://<yourmembershipendpoint>"
+
+.. data:: PING_USER_MEMBERSHIP_TLS_PROVIDER
+    :noindex:
+
+        A custom TLS session provider plugin name
+
+        ::
+
+            PING_USER_MEMBERSHIP_TLS_PROVIDER = "slug-name"
+
+.. data:: PING_USER_MEMBERSHIP_SERVICE
+    :noindex:
+
+        Membership service name used by PING_USER_MEMBERSHIP_TLS_PROVIDER to create a session
+
+        ::
+
+            PING_USER_MEMBERSHIP_SERVICE = "yourmembershipservice"
+
+.. data:: OAUTH2_SECRET
+    :noindex:
+
+        ::
+
+            OAUTH2_SECRET = 'somethingsecret'
+
+.. data:: OAUTH2_ACCESS_TOKEN_URL
+    :noindex:
+
+        ::
+
+            OAUTH2_ACCESS_TOKEN_URL = "https://<youroauthserver> /oauth2/v1/authorize"
+
+
+.. data:: OAUTH2_USER_API_URL
+    :noindex:
+
+        ::
+
+            OAUTH2_USER_API_URL = "https://<youroauthserver>/oauth2/v1/userinfo"
+
+.. data:: OAUTH2_JWKS_URL
+    :noindex:
+
+        ::
+
+            OAUTH2_JWKS_URL = "https://<youroauthserver>/oauth2/v1/keys"
+
+.. data:: OAUTH2_NAME
+    :noindex:
+
+        ::
+
+            OAUTH2_NAME = "Example Oauth2 Provider"
+
+.. data:: OAUTH2_CLIENT_ID
+    :noindex:
+
+        ::
+
+            OAUTH2_CLIENT_ID = "client-id"
+
+.. data:: OAUTH2_REDIRECT_URI
+    :noindex:
+
+        ::
+
+            OAUTH2_REDIRECT_URI = "https://<yourlemurserver>/api/1/auth/oauth2"
+
+.. data:: OAUTH2_AUTH_ENDPOINT
+    :noindex:
+
+        ::
+
+            OAUTH2_AUTH_ENDPOINT = "https://<youroauthserver>/oauth2/v1/authorize"
+
+.. data:: OAUTH2_VERIFY_CERT
+    :noindex:
+
+        ::
+
+            OAUTH2_VERIFY_CERT = True
+
+.. data:: GOOGLE_CLIENT_ID
+    :noindex:
+
+        ::
+
+            GOOGLE_CLIENT_ID = "client-id"
+
+.. data:: GOOGLE_SECRET
+    :noindex:
+
+        ::
+
+            GOOGLE_SECRET = "somethingsecret"
+
+
+Metric Providers
+~~~~~~~~~~~~~~~~
+
+If you are not using a metric provider you do not need to configure any of these options.
+
+.. data:: ACTIVE_PROVIDERS
+    :noindex:
+
+        A list of metric plugins slugs to be ativated.
+
+        ::
+
+            METRIC_PROVIDERS = ['atlas-metric']
+
+
+Plugin Specific Options
+-----------------------
+
+ACME Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. data:: ACME_DNS_PROVIDER_TYPES
+    :noindex:
+
+        Dictionary of ACME DNS Providers and their requirements.
+
+.. data:: ACME_ENABLE_DELEGATED_CNAME
+    :noindex:
+
+        Enables delegated DNS domain validation using CNAMES.  When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges.
+
+
+The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
+account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
+
+
+.. data:: ACME_PRIVATE_KEY
+    :noindex:
+
+            This is the private key, the account was registered with (in JWK format)
+
+.. data:: ACME_REGR
+    :noindex:
+
+            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
+
+.. data:: ACME_PREFERRED_ISSUER
+    :noindex:
+
+            This is an optional parameter to indicate the preferred chain to retrieve from ACME when finalizing the order.
+            This is applicable to Let's Encrypts recent `migration <https://letsencrypt.org/certificates/>`_ to their
+            own root, where they provide two distinct certificate chains (fullchain_pem vs. alternative_fullchains_pem);
+            the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain
+            is rooted in X1 root CA.
+            Select "X1" to get the shorter chain (currently alternative), leave blank or "DST Root CA X3" for the longer chain.
+
+
+Active Directory Certificate Services Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+.. data:: ADCS_SERVER
+    :noindex:
+
+        FQDN of your ADCS Server
+
+
+.. data:: ADCS_AUTH_METHOD
+    :noindex:
+
+        The chosen authentication method. Either ‘basic’ (the default), ‘ntlm’ or ‘cert’ (SSL client certificate). The next 2 variables are interpreted differently for different methods.
+
+
+.. data:: ADCS_USER
+    :noindex:
+
+        The username (basic) or the path to the public cert (cert) of the user accessing PKI
+
+
+.. data:: ADCS_PWD
+    :noindex:
+
+        The passwd (basic) or the path to the private key (cert) of the user accessing PKI
+
+
+.. data:: ADCS_TEMPLATE
+    :noindex:
+
+        Template to be used for certificate issuing. Usually display name w/o spaces
+        
+.. data:: ADCS_TEMPLATE_<upper(authority.name)>
+    :noindex:
+
+        If there is a config variable ADCS_TEMPLATE_<upper(authority.name)> take the value as Cert template else default to ADCS_TEMPLATE to be compatible with former versions. Template to be used for certificate issuing. Usually display name w/o spaces
+
+.. data:: ADCS_START
+    :noindex:
+
+        Used in ADCS-Sourceplugin. Minimum id of the first certificate to be returned. ID is increased by one until ADCS_STOP. Missing cert-IDs are ignored
+
+.. data:: ADCS_STOP
+    :noindex:
+
+        Used for ADCS-Sourceplugin. Maximum id of the certificates returned. 
+        
+
+.. data:: ADCS_ISSUING
+    :noindex:
+
+        Contains the issuing cert of the CA
+
+
+.. data:: ADCS_ROOT
+    :noindex:
+
+        Contains the root cert of the CA
+
+Entrust Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Enables the creation of Entrust certificates. You need to set the API access up with Entrust support. Check the information in the Entrust Portal as well. 
+Certificates are created as "SERVER_AND_CLIENT_AUTH".
+Caution: Sometimes the entrust API does not respond in a timely manner. This error is handled and reported by the plugin. Should this happen you just have to hit the create button again after to create a valid certificate. 
+The following parameters have to be set in the configuration files.
+
+.. data:: ENTRUST_URL
+    :noindex:
+    
+       This is the url for the Entrust API. Refer to the API documentation.
+       
+.. data:: ENTRUST_API_CERT
+    :noindex:
+    
+       Path to the certificate file in PEM format. This certificate is created in the onboarding process. Refer to the API documentation.
+       
+.. data:: ENTRUST_API_KEY
+    :noindex:
+    
+       Path to the key file in RSA format. This certificate is created in the onboarding process. Refer to the API documentation. Caution: the request library cannot handle encrypted keys. The keyfile therefore has to contain the unencrypted key. Please put this in a secure location on the server.
+       
+.. data:: ENTRUST_API_USER
+    :noindex:
+    
+       String with the API user. This user is created in the onboarding process. Refer to the API documentation.   
+       
+.. data:: ENTRUST_API_PASS
+    :noindex:
+    
+       String with the password for the API user. This password is created in the onboarding process. Refer to the API documentation.
+
+.. data:: ENTRUST_NAME
+    :noindex:
+    
+        String with the name that should appear as certificate owner in the Entrust portal. Refer to the API documentation.
+
+.. data:: ENTRUST_EMAIL
+    :noindex:
+    
+        String with the email address that should appear as certificate contact email in the Entrust portal. Refer to the API documentation.       
+
+.. data:: ENTRUST_PHONE
+    :noindex:
+    
+        String with the phone number that should appear as certificate contact in the Entrust portal. Refer to the API documentation.        
+
+.. data:: ENTRUST_ISSUING
+    :noindex:
+    
+        Contains the issuing cert of the CA
+
+.. data:: ENTRUST_ROOT
+    :noindex:
+    
+        Contains the root cert of the CA
+
+.. data:: ENTRUST_PRODUCT_<upper(authority.name)>
+    :noindex:
+
+        If there is a config variable ENTRUST_PRODUCT_<upper(authority.name)> take the value as cert product name else default to "STANDARD_SSL". Refer to the API documentation for valid products names.
+
+
+.. data:: ENTRUST_CROSS_SIGNED_RSA_L1K
+    :noindex:
+
+        This is optional. Entrust provides support for cross-signed subCAS. One can set ENTRUST_CROSS_SIGNED_RSA_L1K to the respective cross-signed RSA-based subCA PEM and Lemur will replace the retrieved subCA with ENTRUST_CROSS_SIGNED_RSA_L1K.
+
+
+.. data:: ENTRUST_CROSS_SIGNED_ECC_L1F
+    :noindex:
+
+        This is optional. Entrust provides support for cross-signed subCAS. One can set ENTRUST_CROSS_SIGNED_ECC_L1F to the respective cross-signed EC-based subCA PEM and Lemur will replace the retrieved subCA with ENTRUST_CROSS_SIGNED_ECC_L1F.
+
+
+.. data:: ENTRUST_USE_DEFAULT_CLIENT_ID
+    :noindex:
+
+        If set to True, Entrust will use the primary client ID of 1, which applies to most use-case.
+        Otherwise, Entrust will first lookup the clientId before ordering the certificate.
+
+
+Verisign Issuer Plugin
+~~~~~~~~~~~~~~~~~~~~~~
+
+Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur,
+Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation
+for those plugins.
+
+.. data:: VERISIGN_URL
+    :noindex:
+
+        This is the url for the Verisign API
+
+
+.. data:: VERISIGN_PEM_PATH
+    :noindex:
+
+        This is the path to the mutual TLS certificate used for communicating with Verisign
+
+
+.. data:: VERISIGN_FIRST_NAME
+    :noindex:
+
+        This is the first name to be used when requesting the certificate
+
+
+.. data:: VERISIGN_LAST_NAME
+    :noindex:
+
+        This is the last name to be used when requesting the certificate
+
+.. data:: VERISIGN_EMAIL
+    :noindex:
+
+        This is the email to be used when requesting the certificate
+
+
+.. data:: VERISIGN_INTERMEDIATE
+    :noindex:
+
+        This is the intermediate to be used for your CA chain
+
+
+.. data:: VERISIGN_ROOT
+    :noindex:
+
+        This is the root to be used for your CA chain
+
+
+Digicert Issuer Plugin
+~~~~~~~~~~~~~~~~~~~~~~
+
+The following configuration properties are required to use the Digicert issuer plugin.
+
+
+.. data:: DIGICERT_URL
+    :noindex:
+
+            This is the url for the Digicert API (e.g. https://www.digicert.com)
+
+
+.. data:: DIGICERT_ORDER_TYPE
+    :noindex:
+
+            This is the type of certificate to order. (e.g. ssl_plus, ssl_ev_plus see: https://www.digicert.com/services/v2/documentation/order/overview-submit)
+
+
+.. data:: DIGICERT_API_KEY
+    :noindex:
+
+            This is the Digicert API key
+
+
+.. data:: DIGICERT_ORG_ID
+    :noindex:
+
+            This is the Digicert organization ID tied to your API key
+
+
+.. data:: DIGICERT_ROOT
+    :noindex:
+
+            This is the root to be used for your CA chain
+
+
+.. data:: DIGICERT_DEFAULT_VALIDITY_DAYS
+    :noindex:
+
+            This is the default validity (in days), if no end date is specified. (Default: 397)
+
+
+.. data:: DIGICERT_MAX_VALIDITY_DAYS
+    :noindex:
+
+            This is the maximum validity (in days). (Default: value of DIGICERT_DEFAULT_VALIDITY_DAYS)
+
+
+.. data:: DIGICERT_PRIVATE
+    :noindex:
+
+            This is whether or not to issue a private certificate. (Default: False)
+
+
+CFSSL Issuer Plugin
+~~~~~~~~~~~~~~~~~~~
+
+The following configuration properties are required to use the CFSSL issuer plugin.
+
+.. data:: CFSSL_URL
+    :noindex:
+
+        This is the URL for the CFSSL API
+
+.. data:: CFSSL_ROOT
+    :noindex:
+
+        This is the root to be used for your CA chain
+
+.. data:: CFSSL_INTERMEDIATE
+    :noindex:
+
+        This is the intermediate to be used for your CA chain
+
+.. data:: CFSSL_KEY
+    :noindex:
+
+        This is the hmac key to authenticate to the CFSSL service. (Optional)
+
+
+Hashicorp Vault Source/Destination Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Lemur can import and export certificate data to and from a Hashicorp Vault secrets store. Lemur can connect to a different Vault service per source/destination.
+
+.. note:: This plugin does not supersede or overlap the 3rd party Vault Issuer plugin.
+
+.. note:: Vault does not have any configuration properties however it does read from a file on disk for a vault access token. The Lemur service account needs read access to this file.
+
+Vault Source
+""""""""""""
+
+The Vault Source Plugin will read from one Vault object location per source defined. There is expected to be one or more certificates defined in each object in Vault.
+
+Vault Destination
+"""""""""""""""""
+
+A Vault destination can be one object in Vault or a directory where all certificates will be stored as their own object by CN.
+
+Vault Destination supports a regex filter to prevent certificates with SAN that do not match the regex filter from being deployed. This is an optional feature per destination defined.
+
+
+AWS Source/Destination Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions.
+
+.. note:: AWS usage is completely optional. Lemur can upload, find and manage TLS certificates in AWS. But is not required to do so.
+
+Setting up IAM roles
+""""""""""""""""""""
+
+Lemur's AWS plugin uses boto heavily to talk to all the AWS resources it manages. By default it uses the on-instance credentials to make the necessary calls.
+
+In order to limit the permissions, we will create two new IAM roles for Lemur. You can name them whatever you would like but for example sake we will be calling them LemurInstanceProfile and Lemur.
+
+Lemur uses to STS to talk to different accounts. For managing one account this isn't necessary but we will still use it so that we can easily add new accounts.
+
+LemurInstanceProfile is the IAM role you will launch your instance with. It actually has almost no rights. In fact it should really only be able to use STS to assume role to the Lemur role.
+
+Here are example policies for the LemurInstanceProfile:
+
+SES-SendEmail
+
+.. code-block:: python
+
+    {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Effect": "Allow",
+          "Action": [
+            "ses:SendEmail"
+          ],
+          "Resource": "*"
+        }
+      ]
+    }
+
+
+STS-AssumeRole
+
+.. code-block:: python
+
+    {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Effect": "Allow",
+          "Action":
+            "sts:AssumeRole",
+          "Resource": "*"
+        }
+      ]
+    }
+
+
+
+Next we will create the Lemur IAM role.
+
+.. note::
+
+    The default IAM role that Lemur assumes into is called `Lemur`, if you need to change this ensure you set `LEMUR_INSTANCE_PROFILE` to your role name in the configuration.
+
+
+Here is an example policy for Lemur:
+
+IAM-ServerCertificate
+
+.. code-block:: python
+
+    {
+        "Statement": [
+                    {
+                         "Action": [
+                              "iam:ListServerCertificates",
+                              "iam:UpdateServerCertificate",
+                              "iam:GetServerCertificate",
+                              "iam:UploadServerCertificate"
+                         ],
+                         "Resource": [
+                              "*"
+                         ],
+                         "Effect": "Allow",
+                         "Sid": "Stmt1404836868000"
+                    }
+               ]
+    }
+
+
+.. code-block:: python
+
+    {
+        "Statement": [
+                    {
+                         "Action": [
+                              "elasticloadbalancing:DescribeInstanceHealth",
+                              "elasticloadbalancing:DescribeLoadBalancerAttributes",
+                              "elasticloadbalancing:DescribeLoadBalancerPolicyTypes",
+                              "elasticloadbalancing:DescribeLoadBalancerPolicies",
+                              "elasticloadbalancing:DescribeLoadBalancers",
+                              "elasticloadbalancing:DeleteLoadBalancerListeners",
+                              "elasticloadbalancing:CreateLoadBalancerListeners"
+                         ],
+                         "Resource": [
+                              "*"
+                         ],
+                         "Effect": "Allow",
+                         "Sid": "Stmt1404841912000"
+                    }
+               ]
+    }
+
+
+Setting up STS access
+"""""""""""""""""""""
+
+Once we have setup our accounts we need to ensure that we create a trust relationship so that LemurInstanceProfile can assume the Lemur role.
+
+In the AWS console select the Lemur IAM role and select the Trust Relationships tab and click Edit Trust Relationship
+
+Below is an example policy:
+
+.. code-block:: python
+
+    {
+      "Version": "2008-10-17",
+      "Statement": [
+        {
+          "Sid": "",
+          "Effect": "Allow",
+          "Principal": {
+            "AWS": [
+              "arn:aws:iam::<awsaccountnumber>:role/LemurInstanceProfile",
+            ]
+          },
+          "Action": "sts:AssumeRole"
+        }
+      ]
+    }
+
+
+Adding N+1 accounts
+"""""""""""""""""""
+
+To add another account we go to the new account and create a new Lemur IAM role with the same policy as above.
+
+Then we would go to the account that Lemur is running is and edit the trust relationship policy.
+
+An example policy:
+
+.. code-block:: python
+
+    {
+      "Version": "2008-10-17",
+      "Statement": [
+        {
+          "Sid": "",
+          "Effect": "Allow",
+          "Principal": {
+            "AWS": [
+              "arn:aws:iam::<awsaccountnumber>:role/LemurInstanceProfile",
+              "arn:aws:iam::<awsaccountnumber1>:role/LemurInstanceProfile",
+            ]
+          },
+          "Action": "sts:AssumeRole"
+        }
+      ]
+    }
+
+Setting up SES
+""""""""""""""
+
+Lemur has built in support for sending it's certificate notifications via Amazon's simple email service (SES). To force
+Lemur to use SES ensure you are the running as the IAM role defined above and that you have followed the steps outlined
+in Amazon's documentation `Setting up Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/setting-up-ses.html>`_
+
+The configuration::
+
+    LEMUR_MAIL = 'lemur.example.com'
+
+Will be the sender of all notifications, so ensure that it is verified with AWS.
+
+SES if the default notification gateway and will be used unless SMTP settings are configured in the application configuration
+settings.
+
+PowerDNS ACME Plugin
+~~~~~~~~~~~~~~~~~~~~~~
+
+The following configuration properties are required to use the PowerDNS ACME Plugin for domain validation.
+
+
+.. data:: ACME_POWERDNS_DOMAIN
+    :noindex:
+
+            This is the FQDN for the PowerDNS API (without path)
+
+
+.. data:: ACME_POWERDNS_SERVERID
+    :noindex:
+
+            This is the ServerID attribute of the PowerDNS API Server (i.e. "localhost")
+
+
+.. data:: ACME_POWERDNS_APIKEYNAME
+    :noindex:
+
+            This is the Key name to use for authentication (i.e. "X-API-Key")
+
+
+.. data:: ACME_POWERDNS_APIKEY
+    :noindex:
+
+            This is the API Key to use for authentication (i.e. "Password")
+
+
+.. data:: ACME_POWERDNS_RETRIES
+    :noindex:
+
+            This is the number of times DNS Verification should be attempted (i.e. 20)
+
+
+.. data:: ACME_POWERDNS_VERIFY
+    :noindex:
+
+            This configures how TLS certificates on the PowerDNS API target are validated.  The PowerDNS Plugin depends on the PyPi requests library, which supports the following options for the verify parameter:
+
+            True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default)
+
+            False: Disables certificate validation (Not Recommended)
+
+            File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle.
+
+.. _CommandLineInterface:
+
+Command Line Interface
+======================
+
+Lemur installs a command line script under the name ``lemur``. This will allow you to
+perform most required operations that are unachievable within the web UI.
+
+If you're using a non-standard configuration location, you'll need to prefix every command with
+--config (excluding create_config, which is a special case). For example::
+
+    lemur --config=/etc/lemur.conf.py help
+
+For a list of commands, you can also use ``lemur help``, or ``lemur [command] --help``
+for help on a specific command.
+
+.. note:: The script is powered by a library called `Flask-Script <https://github.com/smurfix/flask-script>`_
+
+Builtin Commands
+----------------
+
+All commands default to `~/.lemur/lemur.conf.py` if a configuration is not specified.
+
+.. data:: create_config
+
+    Creates a default configuration file for Lemur.
+
+    Path defaults to ``~/.lemur/lemur.config.py``
+
+    ::
+
+        lemur create_config .
+
+    .. note::
+        This command is a special case and does not depend on the configuration file
+        being set.
+
+
+.. data:: init
+
+    Initializes the configuration file for Lemur.
+
+    ::
+
+        lemur -c /etc/lemur.conf.py init
+
+
+.. data:: start
+
+    Starts a Lemur service. You can also pass any flag that Gunicorn uses to specify the webserver configuration.
+
+    ::
+
+        lemur start -w 6 -b 127.0.0.1:8080
+
+
+.. data:: db upgrade
+
+    Performs any needed database migrations.
+
+    ::
+
+        lemur db upgrade
+
+
+.. data:: check_revoked
+
+    Traverses every certificate that Lemur is aware of and attempts to understand its validity.
+    It utilizes both OCSP and CRL. If Lemur is unable to come to a conclusion about a certificates
+    validity its status is marked 'unknown'.
+
+
+.. data:: sync
+
+    Sync attempts to discover certificates in the environment that were not created by Lemur. If you wish to only sync
+    a few sources you can pass a comma delimited list of sources to sync.
+
+    ::
+
+        lemur sync -s source1,source2
+
+
+    Additionally you can also list the available sources that Lemur can sync.
+
+    ::
+
+        lemur sync
+
+
+.. data:: notify
+
+    Will traverse all current notifications and see if any of them need to be triggered.
+
+    ::
+
+        lemur notify
+
+
+.. data:: acme
+
+    Handles all ACME related tasks, like ACME plugin testing.
+
+    ::
+
+        lemur acme
+
+
+Sub-commands
+------------
+
+Lemur includes several sub-commands for interacting with Lemur such as creating new users, creating new roles and even
+issuing certificates.
+
+The best way to discover these commands is by using the built in help pages
+
+    ::
+
+        lemur --help
+
+
+and to get help on sub-commands
+
+    ::
+
+        lemur certificates --help
+
+
+
+Upgrading Lemur
+===============
+
+To upgrade Lemur to the newest release you will need to ensure you have the latest code and have run any needed
+database migrations.
+
+To get the latest code from github run
+
+    ::
+
+        cd <lemur-source-directory>
+        git pull -t <version>
+        python setup.py develop
+
+
+.. note::
+    It's important to grab the latest release by specifying the release tag. This tags denote stable versions of Lemur.
+    If you want to try the bleeding edge version of Lemur you can by using the master branch.
+
+
+After you have the latest version of the Lemur code base you must run any needed database migrations. To run migrations
+
+    ::
+
+        cd <lemur-source-directory>/lemur
+        lemur db upgrade
+
+
+This will ensure that any needed tables or columns are created or destroyed.
+
+.. note::
+    Internally, this uses `Alembic <http://alembic.zzzcomputing.com/en/latest/>`_ to manage database migrations.
+
+.. note::
+    By default Alembic looks for the `migrations` folder in the current working directory.The migrations folder is
+    located under `<LEMUR_HOME>/lemur/migrations` if you are running the lemur command from any location besides
+    `<LEMUR_HOME>/lemur` you will need to pass the `-d` flag to specify the absolute file path to the `migrations` folder.
+
+Plugins
+=======
+
+There are several interfaces currently available to extend Lemur. These are a work in
+progress and the API is not frozen.
+
+Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec.
+
+Verisign/Symantec
+-----------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for the VICE 2.0 API
+
+
+Cryptography
+------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Issuer
+:Description:
+    Toy certificate authority that creates self-signed certificate authorities.
+    Allows for the creation of arbitrary authorities and end-entity certificates.
+    This is *not* recommended for production use.
+
+
+Acme
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>,
+    Chad Sine <csine@netflix.com>
+:Type:
+    Issuer
+:Description:
+    Adds support for the ACME protocol (including LetsEncrypt) with domain validation using several providers.
+
+
+Atlas
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
+:Type:
+    Metric
+:Description:
+    Adds basic support for the `Atlas <https://github.com/Netflix/atlas/wiki>`_ telemetry system.
+
+
+Email
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
+:Type:
+    Notification
+:Description:
+    Adds support for basic email notifications via SES.
+
+
+Slack
+-----
+
+:Authors:
+    Harm Weites <harm@weites.com>
+:Type:
+    Notification
+:Description:
+    Adds support for slack notifications.
+
+
+AWS (Source)
+------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
+:Type:
+    Source
+:Description:
+    Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment.
+
+
+AWS (Destination)
+-----------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Curtis Castrapel <ccastrapel@netflix.com>,
+    Hossein Shafagh <hshafagh@netflix.com>
+:Type:
+    Destination
+:Description:
+    Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment.
+
+
+AWS (SNS Notification)
+----------------------
+
+:Authors:
+    Jasmine Schladen <jschladen@netflix.com>
+:Type:
+    Notification
+:Description:
+    Adds support for SNS notifications. SNS notifications (like other notification plugins) are currently only supported
+    for certificate expiration. Configuration requires a region, account number, and SNS topic name; these elements
+    are then combined to build the topic ARN. Lemur must have access to publish messages to the specified SNS topic.
+
+
+Kubernetes
+----------
+
+:Authors:
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Destination
+:Description:
+    Allows Lemur to upload generated certificates to the Kubernetes certificate store.
+
+
+Java
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Generates java compatible .jks keystores and truststores from Lemur managed certificates.
+
+
+Openssl
+-------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Leverages Openssl to support additional export formats (pkcs12)
+
+
+CFSSL
+-----
+
+:Authors:
+    Charles Hendrie <chad.hendrie@thomsonreuters.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for generating certificates from the private certificate authority CFSSL
+
+Vault
+-----
+
+:Authors:
+    Christopher Jolley <chris@alwaysjolley.com>
+:Type:
+    Source
+:Description:
+    Source plugin imports certificates from Hashicorp Vault secret store.
+
+Vault
+-----
+
+:Authors:
+    Christopher Jolley <chris@alwaysjolley.com>
+:Type:
+    Destination
+:Description:
+    Destination plugin to deploy certificates to Hashicorp Vault secret store.
+
+
+3rd Party Plugins
+=================
+
+The following plugins are available and maintained by members of the Lemur community:
+
+Digicert
+--------
+
+:Authors:
+    Chris Dorros
+:Type:
+    Issuer
+:Description:
+    Adds support for basic Digicert
+:Links:
+    https://github.com/opendns/lemur-digicert
+
+
+InfluxDB
+--------
+
+:Authors:
+    Titouan Christophe
+:Type:
+    Metric
+:Description:
+    Sends key metrics to InfluxDB
+:Links:
+    https://github.com/titouanc/lemur-influxdb
+
+Hashicorp Vault
+---------------
+
+:Authors:
+    Ron Cohen
+:Type:
+    Issuer
+:Description:
+    Adds support for basic Vault PKI secret backend.
+:Links:
+    https://github.com/RcRonco/lemur_vault
+
+
+Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
+get it added.
+
+Want to create your own extension? See :doc:`../developer/plugins/index` to get started.
+
+
+Identity and Access Management
+==============================
+
+Lemur uses a Role Based Access Control (RBAC) mechanism to control which users have access to which resources. When a
+user is first created in Lemur they can be assigned one or more roles. These roles are typically dynamically created
+depending on an external identity provider (Google, LDAP, etc.), or are hardcoded within Lemur and associated with special
+meaning.
+
+Within Lemur there are three main permissions: AdminPermission, CreatorPermission, OwnerPermission. Sub-permissions such
+as ViewPrivateKeyPermission are compositions of these three main Permissions.
+
+Lets take a look at how these permissions are used:
+
+Each `Authority` has a set of roles associated with it. If a user is also associated with the same roles
+that the `Authority` is associated with, Lemur allows that user to user/view/update that `Authority`.
+
+This RBAC is also used when determining which users can access which certificate private key. Lemur's current permission
+structure is setup such that if the user is a `Creator` or `Owner` of a given certificate they are allow to view that
+private key. Owners can also be a role name, such that any user with the same role as owner will be allowed to view the
+private key information.
+
+These permissions are applied to the user upon login and refreshed on every request.
+
+.. seealso::
+
+    `Flask-Principal <https://pythonhosted.org/Flask-Principal>`_

docs/administration/index.rst

@@ -1,660 +0,0 @@
-Configuration
-=============
-
-.. warning::
-    There are many secrets that Lemur uses that must be protected. All of these options are set via the Lemur configuration
-    file. It is highly advised that you do not store your secrets in this file! Lemur provides functions
-    that allow you to encrypt files at rest and decrypt them when it's time for deployment. See :ref:`Credential Management <CredentialManagement>`
-    for more information.
-
-Basic Configuration
--------------------
-
-.. data:: LOG_LEVEL
-    :noindex:
-
-    ::
-
-        LOG_LEVEL = "DEBUG"
-
-.. data:: LOG_FILE
-    :noindex:
-
-    ::
-
-        LOG_FILE = "/logs/lemur/lemur-test.log"
-
-
-.. data:: debug
-    :noindex:
-
-    Sets the flask debug flag to true (if supported by the webserver)
-
-    ::
-
-        debug = False
-
-
-    .. warning::
-        This should never be used in a production environment as it exposes Lemur to
-        remote code execution through the debug console.
-
-
-.. data:: CORS
-    :noindex:
-
-    Allows for cross domain requests, this is most commonly used for development but could
-    be use in production if you decided to host the webUI on a different domain than the server.
-
-    Use this cautiously, if you're not sure. Set it to `False`
-
-    ::
-
-        CORS = False
-
-
-.. data:: SQLACHEMY_DATABASE_URI
-    :noindex:
-
-        If you have ever used sqlalchemy before this is the standard connection string used. Lemur uses a postgres database and the connection string would look something like:
-
-    ::
-
-        SQLALCHEMY_DATABASE_URI = 'postgresql://<user>:<password>@<hostname>:5432/lemur'
-
-
-.. data:: LEMUR_RESTRICTED_DOMAINS
-    :noindex:
-
-        This allows the administrator to mark a subset of domains or domains matching a particular regex as
-        *restricted*. This means that only an administrator is allows to issue the domains in question.
-
-.. data:: LEMUR_TOKEN_SECRET
-    :noindex:
-
-        The TOKEN_SECRET is the secret used to create JWT tokens that are given out to users. This should be securely generated and kept private.
-
-    ::
-
-        LEMUR_TOKEN_SECRET = 'supersecret'
-
-    An example of how you might generate a random string:
-
-        >>> import random
-        >>> secret_key = ''.join(random.choice(string.ascii_uppercase) for x in range(6))
-        >>> secret_key = secret_key + ''.join(random.choice("~!@#$%^&*()_+") for x in range(6))
-        >>> secret_key = secret_key + ''.join(random.choice(string.ascii_lowercase) for x in range(6))
-        >>> secret_key = secret_key + ''.join(random.choice(string.digits) for x in range(6))
-
-
-.. data:: LEMUR_ENCRYPTION_KEYS
-    :noindex:
-
-        The LEMUR_ENCRYPTION_KEYS is used to encrypt data at rest within Lemur's database. Without a key Lemur will refuse
-        to start. Multiple keys can be provided to facilitate key rotation. The first key in the list is used for
-        encryption and all keys are tried for decryption until one works. Each key must be 32 URL safe base-64 encoded bytes.
-
-        Running lemur create_config will securely generate a key for your configuration file.
-        If you would like to generate your own, we recommend the following method:
-
-            >>> import os
-            >>> import base64
-            >>> base64.urlsafe_b64encode(os.urandom(32))
-
-    ::
-
-        LEMUR_ENCRYPTION_KEYS = ['1YeftooSbxCiX2zo8m1lXtpvQjy27smZcUUaGmffhMY=', 'LAfQt6yrkLqOK5lwpvQcT4jf2zdeTQJV1uYeh9coT5s=']
-
-
-Certificate Default Options
----------------------------
-
-Lemur allows you to fine tune your certificates to your organization. The following defaults are presented in the UI
-and are used when Lemur creates the CSR for your certificates.
-
-
-.. data:: LEMUR_DEFAULT_COUNTRY
-    :noindex:
-
-    ::
-
-        LEMUR_DEFAULT_COUNTRY = "US"
-
-
-.. data:: LEMUR_DEFAULT_STATE
-    :noindex:
-
-    ::
-
-        LEMUR_DEFAULT_STATE = "California"
-
-
-.. data:: LEMUR_DEFAULT_LOCATION
-    :noindex:
-
-    ::
-
-        LEMUR_DEFAULT_LOCATION = "Los Gatos"
-
-
-.. data:: LEMUR_DEFAULT_ORGANIZATION
-    :noindex:
-
-    ::
-
-        LEMUR_DEFAULT_ORGANIZATION = "Netflix"
-
-
-.. data:: LEMUR_DEFAULT_ORGANIZATION_UNIT
-    :noindex:
-
-    ::
-
-        LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = "Operations"
-
-
-Notification Options
---------------------
-
-Lemur currently has very basic support for notifications. Currently only expiration notifications are supported. Actual notification
-is handled by the notification plugins that you have configured. Lemur ships with the 'Email' notification that allows expiration emails
-to be sent to subscribers.
-
-Templates for expiration emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
-Notifications are sent to the certificate creator, owner and security team as specified by the `LEMUR_SECURITY_TEAM_EMAIL` configuration parameter.
-
-Certificates marked as inactive will **not** be notified of upcoming expiration. This enables a user to essentially
-silence the expiration. If a certificate is active and is expiring the above will be notified according to the `LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS` or
-30, 15, 2 days before expiration if no intervals are set.
-
-Lemur supports sending certification expiration notifications through SES and SMTP.
-
-
-.. data:: LEMUR_EMAIL_SENDER
-    :noindex:
-
-            Specifies which service will be delivering notification emails. Valid values are `SMTP` or `SES`
-
-.. note::
-    If using STMP as your provider you will need to define additional configuration options as specified by Flask-Mail.
-    See: `Flask-Mail <https://pythonhosted.org/Flask-Mail>`_
-
-    If you are using SES the email specified by the `LEMUR_MAIL` configuration will need to be verified by AWS before
-    you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
-
-.. data:: LEMUR_MAIL
-    :noindex:
-
-            Lemur sender's email
-
-        ::
-
-        LEMUR_MAIL = 'lemur.example.com'
-
-
-.. data:: LEMUR_SECURITY_TEAM_EMAIL
-    :noindex:
-
-            This is an email or list of emails that should be notified when a certificate is expiring. It is also the contact email address for any discovered certificate.
-
-        ::
-
-        LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
-
-
-.. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS
-    :noindex:
-
-            Lemur notification intervals
-
-        ::
-
-        LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
-
-
-Authority Options
------------------
-
-Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur,
-Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation
-for those plugins.
-
-.. data:: VERISIGN_URL
-    :noindex:
-
-        This is the url for the Verisign API
-
-
-.. data:: VERISIGN_PEM_PATH
-    :noindex:
-
-        This is the path to the mutual TLS certificate used for communicating with Verisign
-
-
-.. data:: VERISIGN_FIRST_NAME
-    :noindex:
-
-        This is the first name to be used when requesting the certificate
-
-
-.. data:: VERISIGN_LAST_NAME
-    :noindex:
-
-        This is the last name to be used when requesting the certificate
-
-.. data:: VERISIGN_EMAIL
-    :noindex:
-
-        This is the email to be used when requesting the certificate
-
-
-.. data:: VERISIGN_INTERMEDIATE
-    :noindex:
-
-        This is the intermediate to be used for your CA chain
-
-
-.. data:: VERISIGN_ROOT
-    :noindex:
-
-        This is the root to be used for your CA chain
-
-
-Authentication
---------------
-Lemur currently supports Basic Authentication and Ping OAuth2 out of the box. Additional flows can be added relatively easily.
-If you are not using Ping you do not need to configure any of these options.
-
-For more information about how to use social logins, see: `Satellizer <https://github.com/sahat/satellizer>`_
-
-.. data:: PING_SECRET
-    :noindex:
-
-    ::
-
-        PING_SECRET = 'somethingsecret'
-
-.. data:: PING_ACCESS_TOKEN_URL
-    :noindex:
-
-    ::
-
-        PING_ACCESS_TOKEN_URL = "https://<yourpingserver>/as/token.oauth2"
-
-
-.. data:: PING_USER_API_URL
-    :noindex:
-
-    ::
-
-        PING_USER_API_URL = "https://<yourpingserver>/idp/userinfo.openid"
-
-.. data:: PING_JWKS_URL
-    :noindex:
-
-    ::
-
-        PING_JWKS_URL = "https://<yourpingserver>/pf/JWKS"
-
-
-
-AWS Plugin Configuration
-========================
-
-In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions.
-
-.. note:: AWS usage is completely optional. Lemur can upload, find and manage TLS certificates in AWS. But is not required to do so.
-
-Setting up IAM roles
---------------------
-
-Lemur's AWS plugin uses boto heavily to talk to all the AWS resources it manages. By default it uses the on-instance credentials to make the necessary calls.
-
-In order to limit the permissions, we will create two new IAM roles for Lemur. You can name them whatever you would like but for example sake we will be calling them LemurInstanceProfile and Lemur.
-
-Lemur uses to STS to talk to different accounts. For managing one account this isn't necessary but we will still use it so that we can easily add new accounts.
-
-LemurInstanceProfile is the IAM role you will launch your instance with. It actually has almost no rights. In fact it should really only be able to use STS to assume role to the Lemur role.
-
-Here are example policies for the LemurInstanceProfile:
-
-SES-SendEmail
-
-.. code-block:: python
-
-    {
-      "Version": "2012-10-17",
-      "Statement": [
-        {
-          "Effect": "Allow",
-          "Action": [
-            "ses:SendEmail"
-          ],
-          "Resource": "*"
-        }
-      ]
-    }
-
-
-STS-AssumeRole
-
-.. code-block:: python
-
-    {
-      "Version": "2012-10-17",
-      "Statement": [
-        {
-          "Effect": "Allow",
-          "Action":
-            "sts:AssumeRole",
-          "Resource": "*"
-        }
-      ]
-    }
-
-
-
-Next we will create the the Lemur IAM role.
-
-.. note::
-
-    The default IAM role that Lemur assumes into is called `Lemur`, if you need to change this ensure you set `LEMUR_INSTANCE_PROFILE` to your role name in the configuration.
-
-
-Here is an example policy for Lemur:
-
-IAM-ServerCertificate
-
-.. code-block:: python
-
-    {
-        "Statement": [
-                    {
-                         "Action": [
-                              "iam:ListServerCertificates",
-                              "iam:UpdateServerCertificate",
-                              "iam:GetServerCertificate",
-                              "iam:UploadServerCertificate"
-                         ],
-                         "Resource": [
-                              "*"
-                         ],
-                         "Effect": "Allow",
-                         "Sid": "Stmt1404836868000"
-                    }
-               ]
-    }
-
-
-.. code-block:: python
-
-    {
-        "Statement": [
-                    {
-                         "Action": [
-                              "elasticloadbalancing:DescribeInstanceHealth",
-                              "elasticloadbalancing:DescribeLoadBalancerAttributes",
-                              "elasticloadbalancing:DescribeLoadBalancerPolicyTypes",
-                              "elasticloadbalancing:DescribeLoadBalancerPolicies",
-                              "elasticloadbalancing:DescribeLoadBalancers",
-                              "elasticloadbalancing:DeleteLoadBalancerListeners",
-                              "elasticloadbalancing:CreateLoadBalancerListeners"
-                         ],
-                         "Resource": [
-                              "*"
-                         ],
-                         "Effect": "Allow",
-                         "Sid": "Stmt1404841912000"
-                    }
-               ]
-    }
-
-
-Setting up STS access
----------------------
-Once we have setup our accounts we need to ensure that we create a trust relationship so that LemurInstanceProfile can assume the Lemur role.
-
-In the AWS console select the Lemur IAM role and select the Trust Relationships tab and click Edit Trust Relationship
-
-Below is an example policy:
-
-.. code-block:: python
-
-    {
-      "Version": "2008-10-17",
-      "Statement": [
-        {
-          "Sid": "",
-          "Effect": "Allow",
-          "Principal": {
-            "AWS": [
-              "arn:aws:iam::<awsaccountnumber>:role/LemurInstanceProfile",
-            ]
-          },
-          "Action": "sts:AssumeRole"
-        }
-      ]
-    }
-
-
-Adding N+1 accounts
--------------------
-
-To add another account we go to the new account and create a new Lemur IAM role with the same policy as above.
-
-Then we would go to the account that Lemur is running is and edit the trust relationship policy.
-
-An example policy:
-
-.. code-block:: python
-
-    {
-      "Version": "2008-10-17",
-      "Statement": [
-        {
-          "Sid": "",
-          "Effect": "Allow",
-          "Principal": {
-            "AWS": [
-              "arn:aws:iam::<awsaccountnumber>:role/LemurInstanceProfile",
-              "arn:aws:iam::<awsaccountnumber1>:role/LemurInstanceProfile",
-            ]
-          },
-          "Action": "sts:AssumeRole"
-        }
-      ]
-    }
-
-Setting up SES
---------------
-
-Lemur has built in support for sending it's certificate notifications via Amazon's simple email service (SES). To force
-Lemur to use SES ensure you are the running as the IAM role defined above and that you have followed the steps outlined
-in Amazon's documentation `Setting up Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/setting-up-ses.html>`_
-
-The configuration::
-
-    LEMUR_MAIL = 'lemur.example.com'
-
-Will be the sender of all notifications, so ensure that it is verified with AWS.
-
-SES if the default notification gateway and will be used unless SMTP settings are configured in the application configuration
-settings.
-
-Upgrading Lemur
-===============
-
-Lemur provides an easy way to upgrade between versions. Simply download the newest
-version of Lemur from pypi and then apply any schema changes with the following command.
-
-.. code-block:: bash
-
-    $ lemur db upgrade
-
-.. note:: Internally, this uses `Alembic <https://alembic.readthedocs.org/en/latest/>`_ to manage database migrations.
-
-.. _CommandLineInterface:
-
-Command Line Interface
-======================
-
-Lemur installs a command line script under the name ``lemur``. This will allow you to
-perform most required operations that are unachievable within the web UI.
-
-If you're using a non-standard configuration location, you'll need to prefix every command with
---config (excluding create_config, which is a special case). For example::
-
-    lemur --config=/etc/lemur.conf.py help
-
-For a list of commands, you can also use ``lemur help``, or ``lemur [command] --help``
-for help on a specific command.
-
-.. note:: The script is powered by a library called `Flask-Script <https://github.com/smurfix/flask-script>`_
-
-Builtin Commands
-----------------
-
-All commands default to `~/.lemur/lemur.conf.py` if a configuration is not specified.
-
-.. data:: create_config
-
-    Creates a default configuration file for Lemur.
-
-    Path defaults to ``~/.lemur/lemur.config.py``
-
-    ::
-
-        lemur create_config .
-
-    .. note::
-        This command is a special case and does not depend on the configuration file
-        being set.
-
-
-.. data:: init
-
-    Initializes the configuration file for Lemur.
-
-    ::
-
-        lemur -c /etc/lemur.conf.py init
-
-
-.. data:: start
-
-    Starts a Lemur service. You can also pass any flag that Gunicorn uses to specify the webserver configuration.
-
-    ::
-
-        lemur start -w 6 -b 127.0.0.1:8080
-
-
-.. data:: db upgrade
-
-    Performs any needed database migrations.
-
-    ::
-
-        lemur db upgrade
-
-
-.. data:: check_revoked
-
-    Traverses every certificate that Lemur is aware of and attempts to understand its validity.
-    It utilizes both OCSP and CRL. If Lemur is unable to come to a conclusion about a certificates
-    validity its status is marked 'unknown'
-
-
-.. data:: sync
-
-    Sync attempts to discover certificates in the environment that were not created by Lemur. If you wish to only sync
-    a few sources you can pass a comma delimited list of sources to sync
-
-    ::
-
-        lemur sync source1,source2
-
-
-    Additionally you can also list the available sources that Lemur can sync
-
-    ::
-
-        lemur sync -list
-
-
-Sub-commands
-------------
-
-Lemur includes several sub-commands for interacting with Lemur such as creating new users, creating new roles and even
-issuing certificates.
-
-The best way to discover these commands is by using the built in help pages
-
-    ::
-
-        lemur --help
-
-
-and to get help on sub-commands
-
-    ::
-
-        lemur certificates --help
-
-
-Identity and Access Management
-==============================
-
-Lemur uses a Role Based Access Control (RBAC) mechanism to control which users have access to which resources. When a
-user is first created in Lemur they can be assigned one or more roles. These roles are typically dynamically created
-depending on a external identity provider (Google, LDAP, etc.,) or are hardcoded within Lemur and associated with special
-meaning.
-
-Within Lemur there are three main permissions: AdminPermission, CreatorPermission, OwnerPermission. Sub-permissions such
-as ViewPrivateKeyPermission are compositions of these three main Permissions.
-
-Lets take a look at how these permissions are used:
-
-Each `Authority` has a set of roles associated with it. If a user is also associated with the same roles
-that the `Authority` is associated with, Lemur allows that user to user/view/update that `Authority`.
-
-This RBAC is also used when determining which users can access which certificate private key. Lemur's current permission
-structure is setup such that if the user is a `Creator` or `Owner` of a given certificate they are allow to view that
-private key. Owners can also be a role name, such that any user with the same role as owner will be allowed to view the
-private key information.
-
-These permissions are applied to the user upon login and refreshed on every request.
-
-.. seealso::
-    `Flask-Principal <https://pythonhosted.org/Flask-Principal>`_
-
-
-Upgrading Lemur
-===============
-
-To upgrade Lemur to the newest release you will need to ensure you have the lastest code and have run any needed
-database migrations.
-
-To get the latest code from github run
-
-    ::
-
-        cd <lemur-source-directory>
-        git pull -t <version>
-        python setup.py develop
-
-
-.. note::
-    It's important to grab the latest release by specifying the release tag. This tags denote stable versions of Lemur.
-    If you want to try the bleeding edge version of Lemur you can by using the master branch.
-
-
-After you have the latest version of the Lemur code base you must run any needed database migrations. To run migrations
-
-    ::
-
-        cd <lemur-source-directory>/lemur
-        lemur db upgrade
-
-
-This will ensure that any needed tables or columns are created or destroyed.

docs/conf.py

@@ -13,41 +13,53 @@
 # serve to show the default.
 import sys
 import os
+from unittest.mock import MagicMock
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath(".."))
+
+# Mock packages that cannot be installed on rtd
+on_rtd = os.environ.get("READTHEDOCS") == "True"
+if on_rtd:
+
+    class Mock(MagicMock):
+        @classmethod
+        def __getattr__(cls, name):
+            return MagicMock()
+
+    MOCK_MODULES = ["ldap"]
+    sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
+
+autodoc_mock_imports = ["python-ldap", "acme", "certsrv", "dnspython3", "dyn", "factory-boy", "flask_replicated",
+                        "josepy", "logmatic", "pem"]
 
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinxcontrib.autohttp.flask',
-    'sphinx.ext.todo',
-]
+extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autohttp.flask", "sphinx.ext.todo"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'lemur'
-copyright = u'2015, Netflix Inc.'
+project = u"lemur"
+copyright = u"2018, Netflix Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -56,187 +68,186 @@ copyright = u'2015, Netflix Inc.'
 base_dir = os.path.join(os.path.dirname(__file__), os.pardir)
 about = {}
 with open(os.path.join(base_dir, "lemur", "__about__.py")) as f:
-    exec(f.read(), about)
+    exec(f.read(), about)  # nosec
 
 version = release = about["__version__"]
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ["_build"]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# keep_warnings = False
 
 
 # -- Options for HTML output ----------------------------------------------
 
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-html_theme = 'default'
+# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
+on_rtd = os.environ.get("READTHEDOCS", None) == "True"
+
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+
+    html_theme = "sphinx_rtd_theme"
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'lemurdoc'
+htmlhelp_basename = "lemurdoc"
 
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'lemur.tex', u'Lemur Documentation',
-   u'Kevin Glisson', 'manual'),
+    ("index", "lemur.tex", u"Lemur Documentation", u"Netflix Security", "manual")
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'Lemur', u'Lemur Documentation',
-     [u'Kevin Glisson'], 1)
-]
+man_pages = [("index", "Lemur", u"Lemur Documentation", [u"Netflix Security"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -245,19 +256,25 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Lemur', u'Lemur Documentation',
-   u'Kevin Glisson', 'Lemur', 'SSL Certificate Management',
-   'Miscellaneous'),
+    (
+        "index",
+        "Lemur",
+        u"Lemur Documentation",
+        u"Netflix Security",
+        "Lemur",
+        "SSL Certificate Management",
+        "Miscellaneous",
+    )
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False

docs/developer/index.rst

@@ -22,12 +22,18 @@ Once you've got all that, the rest is simple:
     # If you have a fork, you'll want to clone it instead
     git clone git://github.com/netflix/lemur.git
 
-    # Create a python virtualenv
-    mkvirtualenv lemur
+    # Create and activate python virtualenv from within the lemur repo
+    python3 -m venv env
+    . env/bin/activate
+
+    # Install doc requirements
 
-    # Make the magic happen
     make dev-docs
 
+    # Make the docs
+    cd docs
+    make html
+
 Running ``make dev-docs`` will install the basic requirements to get Sphinx running.
 
 
@@ -37,18 +43,35 @@ Building Documentation
 Inside the ``docs`` directory, you can run ``make`` to build the documentation.
 See ``make help`` for available options and the `Sphinx Documentation <http://sphinx-doc.org/contents.html>`_ for more information.
 
+Adding New Modules to Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When a new module is added, it will need to be added to the documentation.
+Ideally, we might rely on `sphinx-apidoc <https://www.sphinx-doc.org/en/master/man/sphinx-apidoc.html>`_ to autogenerate our documentation.
+Unfortunately, this causes some build problems.
+Instead, you'll need to add new modules by hand.
 
 Developing Against HEAD
 -----------------------
 
 We try to make it easy to get up and running in a development environment using a git checkout
-of Lemur. You'll want to make sure you have a few things on your local system first:
+of Lemur. There are two ways to run Lemur locally: directly on your development machine, or
+in a Docker container.
+
+**Running in a Docker container**
+
+Look at the `lemur-docker <https://github.com/Netflix/lemur-docker>`_ project.
+Usage instructions are self-contained in the README for that project.
+
+**Running directly on your development machine**
+
+You'll want to make sure you have a few things on your local system first:
 
 * python-dev (if you're on OS X, you already have this)
 * pip
 * virtualenv (ideally virtualenvwrapper)
 * node.js (for npm and building css/javascript)
-* (Optional) Potgresql
+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
 
 Once you've got all that, the rest is simple:
 
@@ -58,7 +81,7 @@ Once you've got all that, the rest is simple:
     git clone git://github.com/lemur/lemur.git
 
     # Create a python virtualenv
-    mkvirtualenv lemur
+    python3 -m venv env
 
     # Make the magic happen
     make
@@ -77,6 +100,7 @@ Create a default Lemur configuration just as if this were a production instance:
 
 ::
 
+    lemur create_config
     lemur init
 
 You'll likely want to make some changes to the default configuration (we recommend developing against Postgres, for example). Once done, migrate your database using the following command:
@@ -86,7 +110,15 @@ You'll likely want to make some changes to the default configuration (we recomme
 	lemur upgrade
 
 
-.. note:: The ``upgrade`` shortcut is simply a shorcut to Alembic's upgrade command.
+.. note:: The ``upgrade`` shortcut is simply a shortcut to Alembic's upgrade command.
+
+
+Running tests with Docker and docker-compose
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you just want to run tests in a Docker container, you can use Docker and docker-compose for running the tests with ``docker-compose run test`` directly in the ``lemur`` project.
+
+(For running the Lemur service in Docker, see `lemur-docker <https://github.com/Netflix/lemur-docker>`_.)
 
 
 Coding Standards
@@ -113,6 +145,12 @@ HTML:
   2 Spaces
 
 
+Git hooks
+~~~~~~~~~
+
+To help developers maintain the above standards, Lemur includes a configuration file for Yelp's `pre-commit <http://pre-commit.com/>`_. This is an optional dependency and is not required in order to contribute to Lemur.
+
+
 Running the Test Suite
 ----------------------
 
@@ -122,7 +160,7 @@ The test suite consists of multiple parts, testing both the Python and JavaScrip
 
     make test
 
-If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``test-js`` for the JavaScript tests.
+If you only need to run the Python tests, you can do so with ``make test-python``, as well as ``make test-js`` for the JavaScript tests.
 
 
 You'll notice that the test suite is structured based on where the code lives, and strongly encourages using the mock library to drive more accurate individual tests.
@@ -133,7 +171,7 @@ You'll notice that the test suite is structured based on where the code lives, a
 Static Media
 ------------
 
-Lemur uses a library that compiles it's static media assets (LESS and JS files) automatically. If you're developing using
+Lemur uses a library that compiles its static media assets (LESS and JS files) automatically. If you're developing using
 runserver you'll see changes happen not only in the original files, but also the minified or processed versions of the file.
 
 If you've made changes and need to compile them by hand for any reason, you can do so by running:
@@ -144,8 +182,19 @@ If you've made changes and need to compile them by hand for any reason, you can
 
 The minified and processed files should be committed alongside the unprocessed changes.
 
+It's also important to note that Lemur's frontend and API are not tied together. The API does not serve any of the static assets, we rely on nginx or some other file server to server all of the static assets.
+During development that means we need an additional server to serve those static files for the GUI.
+
+This is accomplished with a Gulp task:
+
+::
+
+    ./node_modules/.bin/gulp serve
+
+The gulp task compiles all the JS/CSS/HTML files and opens the Lemur welcome page in your default browsers. Additionally any changes to made to the JS/CSS/HTML with be reloaded in your browsers.
+
 Developing with Flask
-----------------------
+---------------------
 
 Because Lemur is just Flask, you can use all of the standard Flask functionality. The only difference is you'll be accessing commands that would normally go through manage.py using the ``lemur`` CLI helper instead.
 
@@ -164,7 +213,7 @@ Schema changes should always introduce the new schema in a commit, and then intr
 
 Removing columns and tables requires a slightly more painful flow, and should resemble the follow multi-commit flow:
 
-- Remove all references to the column or table (but dont remove the Model itself)
+- Remove all references to the column or table (but don't remove the Model itself)
 - Remove the model code
 - Remove the table or column
 
@@ -180,19 +229,116 @@ You can see a list of open pull requests (pending changes) by visiting https://g
 
 Pull requests should be against **master** and pass all TravisCI checks
 
-Plugins
-=======
+
+Writing a Plugin
+================
 
 .. toctree::
-    :maxdepth: 1
+    :maxdepth: 2
 
     plugins/index
 
+
+REST API
+========
+
+Lemur's front end is entirely API driven. Any action that you can accomplish via the UI can also be accomplished by the
+API. The following is documents and provides examples on how to make requests to the Lemur API.
+
+Authentication
+--------------
+
+.. automodule:: lemur.auth.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Destinations
+------------
+
+.. automodule:: lemur.destinations.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Notifications
+-------------
+
+.. automodule:: lemur.notifications.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Users
+-----
+
+.. automodule:: lemur.users.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Roles
+-----
+
+.. automodule:: lemur.roles.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Certificates
+------------
+
+.. automodule:: lemur.certificates.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Authorities
+-----------
+
+.. automodule:: lemur.authorities.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Domains
+-------
+
+.. automodule:: lemur.domains.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Endpoints
+---------
+
+.. automodule:: lemur.endpoints.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Logs
+----
+
+.. automodule:: lemur.logs.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Sources
+-------
+
+.. automodule:: lemur.sources.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
 Internals
 =========
 
 .. toctree::
-    :maxdepth: 1
+    :maxdepth: 2
 
     internals/lemur
-

docs/developer/internals/lemur.certificates.rst

@@ -1,15 +1,6 @@
 certificates Package
 ====================
 
-:mod:`exceptions` Module
-------------------------
-
-.. automodule:: lemur.certificates.exceptions
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`models` Module
 --------------------
 

docs/developer/internals/lemur.defaults.rst

@@ -0,0 +1,29 @@
+defaults Package
+================
+
+:mod:`defaults` Module
+----------------------------------------
+
+.. automodule:: lemur.defaults
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module	
+-----------------------------	
+
+.. automodule:: lemur.defaults.schemas	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module	
+---------------------------	
+
+.. automodule:: lemur.defaults.views	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.deployment.rst

@@ -0,0 +1,20 @@
+deployment Package
+===================
+
+:mod:`deployment` Module
+----------------------------------------
+
+.. automodule:: lemur.deployment
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module	
+------------------------------	
+
+.. automodule:: lemur.deployment.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.endpoints.rst

@@ -0,0 +1,56 @@
+endpoints Package
+===================
+
+:mod:`endpoints` Module
+----------------------------------------
+
+.. automodule:: lemur.endpoints
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module	
+--------------------------	
+
+.. automodule:: lemur.endpoints.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module	
+-----------------------------	
+
+.. automodule:: lemur.endpoints.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module	
+------------------------------	
+
+.. automodule:: lemur.endpoints.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module	
+------------------------------	
+
+.. automodule:: lemur.endpoints.service	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module	
+----------------------------	
+
+.. automodule:: lemur.endpoints.views	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.logs.rst

@@ -0,0 +1,47 @@
+logs Package
+===================
+
+:mod:`logs` Module
+--------------------
+
+.. automodule:: lemur.logs
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module
+------------------------------	
+
+.. automodule:: lemur.logs.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module
+------------------------------
+
+.. automodule:: lemur.logs.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.logs.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.logs.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_acme.rst

@@ -0,0 +1,83 @@
+lemur_acme package	
+=================================	
+
+:mod:`lemur_acme` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_acme
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`acme_handlers` Module
+-----------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.acme_handlers	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`challenge_types` Module
+-------------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.challenge_types	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cloudflare` Module
+-------------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.cloudflare	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`dyn` Module
+------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.dyn	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+---------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.plugin	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`powerdns` Module
+-----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.powerdns	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`route53` Module
+----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.route53	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`ultradns` Module
+-----------------------------------------	
+
+.. automodule:: lemur.plugins.lemur_acme.ultradns	
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_atlas.rst

@@ -0,0 +1,20 @@
+lemur_atlas package
+==================================
+
+:mod:`lemur_atlas` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_atlas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_atlas.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_cfssl.rst

@@ -0,0 +1,20 @@
+lemur_cfssl Package
+===================
+
+:mod:`lemur_cfssl` Package
+--------------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_cryptography.rst

@@ -0,0 +1,20 @@
+lemur_cryptography package
+==================================
+
+:mod:`lemur_cryptography` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_cryptography
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_cryptography.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_digicert.rst

@@ -0,0 +1,20 @@
+lemur_digicert package
+==================================
+
+:mod:`lemur_digicert` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_digicert
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_digicert.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_jks.rst

@@ -0,0 +1,20 @@
+lemur_jks package
+==================================
+
+:mod:`lemur_jks` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_jks
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_jks.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_kubernetes.rst

@@ -0,0 +1,20 @@
+lemur_kubernetes package
+==================================
+
+:mod:`lemur_kubernetes` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_kubernetes
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_kubernetes.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_openssl.rst

@@ -0,0 +1,20 @@
+lemur_openssl package
+==================================
+
+:mod:`lemur_openssl` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_openssl
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_openssl.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_slack.rst

@@ -0,0 +1,20 @@
+lemur_slack package
+==================================
+
+:mod:`lemur_slack` Module
+----------------------------------------
+
+.. automodule:: lemur.plugins.lemur_slack
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_slack.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_verisign.rst

@@ -10,15 +10,6 @@ lemur_verisign Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`constants` Module
------------------------
-
-.. automodule:: lemur.plugins.lemur_verisign.constants
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`plugin` Module
 --------------------
 

docs/developer/internals/lemur.plugins.rst

@@ -27,6 +27,6 @@ Subpackages
     lemur.plugins.base
     lemur.plugins.bases
     lemur.plugins.lemur_aws
-    lemur.plugins.lemur_cloudca
+    lemur.plugins.lemur_cfssl
     lemur.plugins.lemur_email
     lemur.plugins.lemur_verisign

docs/developer/internals/lemur.reporting.rst

@@ -0,0 +1,38 @@
+reporting Package
+===================
+
+:mod:`reporting` Module
+----------------------------------------
+
+.. automodule:: lemur.reporting
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module
+------------------------------	
+
+.. automodule:: lemur.reporting.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.reporting.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.reporting.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.rst

@@ -28,15 +28,6 @@ lemur Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`decorators` Module
-------------------------
-
-.. automodule:: lemur.decorators
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`exceptions` Module
 ------------------------
 
@@ -96,5 +87,19 @@ Subpackages
     lemur.notifications
     lemur.plugins
     lemur.roles
-    lemur.status
     lemur.users
+    lemur.sources
+    lemur.logs
+    lemur.reporting
+    lemur.tests
+    lemur.deployment
+    lemur.endpoints
+    lemur.defaults
+    lemur.plugins.lemur_acme
+    lemur.plugins.lemur_atlas
+    lemur.plugins.lemur_cryptography
+    lemur.plugins.lemur_digicert
+    lemur.plugins.lemur_jks
+    lemur.plugins.lemur_kubernetes
+    lemur.plugins.lemur_openssl
+    lemur.plugins.lemur_slack

docs/developer/internals/lemur.sources.rst

@@ -0,0 +1,56 @@
+sources Package
+===================
+
+:mod:`sources` Module
+----------------------
+
+.. automodule:: lemur.sources
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`cli` Module
+------------------------------	
+
+.. automodule:: lemur.sources.cli
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`models` Module
+------------------------------
+
+.. automodule:: lemur.sources.models
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`schemas` Module
+------------------------------
+
+.. automodule:: lemur.sources.schemas
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`service` Module
+------------------------------
+
+.. automodule:: lemur.sources.service
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`views` Module
+------------------------------
+
+.. automodule:: lemur.sources.views
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.status.rst

@@ -1,11 +0,0 @@
-status Package
-==============
-
-:mod:`views` Module
--------------------
-
-.. automodule:: lemur.status.views
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:

docs/developer/internals/lemur.tests.rst

@@ -0,0 +1,11 @@
+tests Package
+=============
+
+:mod:`tests` Module
+--------------------
+
+.. automodule:: lemur.tests
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/plugins/index.rst

@@ -1,6 +1,3 @@
-Writing a Plugin
-================
-
 Several interfaces exist for extending Lemur:
 
 * Issuer (lemur.plugins.base.issuer)
@@ -28,7 +25,7 @@ if you want to pull the version using pkg_resources (which is what we recommend)
     try:
         VERSION = __import__('pkg_resources') \
             .get_distribution(__name__).version
-    except Exception, e:
+    except Exception as e:
         VERSION = 'unknown'
 
 Inside of ``plugin.py``, you'll declare your Plugin class::
@@ -73,10 +70,18 @@ at multiple plugins within your package::
         },
     )
 
+Once your plugin files are in place and the ``/www/lemur/setup.py`` file has been modified, you can load your plugin into your instance by reinstalling lemur:
+::
+
+    (lemur)$cd /www/lemur
+    (lemur)$pip install -e .
+
 That's it! Users will be able to install your plugin via ``pip install <package name>``.
 
 .. SeeAlso:: For more information about python packages see `Python Packaging <https://packaging.python.org/en/latest/distributing.html>`_
 
+.. SeeAlso:: For an example of a plugin operation outside of Lemur's core, see `lemur-digicert <https://github.com/opendns/lemur-digicert>`_
+
 .. _PluginInterfaces:
 
 Plugin Interfaces
@@ -95,10 +100,16 @@ If you have a third party or internal service that creates authorities (EJBCA, e
 it can treat any issuer plugin as both a source of creating new certificates as well as new authorities.
 
 
-The `IssuerPlugin` exposes two functions::
+The `IssuerPlugin` exposes four functions functions::
 
-    def create_certificate(self, options):
+    def create_certificate(self, csr, issuer_options):
         # requests.get('a third party')
+    def revoke_certificate(self, certificate, reason):
+        # requests.put('a third party')
+    def get_ordered_certificate(self, order_id):
+        # requests.get('already existing certificate')
+    def canceled_ordered_certificate(self, pending_cert, **kwargs):
+        # requests.put('cancel an order that has yet to be issued')
 
 Lemur will pass a dictionary of all possible options for certificate creation. Including a valid CSR, and the raw options associated with the request.
 
@@ -134,15 +145,34 @@ The `IssuerPlugin` doesn't have any options like Destination, Source, and Notifi
 any fields you might need to submit a request to a third party. If there are additional options you need
 in your plugin feel free to open an issue, or look into adding additional options to issuers yourself.
 
+**Asynchronous Certificates**
+An issuer may take some time to actually issue a certificate for an order.  In this case, a `PendingCertificate` is returned, which holds information to recreate a `Certificate` object at a later time.  Then, `get_ordered_certificate()` should be run periodically via `python manage.py pending_certs fetch -i all` to attempt to retrieve an ordered certificate::
+
+    def get_ordered_ceriticate(self, order_id):
+        # order_id is the external id of the order, not the external_id of the certificate
+        # retrieve an order, and check if there is an issued certificate attached to it
+
+`cancel_ordered_certificate()` should be implemented to allow an ordered certificate to be canceled before it is issued::
+
+        def cancel_ordered_certificate(self, pending_cert, **kwargs):
+            # pending_cert should contain the necessary information to match an order
+            # kwargs can be given to provide information to the issuer for canceling
+
 Destination
 -----------
 
 Destination plugins allow you to propagate certificates managed by Lemur to additional third parties. This provides flexibility when
 different orchestration systems have their own way of manage certificates or there is an existing system you wish to integrate with Lemur.
 
+By default destination plugins have a private key requirement. If your plugin does not require a certificates private key mark `requires_key = False`
+in the plugins base class like so::
+
+    class MyDestinationPlugin(DestinationPlugin):
+        requires_key = False
+
 The DestinationPlugin requires only one function to be implemented::
 
-    def upload(self, cert, private_key, cert_chain, options, **kwargs):
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
         # request.post('a third party')
 
 Additionally the DestinationPlugin allows the plugin author to add additional options
@@ -151,25 +181,25 @@ that can be used to help define sub-destinations.
 For example, if we look at the aws-destination plugin we can see that it defines an `accountNumber` option::
 
     options = [
-      {
-          'name': 'accountNumber',
-          'type': 'int',
-          'required': True,
-          'validation': '/^[0-9]{12,12}$/',
-          'helpMessage': 'Must be a valid AWS account number!',
-      }
+        {
+            'name': 'accountNumber',
+            'type': 'int',
+            'required': True,
+            'validation': '/^[0-9]{12,12}$/',
+            'helpMessage': 'Must be a valid AWS account number!',
+        }
     ]
 
 By defining an `accountNumber` we can make this plugin handle many N number of AWS accounts instead of just one.
 
 The schema for defining plugin options are pretty straightforward:
 
-  - **Name**: name of the variable you wish to present the user, snake case (snakeCase) is preferrred as Lemur
+  - **Name**: name of the variable you wish to present the user, snake case (snakeCase) is preferred as Lemur
     will parse these and create pretty variable titles
   - **Type** there are currently four supported variable types
       - **Int** creates an html integer box for the user to enter integers into
       - **Str** creates a html text input box
-      - **Boolean** creates a checkbox for the user to signify truithyness
+      - **Boolean** creates a checkbox for the user to signify truthiness
       - **Select** creates a select box that gives the user a list of options
           - When used a `available` key must be provided with a list of selectable options
   - **Required** determines if this option is required, this **must be a boolean value**
@@ -185,21 +215,25 @@ Notification
 ------------
 
 Lemur includes the ability to create Email notifications by **default**. These notifications
-currently come in the form of expiration noticies. Lemur periodically checks certifications expiration dates and
+currently come in the form of expiration and rotation notices for all certificates, expiration notices for CA certificates,
+and ACME certificate creation failure notices. Lemur periodically checks certificate expiration dates and
 determines if a given certificate is eligible for notification. There are currently only two parameters used to
 determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
 of days the current date (UTC) is from that expiration date.
 
-There are currently two objects that available for notification plugins the first is `NotficationPlugin`. This is the base object for
-any notification within Lemur. Currently the only support notification type is an certificate expiration notification. If you
+Certificate expiration notifications can also be configured for Slack or AWS SNS. Other notifications are not configurable.
+Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email.
+
+There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for
+any notification within Lemur. Currently the only supported notification type is a certificate expiration notification. If you
 are trying to create a new notification type (audit, failed logins, etc.) this would be the object to base your plugin on.
 You would also then need to build additional code to trigger the new notification type.
 
-The second is `ExpirationNotificationPlugin`, this object inherits from `NotificationPlugin` object.
-You will most likely want to base your plugin on, if you want to add new channels for expiration notices (Slack, Hipcat, Jira, etc.). It adds default options that are required by
-by all expiration notifications (interval, unit). This interface expects for the child to define the following function::
+The second is `ExpirationNotificationPlugin`, which inherits from the `NotificationPlugin` object.
+You will most likely want to base your plugin on this object if you want to add new channels for expiration notices (HipChat, Jira, etc.). It adds default options that are required by
+all expiration notifications (interval, unit). This interface expects for the child to define the following function::
 
-    def send(self):
+    def send(self, notification_type, message, targets, options, **kwargs):
         #  request.post("some alerting infrastructure")
 
 
@@ -207,27 +241,59 @@ Source
 ------
 
 When building Lemur we realized that although it would be nice if every certificate went through Lemur to get issued, but this is not
-always be the case. Often times there are third parties that will issue certificates on your behalf and these can get deployed
+always be the case. Oftentimes there are third parties that will issue certificates on your behalf and these can get deployed
 to infrastructure without any interaction with Lemur. In an attempt to combat this and try to track every certificate, Lemur has a notion of
 certificate **Sources**. Lemur will contact the source at periodic intervals and attempt to **sync** against the source. This means downloading or discovering any
-certificate Lemur does not know about and adding the certificate to it's inventory to be tracked and alerted on.
+certificate Lemur does not know about and adding the certificate to its inventory to be tracked and alerted on.
 
 The `SourcePlugin` object has one default option of `pollRate`. This controls the number of seconds which to get new certificates.
 
- .. warning::
-    Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. A lock file is generated to guarantee that 
+.. warning::
+    Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. A lock file is generated to guarantee that
     only one sync is running at a time. It also means that the minimum resolution of a source plugin poll rate is effectively 15min. You can always specify a faster cron
     job if you need a higher resolution sync job.
 
 
 The `SourcePlugin` object requires implementation of one function::
 
-      def get_certificates(self, **kwargs):
+      def get_certificates(self, options, **kwargs):
           #  request.get("some source of certificates")
 
 
-.. Note::
-    Often times to facilitate code re-use it makes sense put source and destination plugins into one package.
+.. note::
+    Oftentimes to facilitate code re-use it makes sense put source and destination plugins into one package.
+
+
+Export
+------
+
+Formats, formats and more formats. That's the current PKI landscape. See the always relevant `xkcd <https://xkcd.com/927/>`_.
+Thankfully Lemur supports the ability to output your certificates into whatever format you want. This integration comes by the way
+of Export plugins. Support is still new and evolving, the goal of these plugins is to return raw data in a new format that
+can then be used by any number of applications. Included in Lemur is the `JavaExportPlugin` which currently supports generating
+a Java Key Store (JKS) file for use in Java based applications.
+
+
+The `ExportPlugin` object requires the implementation of one function::
+
+    def export(self, body, chain, key, options, **kwargs):
+        # sys.call('openssl hokuspocus')
+        # return "extension", passphrase, raw
+
+
+.. note::
+    Support of various formats sometimes relies on external tools system calls. Always be mindful of sanitizing any input to these calls.
+
+
+Custom TLS Provider
+-------------------
+
+Managing TLS at the enterprise scale could be hard and often organizations offer custom wrapper implementations. It could
+be ideal to use those while making calls to internal services. The `TLSPlugin` would help to achieve this. It requires the
+implementation of one function which creates a TLS session::
+
+     def session(self, server_application):
+        # return active session
 
 
 Testing
@@ -246,9 +312,9 @@ Augment your setup.py to ensure at least the following:
 
    setup(
        # ...
-      install_requires=[
+       install_requires=[
           'lemur',
-      ]
+       ]
    )
 
 
@@ -259,11 +325,7 @@ The ``conftest.py`` file is our main entry-point for py.test. We need to configu
 
 .. code-block:: python
 
-   from __future__ import absolute_import
-
-   pytest_plugins = [
-       'lemur.utils.pytest'
-   ]
+   from lemur.tests.conftest import *  # noqa
 
 
 Test Cases
@@ -273,14 +335,18 @@ You can now inherit from Lemur's core test classes. These are Django-based and e
 
 .. code-block:: python
 
-   # test_myextension.py
-   from __future__ import absolute_import
+    import pytest
+    from lemur.tests.vectors import INTERNAL_CERTIFICATE_A_STR, INTERNAL_PRIVATE_KEY_A_STR
 
-   from lemur.testutils import TestCase
+    def test_export_keystore(app):
+        from lemur.plugins.base import plugins
+        p = plugins.get('java-keystore-jks')
+        options = [{'name': 'passphrase', 'value': 'test1234'}]
+        with pytest.raises(Exception):
+            p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
 
-   class MyExtensionTest(TestCase):
-       def test_simple(self):
-          assert 1 != 2
+        raw = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
+        assert raw != b""
 
 
 Running Tests
@@ -292,13 +358,14 @@ Running tests follows the py.test standard. As long as your test files and metho
 
     $ py.test -v
     ============================== test session starts ==============================
-    platform darwin -- Python 2.7.9 -- py-1.4.26 -- pytest-2.6.4/python2.7
-    plugins: django
-    collected 1 items
+    platform darwin -- Python 2.7.10, pytest-2.8.5, py-1.4.30, pluggy-0.3.1
+    cachedir: .cache
+    plugins: flask-0.10.0
+    collected 346 items
 
-    tests/test_myextension.py::MyExtensionTest::test_simple PASSED
+    lemur/plugins/lemur_acme/tests/test_acme.py::test_get_certificates PASSED
 
     =========================== 1 passed in 0.35 seconds ============================
 
 
-.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above. View the source: # TODO
+.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above.

docs/developer/rest.rst

@@ -1,66 +0,0 @@
-Lemur's front end is entirely API driven. Any action that you can accomplish via the UI can also be accomplished by the
-UI. The following is documents and provides examples on how to make requests to the Lemur API.
-
-Authentication
---------------
-
-.. automodule:: lemur.auth.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Destinations
-------------
-
-.. automodule:: lemur.destinations.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Notifications
--------------
-
-.. automodule:: lemur.notifications.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Users
------
-
-.. automodule:: lemur.users.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Roles
------
-
-.. automodule:: lemur.roles.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Certificates
-------------
-
-.. automodule:: lemur.certificates.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Authorities
------------
-
-.. automodule:: lemur.authorities.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Domains
--------
-
-.. automodule:: lemur.domains.views
-    :members:
-    :undoc-members:
-    :show-inheritance:

docs/doing-a-release.rst

@@ -1,32 +1,65 @@
 Doing a release
 ===============
 
-Doing a release of ``lemur`` requires a few steps.
+Doing a release of ``lemur`` is now mostly automated and consists of the following steps:
 
-Bumping the version number
---------------------------
+* Raise a PR to add the release date and summary in the :doc:`/changelog`.
+* Merge above PR and create a new `Github release <https://github.com/Netflix/lemur/releaes>`_: set the tag starting with v, e.g., v0.9.0
+
+The `publish workflow <https://github.com/Netflix/lemur/actions/workflows/lemur-publish-release-pypi.yml>`_ uses the git
+tag to set the release version.
+
+The following describes the manual release steps, which is now obsolete:
+
+Manually Bumping the version number
+-----------------------------------
 
 The next step in doing a release is bumping the version number in the
 software.
 
 * Update the version number in ``lemur/__about__.py``.
 * Set the release date in the :doc:`/changelog`.
-* Do a commit indicating this.
-* Send a pull request with this.
+* Do a commit indicating this, and raise a pull request with this.
 * Wait for it to be merged.
 
-Performing the release
-----------------------
+Manually Performing the release
+-------------------------------
 
 The commit that merged the version number bump is now the official release
-commit for this release. You will need to have ``gpg`` installed and a ``gpg``
-key in order to do a release. Once this has happened:
+commit for this release. You need an `API key <https://pypi.org/manage/account/#api-tokens>`_,
+which requires permissions to maintain the Lemur `project  <https://pypi.org/project/lemur/>`_.
 
-* Run ``invoke release {version}``.
+For creating the release, follow these steps (more details `here <https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives>`_)
 
-The release should now be available on PyPI and a tag should be available in
+* Make sure you have the latest versions of setuptools and wheel installed:
+
+    ``python3 -m pip install --user --upgrade setuptools wheel``
+
+* Now run this command from the same directory where setup.py is located:
+
+    ``python3 setup.py sdist bdist_wheel``
+
+* Once completed it should generate two files in the dist directory:
+
+.. code-block:: pycon
+
+    $ ls dist/
+    lemur-0.8.0-py2.py3-none-any.whl	lemur-0.8.0.tar.gz
+
+
+* In this step, the distribution will be uploaded. You’ll need to install Twine:
+
+    ``python3 -m pip install --user --upgrade twine``
+
+* Once installed, run Twine to upload all of the archives under dist. Once installed, run Twine to upload all of the archives under dist:
+
+    ``python3 -m twine upload --repository pypi dist/*``
+
+The release should now be available on `PyPI Lemur <https://pypi.org/project/lemur/>`_ and a tag should be available in
 the repository.
 
+Make sure to also make a github `release <https://github.com/Netflix/lemur/releases>`_ which will pick up the latest version.
+
 Verifying the release
 ---------------------
 

docs/faq.rst

@@ -6,7 +6,7 @@ Common Problems
 
 In my startup logs I see *'Aborting... Lemur cannot locate db encryption key, is LEMUR_ENCRYPTION_KEYS set?'*
   You likely have not correctly configured **LEMUR_ENCRYPTION_KEYS**. See
-  :doc:`administration/index` for more information.
+  :doc:`administration` for more information.
 
 
 I am seeing Lemur's javascript load in my browser but not the CSS.

docs/guide/index.rst

@@ -18,7 +18,7 @@ that Lemur can then manage.
 
 .. figure:: create_authority.png
 
-    Enter a authority name and short description about the authority. Enter an owner,
+    Enter an authority name and short description about the authority. Enter an owner,
     and certificate common name. Depending on the authority and the authority/issuer plugin
     these values may or may not be used.
 
@@ -37,18 +37,20 @@ Create a New Certificate
 
 .. figure:: create_certificate.png
 
-    Enter an owner, short description and the authority you wish to issue this certificate.
-    Enter a common name into the certificate, if no validity range is selected two years is
-    the default.
+    Enter an owner, common name, short description and certificate authority you wish to issue this certificate.
+    Depending upon the selected CA, the UI displays default validity of the certificate. You can select different
+    validity by entering a custom date, if supported by the CA.
+
+    You can also add `Subject Alternate Names` or SAN for certificates that need to include more than one domains,
+    The first domain is the Common Name and all other domains are added here as DNSName entries.
 
     You can add notification options and upload the created certificate to a destination, both
     of these are editable features and can be changed after the certificate has been created.
 
 .. figure:: certificate_extensions.png
 
-    These options are typically for advanced users, the one exception is the `Subject Alternate Names` or SAN.
-    For certificates that need to include more than one domains, the first domain is the Common Name and all
-    other domains are added here as DNSName entries.
+    These options are typically for advanced users. Lemur creates ECC based certificate (ECCPRIME256V1 in particular)
+    by default. One can change the key type using the dropdown option listed here.
 
 
 Import an Existing Certificate
@@ -56,13 +58,14 @@ Import an Existing Certificate
 
 .. figure:: upload_certificate.png
 
-    Enter a owner, short description and public certificate. If there are intermediates and private keys
+    Enter an owner, short description and public certificate. If there are intermediates and private keys
     Lemur will track them just as it does if the certificate were created through Lemur. Lemur generates
-    a certificate name but you can override that by passing a value to the `Custom Name` field.
+    a certificate name but you can override that by passing a value to the `Custom Certificate Name` field.
 
     You can add notification options and upload the created certificate to a destination, both
     of these are editable features and can be changed after the certificate has been created.
 
+.. _CreateANewUser:
 
 Create a New User
 ~~~~~~~~~~~~~~~~~

docs/index.rst

@@ -27,8 +27,7 @@ Administration
 .. toctree::
     :maxdepth: 2
 
-    administration/index
-    plugins/index
+    administration
 
 Developers
 ----------
@@ -38,17 +37,24 @@ Developers
 
     developer/index
 
-
-REST API
+Security
 --------
 
 .. toctree::
     :maxdepth: 2
 
-    developer/rest
+    security
+
+Doing a Release
+---------------
+
+.. toctree::
+    :maxdepth: 1
+
+    doing-a-release
 
 FAQ
-----
+---
 
 .. toctree::
     :maxdepth: 1

docs/license/index.rst

@@ -17,4 +17,5 @@ A list of additional contributors can be seen on `GitHub <https://github.com/net
 Lemur License
 -------------
 
-.. include:: ../../LICENSE
+.. include:: ../../LICENSE
+    :literal:

docs/plugins/index.rst

@@ -1,20 +0,0 @@
-Plugins
-=======
-
-There are several interfaces currently available to extend Lemur. These are a work in
-progress and the API is not frozen.
-
-Bundled Plugins
----------------
-
-Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec and CloudCA services.
-
-3rd Party Extensions
---------------------
-
-The following extensions are available and maintained by members of the Lemur community:
-
-Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
-get it added.
-
-Want to create your own extension? See :doc:`../developer/plugins/index` to get started.

docs/production/index.rst

@@ -37,20 +37,22 @@ Entropy
 -------
 
 Lemur generates private keys for the certificates it creates. This means that it is vitally important that Lemur has enough entropy to draw from. To generate private keys Lemur uses the python library `Cryptography <https://cryptography.io>`_. In turn Cryptography uses OpenSSL bindings to generate
-keys just like you might from the OpenSSL command line. OpenSSL draws it's initial entropy from system during startup and uses PRNGs to generate a stream of random bytes (as output by /dev/urandom) whenever it needs to do a cryptographic operation.
+keys just like you might from the OpenSSL command line. OpenSSL draws its initial entropy from system during startup and uses PRNGs to generate a stream of random bytes (as output by /dev/urandom) whenever it needs to do a cryptographic operation.
 
 What does all this mean? Well in order for the keys
 that Lemur generates to be strong, the system needs to interact with the outside world. This is typically accomplished through the systems hardware (thermal, sound, video user-input, etc.) since the physical world is much more "random" than the computer world.
 
-If you are running Lemur on its own server with its own hardware "bare metal" then the entropy of the system is typically "good enough" for generating keys. If however you are using an VM on shared hardware there is a potential that your initial seed data (data that was initially
-fed to the PRNG) is not very good. What's more VMs have been known to be unable to inject more entropy into the system once it has been started. This is because there is typically very little interaction with the server once it has been started.
+If you are running Lemur on its own server with its own hardware "bare metal" then the entropy of the system is typically "good enough" for generating keys. If however you are using a VM on shared hardware there is a potential that your initial seed data (data that was initially
+fed to the PRNG) is not very good. What's more, VMs have been known to be unable to inject more entropy into the system once it has been started. This is because there is typically very little interaction with the server once it has been started.
 
 The amount of effort you wish to expend ensuring that Lemur has good entropy to draw from is up to your specific risk tolerance and how Lemur is configured.
 
 If you wish to generate more entropy for your system we would suggest you take a look at the following resources:
 
-- `WES-entropy-client <https://github.com/WhitewoodCrypto/WES-entropy-client>`_
-- `haveaged <http://www.issihosts.com/haveged/>`_
+- `WES-entropy-client <https://github.com/Virginian/WES-entropy-client>`_
+- `haveged <http://www.issihosts.com/haveged/>`_
+
+The original *WES-entropy-client* repository by WhitewoodCrypto was removed, the link now points to a fork of it.
 
 For additional information about OpenSSL entropy issues:
 
@@ -72,7 +74,7 @@ Nginx is a very popular choice to serve a Python project:
 Nginx doesn't run any Python process, it only serves requests from outside to
 the Python server.
 
-Therefore there are two steps:
+Therefore, there are two steps:
 
 - Run the Python process.
 - Run Nginx.
@@ -110,7 +112,7 @@ You can make some adjustments to get a better user experience::
        error_log   /var/log/nginx/log/lemur.error.log;
 
        location /api {
-            proxy_pass  http://127.0.0.1:5000;
+            proxy_pass  http://127.0.0.1:8000;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
             proxy_redirect off;
             proxy_buffering off;
@@ -176,7 +178,7 @@ sensitive nature of Lemur and what it controls makes this essential. This is a s
        resolver <IP DNS resolver>;
 
        location /api {
-            proxy_pass  http://127.0.0.1:5000;
+            proxy_pass  http://127.0.0.1:8000;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
             proxy_redirect off;
             proxy_buffering off;
@@ -217,13 +219,30 @@ An example apache config::
         # HSTS (mod_headers is required) (15768000 seconds = 6 months)
         Header always set Strict-Transport-Security "max-age=15768000"
         ...
+
+     # Set the lemur DocumentRoot to static/dist
+     DocumentRoot /www/lemur/lemur/static/dist
+
+     # Uncomment to force http 1.0 connections to proxy
+     # SetEnv force-proxy-request-1.0 1
+
+     #Don't keep proxy connections alive
+     SetEnv proxy-nokeepalive 1
+
+     # Only need to do reverse proxy
+     ProxyRequests Off
+
+     # Proxy requests to the api to the lemur service (and sanitize redirects from it)
+     ProxyPass "/api" "http://127.0.0.1:8000/api"
+     ProxyPassReverse "/api" "http://127.0.0.1:8000/api"
+
     </VirtualHost>
 
 Also included in the configurations above are several best practices when it comes to deploying TLS. Things like enabling
 HSTS, disabling vulnerable ciphers are all good ideas when it comes to deploying Lemur into a production environment.
 
 .. note::
-    This is a rather incomplete apache config for running Lemur (needs mod_wsgi etc.,), if you have a working apache config please let us know!
+    This is a rather incomplete apache config for running Lemur (needs mod_wsgi etc.), if you have a working apache config please let us know!
 
 .. seealso::
     `Mozilla SSL Configuration Generator <https://mozilla.github.io/server-side-tls/ssl-config-generator/>`_
@@ -240,10 +259,10 @@ most of the time), but here is a quick overview on how to use it.
 Create a configuration file named supervisor.ini::
 
     [unix_http_server]
-    file=/tmp/supervisor.sock;
+    file=/tmp/supervisor.sock
 
     [supervisorctl]
-    serverurl=unix:///tmp/supervisor.sock;
+    serverurl=unix:///tmp/supervisor.sock
 
     [rpcinterface:supervisor]
     supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
@@ -257,13 +276,12 @@ Create a configuration file named supervisor.ini::
     nodaemon=false
     minfds=1024
     minprocs=200
-    user=lemur
 
     [program:lemur]
     command=python /path/to/lemur/manage.py manage.py start
 
     directory=/path/to/lemur/
-    environment=PYTHONPATH='/path/to/lemur/'
+    environment=PYTHONPATH='/path/to/lemur/',LEMUR_CONF='/home/lemur/.lemur/lemur.conf.py'
     user=lemur
     autostart=true
     autorestart=true
@@ -296,3 +314,404 @@ Then you can manage the process by running::
 It will start a shell from which you can start/stop/restart the service.
 
 You can read all errors that might occur from /tmp/lemur.log.
+
+.. _PeriodicTasks:
+
+Periodic Tasks
+==============
+
+Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
+celery tasks or cron jobs that run these commands.
+
+The following commands that could/should be run on a periodic basis:
+
+- `notify expirations` `notify authority_expirations`, and `notify security_expiration_summary` (see :ref:`NotificationOptions` for configuration info)
+- `check_revoked`
+- `sync`
+
+If you are using LetsEncrypt, you must also run the following:
+
+- `fetch_all_pending_acme_certs`
+- `remove_old_acme_certs`
+
+How often you run these commands is largely up to the user. `notify` should be run once a day (more often will result in
+duplicate notifications). `check_revoked` is typically run at least once a day.
+`sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
+`remove_old_acme_certs` can be ran more rarely, such as once every week.
+
+Example cron entries::
+
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify security_expiration_summary
+    */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
+
+
+Example Celery configuration (To be placed in your configuration file)::
+
+    CELERYBEAT_SCHEDULE = {
+        'fetch_all_pending_acme_certs': {
+            'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(minute="*"),
+        },
+        'remove_old_acme_certs': {
+            'task': 'lemur.common.celery.remove_old_acme_certs',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=7, minute=30, day_of_week=1),
+        },
+        'clean_all_sources': {
+            'task': 'lemur.common.celery.clean_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=1, minute=0, day_of_week=1),
+        },
+        'sync_all_sources': {
+            'task': 'lemur.common.celery.sync_all_sources',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*/3", minute=5),
+        },
+        'sync_source_destination': {
+            'task': 'lemur.common.celery.sync_source_destination',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour="*"),
+        },
+        'notify_expirations': {
+            'task': 'lemur.common.celery.notify_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        },
+        'notify_authority_expirations': {
+            'task': 'lemur.common.celery.notify_authority_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        },
+        'send_security_expiration_summary': {
+            'task': 'lemur.common.celery.send_security_expiration_summary',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        }
+    }
+
+To enable celery support, you must also have configuration values that tell Celery which broker and backend to use.
+Here are the Celery configuration variables that should be set::
+
+    CELERY_RESULT_BACKEND = 'redis://your_redis_url:6379'
+    CELERY_BROKER_URL = 'redis://your_redis_url:6379/0'
+    CELERY_IMPORTS = ('lemur.common.celery')
+    CELERY_TIMEZONE = 'UTC'
+
+    REDIS_HOST="your_redis_url"
+    REDIS_PORT=6379
+    REDIS_DB=0
+
+Out of the box, every Redis instance supports 16 databases. The default database (`REDIS_DB`) is  set to 0, however, you can use any of the databases from 0-15. Via `redis.conf` more databases can be supported.
+In the `redis://` url, the database number can be added with a slash after the port. (defaults to 0, if omitted)
+
+Do not forget to import crontab module in your configuration file::
+
+    from celery.task.schedules import crontab
+
+You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks.
+The scheduler can be started with::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery beat
+
+And the worker can be started with desired options such as the following::
+
+    LEMUR_CONF='/location/to/conf.py' /location/to/lemur/bin/celery -A lemur.common.celery worker --concurrency 10 -E -n lemurworker1@%%h
+
+supervisor or systemd configurations should be created for these in production environments as appropriate.
+
+Add support for LetsEncrypt/ACME
+================================
+
+LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid
+for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV).
+LetsEncrypt requires that we prove ownership of a domain before we're able to issue a certificate for that domain, each
+time we want a certificate.
+
+The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation
+through the creation of DNS TXT records as well as HTTP validation, reusing the destination concept.
+
+ACME DNS Challenge
+------------------
+
+In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that
+token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must
+prove ownership of all of these domains through this method. The token is typically written to a TXT record at
+-acme_challenge.domain.com. Once we create the appropriate TXT record(s), Lemur will try to validate propagation
+before requesting that LetsEncrypt finalize the certificate request and send us the certificate.
+
+.. figure:: letsencrypt_flow.png
+
+To start issuing certificates through LetsEncrypt, you must enable Celery support within Lemur first. After doing so,
+you need to create a LetsEncrypt authority. To do this, visit
+Authorities -> Create. Set the applicable attributes and click "More Options".
+
+.. figure:: letsencrypt_authority_1.png
+
+You will need to set "Certificate" to LetsEncrypt's active chain of trust for the authority you want to use. To find
+the active chain of trust at the time of writing, please visit `LetsEncrypt
+<https://letsencrypt.org/certificates/>`_.
+
+Under Acme_url, enter in the appropriate endpoint URL. Lemur supports LetsEncrypt's V2 API, and we recommend you to use
+this. At the time of writing, the staging and production URLs for LetsEncrypt V2 are
+https://acme-staging-v02.api.letsencrypt.org/directory and https://acme-v02.api.letsencrypt.org/directory.
+
+.. figure:: letsencrypt_authority_2.png
+
+After creating the authorities, we will need to create a DNS provider. Visit `Admin` -> `DNS Providers` and click
+`Create`. Lemur comes with a few provider plugins built in, with different options. Create a DNS provider with the
+appropriate choices.
+
+.. figure:: create_dns_provider.png
+
+By default, users will need to select the DNS provider that is authoritative over their domain in order for the
+LetsEncrypt flow to function. However, Lemur will attempt to automatically determine the appropriate provider if
+possible. To enable this functionality, periodically (or through Cron/Celery) run `lemur dns_providers get_all_zones`.
+This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
+Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.
+
+ACME HTTP Challenge
+-------------------
+
+The flow for requesting a certificate using the HTTP challenge is not that different from the one described for the DNS
+challenge. The only difference is, that instead of creating a DNS TXT record, a file is uploaded to a Webserver which
+serves the file at `http://<domain>/.well-known/acme-challenge/<token>`
+
+Currently the HTTP challenge also works without Celery, since it's done while creating the certificate, and doesn't
+rely on celery to create the DNS record. This will change when we implement mix & match of acme challenge types.
+
+To create a HTTP compatible Authority, you first need to create a new destination that will be used to deploy the
+challenge token. Visit `Admin` -> `Destination` and click `Create`. The path you provide for the destination needs to
+be the exact path that is called when the ACME providers calls `http://<domain>/.well-known/acme-challenge/`. The
+token part will be added dynamically by the acme_upload.
+Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge.
+
+Afterwards you can create a new certificate authority as described in the DNS challenge, but need to choose
+`Acme HTTP-01` as the plugin type, and then the destination you created beforehand.
+
+LetsEncrypt: pinning to cross-signed ICA
+----------------------------------------
+
+Let's Encrypt has been using a `cross-signed <https://letsencrypt.org/certificates/>`_ intermediate CA by DST Root CA X3,
+which is included in many older devices' TrustStore.
+
+
+Let's Encrypt is `transitioning <https://letsencrypt.org/2019/04/15/transitioning-to-isrg-root.html>`_ to use
+the intermediate CA issued by their own root (ISRG X1) starting from September 29th 2020.
+This is in preparation of concluding the initial bootstrapping of their CA, by having it cross-signed by an older CA.
+
+
+Lemur can temporarily pin to the cross-signed intermediate CA (same public/private key pair as the ICA signed by ISRG X1).
+This will prolong support for incompatible devices.
+
+The following must be added to the config file to activate the pinning (the pinning will be removed by September 2021)::
+
+    # remove or update after Mar 17 16:40:46 2021 GMT
+    IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE = "17/03/21"
+    IDENTRUST_CROSS_SIGNED_LE_ICA = """
+    -----BEGIN CERTIFICATE-----
+    MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+    MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+    DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+    SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+    GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+    AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+    q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+    SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+    Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+    a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+    /PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+    AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+    CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+    bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+    c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+    VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+    ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+    MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+    Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+    AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+    uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+    wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+    X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+    PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+    KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+    -----END CERTIFICATE-----
+    """
+
+
+.. _AcmeAccountReuse:
+
+LetsEncrypt: Using a pre-existing ACME account
+-----------------------------------------------
+
+Let's Encrypt allows reusing an existing ACME account, to create and especially revoke certificates. The current
+implementation in the acme plugin, only allows for a single account for all ACME authorities, which might be an issue,
+when you try to use Let's Encrypt together with another certificate authority that uses the ACME protocol.
+
+To use an existing account, you need to configure the `ACME_PRIVATE_KEY` and `ACME_REGR` variables in the lemur
+configuration.
+
+`ACME_PRIVATE_KEY` needs to be in the JWK format::
+
+    {
+        "kty": "RSA",
+        "n": "yr1qBwHizA7ME_iV32bY10ILp.....",
+        "e": "AQAB",
+        "d": "llBlYhil3I.....",
+        "p": "-5LW2Lewogo.........",
+        "q": "zk6dHqHfHksd.........",
+        "dp": "qfe9fFIu3mu.......",
+        "dq": "cXFO-loeOyU.......",
+        "qi": "AfK1sh0_8sLTb..........."
+    }
+
+
+Using `python-jwt` converting an existing private key in PEM format is quite easy::
+
+    import python_jwt as jwt, jwcrypto.jwk as jwk
+
+    priv_key = jwk.JWK.from_pem(b"""-----BEGIN RSA PRIVATE KEY-----
+    ...
+    -----END RSA PRIVATE KEY-----""")
+
+    print(priv_key.export())
+
+`ACME_REGR` needs to be a valid JSON with a `body` and a `uri` attribute, similar to this::
+
+    {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/<ACCOUNT_NUMBER>"}
+
+The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key.
+
+OpenSSH
+=======
+
+OpenSSH (also known as OpenBSD Secure Shell) is a suite of secure networking utilities based on the Secure Shell (SSH) protocol, which provides a secure channel over an unsecured network in a client–server architecture.
+
+Using a PKI with OpenSSH means you can sign a key for a user and it can log into any server that trust the CA.
+
+Using a CA avoids TOFU or synchronize a list of server public keys to `known_hosts` files.
+
+This is useful when you're managing large number of machines or for an immutable infrastructure.
+
+Add first OpenSSH authority
+---------------------------
+
+To start issuing OpenSSH, you need to create an OpenSSH authority. To do this, visit
+Authorities -> Create. Set the applicable attributes:
+
+- Name : OpenSSH
+- Common Name: example.net
+
+Then click "More Options" and change the plugin value to "OpenSSH".
+
+Just click to "Create" button to add this authority.
+
+.. note:: OpenSSH do not support sub CA feature.
+
+Add a server certificate
+-------------------------
+
+Now visit Certificates -> Create to add a server certificate. Set the applicable attributes:
+
+- Common Name: server.example.net
+
+Then click "More Options" and set the Certificate Template to "Server Certificate".
+
+This step is important, a certificat for a server and for a client is not exactly the same thing.
+In this case "Common Name" and all Subject Alternate Names with type DNSName will be added in the certificate.
+
+Finally click on "Create" button.
+
+Add a client certificate
+------------------------
+
+Now visit Certificates -> Create to add a client certificate. Set the applicable attributes:
+
+- Common Name: example.net
+
+Then click "More Options" and set the Certificate Template to "Client Certificate".
+
+In this case the name of the creator is used as principal (in this documentation we assume that this certificate is created by the user "lemur").
+
+Finally click on "Create" button.
+
+Configure OpenSSH server
+------------------------
+
+Connect to the server.example.net server to correctly configure the OpenSSH server with the CA created previously.
+
+First of all add the CA chain, private and public certificates:
+
+- Create file `/etc/ssh/ca.pub` and copy the "CHAIN" content of the *server certificate* (everything in one line).
+- Create file `/etc/ssh/ssh_host_key` and copy "PRIVATE KEY" content.
+- Create file `/etc/ssh/ssh_host_key.pub` and copy "PUBLIC CERTIFICATE" content (everything in one line).
+
+Set the appropriate right:
+
+.. code-block:: bash
+
+    chmod 600 /etc/ssh/ca.pub /etc/ssh/ssh_host_key
+    chmod 644 /etc/ssh/ssh_host_key.pub
+    chown root: /etc/ssh/ca.pub /etc/ssh/ssh_host_key /etc/ssh/ssh_host_key.pub
+
+Then change OpenSSH server configuration to use these files. Edit `/etc/ssh/sshd_config` and add::
+
+    TrustedUserCAKeys /etc/ssh/ca.pub
+    HostKey /etc/ssh/ssh_host_key
+    HostCertificate /etc/ssh/ssh_host_key.pub
+
+You can remove all other `HostKey` lines.
+
+Finally restart OpenSSH.
+
+Configure the OpenSSH client
+----------------------------
+
+Now you can configure the user's computer.
+
+First of all add private and public certificates:
+
+- Create file `~/.ssh/key` and copy "PRIVATE KEY" content.
+- Create file `~/.ssh/key.pub` and copy "PUBLIC CERTIFICATE" content of the *client certicate* (everything in one line).
+
+Set the appropriate right:
+
+.. code-block:: bash
+
+    chmod 600 ~/.ssh/key.pub ~/.ssh/key
+
+To avoid TOFU, edite the `~/.ssh/known_hosts` file and add a new line (all in one line):
+
+- @cert-authority \*example.net
+- the "CHAIN" content
+
+Now you can connect to server with (here 'lemur' is the principal name and must exists on the server):
+
+.. code-block:: bash
+
+    ssh lemur@server.example.net -i ~/.ssh/key
+
+With this configuration you don't have any line like::
+
+    Warning: Permanently added 'server.example.net,192.168.0.1' (RSA) to the list of known hosts.
+
+And you don't have to enter any password.

docs/quickstart/index.rst

@@ -1,9 +1,10 @@
 Quickstart
 **********
 
-This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.  This guide assumes a clean Ubuntu 14.04 instance, commands may differ based on the OS and configuration being used.
+This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.
+This guide assumes a clean Ubuntu 18.04/20.04 instance, commands may differ based on the OS and configuration being used.
 
-Pressed for time? See the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
+For a quicker alternative, see the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
 
 
 Dependencies
@@ -11,31 +12,36 @@ Dependencies
 
 Some basic prerequisites which you'll need in order to run Lemur:
 
-* A UNIX-based operating system (we test on Ubuntu, develop on OS X)
-* Python 2.7
-* PostgreSQL
+* A UNIX-based operating system (we test on Ubuntu, develop on macOS)
+* Python 3.7 or greater
+* PostgreSQL 9.4 or greater
 * Nginx
+* Node v10.x (LTS)
 
-.. note:: Lemur was built with in AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
+.. note:: Ubuntu 18.04 supports by default Python 3.6.x and Node v8.x
+.. note:: Lemur was built with AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
 
 
 Installing Build Dependencies
 -----------------------------
 
-If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
+If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies:
 
 .. code-block:: bash
 
-    $ sudo apt-get update
-    $ sudo apt-get install install nodejs-legacy python-pip python-dev libpq-dev build-essential libssl-dev libffi-dev nginx git supervisor npm postgresql
+    sudo apt-get update
+    sudo apt-get install nodejs npm python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor postgresql
 
 .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver.  npm is needed if you're installing Lemur from the source (e.g., from git).
 
+.. note:: Installing node from a package manager may create the nodejs bin at  /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
+    sudo ln -s /user/bin/nodejs /usr/bin/node
+
 Now, install Python ``virtualenv`` package:
 
 .. code-block:: bash
 
-    $ sudo pip install -U virtualenv
+    sudo pip install -U virtualenv
 
 
 Setting up an Environment
@@ -45,23 +51,28 @@ In this guide, Lemur will be installed in ``/www``, so you need to create that s
 
 .. code-block:: bash
 
-    $ sudo mkdir /www
-    $ cd /www
+    sudo mkdir /www
+    cd /www
 
 Clone Lemur inside the just created directory and give yourself write permission (we assume ``lemur`` is the user):
 
 .. code-block:: bash
 
-    $ sudo git clone https://github.com/Netflix/lemur
-    $ sudo chown -R lemur lemur/
+    sudo useradd lemur
+    sudo passwd lemur
+    sudo mkdir /home/lemur
+    sudo chown lemur:lemur /home/lemur
+    sudo git clone https://github.com/Netflix/lemur
+    sudo chown -R lemur lemur/
 
 Create the virtual environment, activate it and enter the Lemur's directory:
 
 .. code-block:: bash
 
-    $ virtualenv lemur
-    $ source /www/lemur/bin/activate
-    $ cd lemur
+    su lemur
+    virtualenv -p python3 lemur
+    source /www/lemur/bin/activate
+    cd lemur
 
 .. note:: Activating the environment adjusts your PATH, so that things like pip now install into the virtualenv by default.
 
@@ -73,17 +84,29 @@ Once your system is prepared, ensure that you are in the virtualenv:
 
 .. code-block:: bash
 
-  $ which python
+  which python
 
 And then run:
 
 .. code-block:: bash
 
-  $ make develop
+  make release
 
 .. note:: This command will install npm dependencies as well as compile static assets.
 
 
+You may also run with the urlContextPath variable set. If this is set it will add the desired context path for subsequent calls back to lemur. This will only edit the front end code for calls back to the server, you will have to make sure the server knows about these routes.
+::
+
+  Example:
+    urlContextPath=lemur
+    /api/1/auth/providers -> /lemur/api/1/auth/providers
+
+.. code-block:: bash
+
+  make release urlContextPath={desired context path}
+
+
 Creating a configuration
 ------------------------
 
@@ -93,11 +116,11 @@ Simply run:
 
 .. code-block:: bash
 
-  $ lemur create_config
+  lemur create_config
 
 .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command.
 
-You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment.
+You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment.
 
 
 Update your configuration
@@ -105,12 +128,27 @@ Update your configuration
 
 Once created, you will need to update the configuration file with information about your environment, such as which database to talk to, where keys are stored etc.
 
-.. note:: If you are unfamiliar with with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
-      ``postgresql://userame:password@<database-fqdn>:<database-port>/<database-name>``
+.. code-block:: bash
 
+    vi ~/.lemur/lemur.conf.py
 
-Setup Postgres
---------------
+.. note:: If you are unfamiliar with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
+      ``postgresql://username:password@<database-fqdn>:<database-port>/<database-name>``
+
+Before Lemur will run you need to fill in a few required variables in the configuration file:
+
+.. code-block:: bash
+
+    LEMUR_SECURITY_TEAM_EMAIL
+    #/the e-mail address needs to be enclosed in quotes
+    LEMUR_DEFAULT_COUNTRY
+    LEMUR_DEFAULT_STATE
+    LEMUR_DEFAULT_LOCATION
+    LEMUR_DEFAULT_ORGANIZATION
+    LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
+
+Set Up Postgres
+---------------
 
 For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
 
@@ -118,10 +156,9 @@ First, set a password for the postgres user.  For this guide, we will use ``lemu
 
 .. code-block:: bash
 
-    $ sudo -u postgres psql postgres
-    # \password postgres
-    Enter new password: lemur
-    Enter it again: lemur
+    sudo -u postgres -i
+    psql
+    postgres=# CREATE USER lemur WITH PASSWORD 'lemur';
 
 Once successful, type CTRL-D to exit the Postgres shell.
 
@@ -129,21 +166,15 @@ Next, we will create our new database:
 
 .. code-block:: bash
 
-    $ sudo -u postgres createdb lemur
+    sudo -u postgres createdb lemur
 
 .. _InitializingLemur:
 
-Set a password for lemur user inside Postgres:
-
-.. code-block:: bash
-
-    $ sudo -u postgres psql postgres
-    \password lemur
-    Enter new password: lemur
-    Enter it again: lemur
-
-Again, enter CTRL-D to exit the Postgres shell.
+.. note::
+    For this guide we assume you will use the `postgres` user to connect to your database, when deploying to a VM or container this is often all you will need. If you have a shared database it is recommend you give Lemur its own user.
 
+.. note::
+    Postgres 9.4 or greater is required as Lemur relies advanced data columns (e.g. JSON Column type)
 
 Initializing Lemur
 ------------------
@@ -152,22 +183,38 @@ Lemur provides a helpful command that will initialize your database for you. It
 
 In addition to creating a new user, Lemur also creates a few default email notifications.  These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``.  They basically guarantee that every certificate within Lemur will send one expiration notification to the security team.
 
-Additional notifications can be created through the UI or API.  See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user.
+
+.. code-block:: bash
+
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH SUPERUSER
+
+Additional notifications can be created through the UI or API.  See :ref:`Notification Options <NotificationOptions>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 **Make note of the password used as this will be used during first login to the Lemur UI.**
 
 .. code-block:: bash
 
-    $ lemur db init
-    $ lemur init
+    cd /www/lemur/lemur
+    lemur init
 
-.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+.. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now.
+
+.. code-block:: bash
+
+    sudo -u postgres -i
+    psql
+    postgres=# ALTER USER lemur WITH NOSUPERUSER
 
 
-Setup a Reverse Proxy
----------------------
+.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating a New User <CreateANewUser>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
-By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
+Set Up a Reverse Proxy
+----------------------
+
+By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
 
 
 Proxying with Nginx
@@ -178,7 +225,7 @@ You'll use the builtin ``HttpProxyModule`` within Nginx to handle proxying.  Edi
 ::
 
    location /api {
-        proxy_pass  http://127.0.0.1:5000;
+        proxy_pass  http://127.0.0.1:8000;
         proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
         proxy_redirect off;
         proxy_buffering off;
@@ -199,7 +246,7 @@ After making these changes, restart Nginx service to apply them:
 
 .. code-block:: bash
 
-    $ sudo service nginx restart
+    sudo service nginx restart
 
 
 Starting the Web Service
@@ -220,7 +267,7 @@ you can pass that via the ``--config`` option.
   # the correct host and port!
   lemur --config=/etc/lemur.conf.py start -b 127.0.0.1:8000
 
-You should now be able to test the web service by visiting ``http://localhost:5000/``.
+You should now be able to test the web service by visiting ``http://localhost:8000/``.
 
 
 Running Lemur as a Service
@@ -242,8 +289,8 @@ Configuring Supervisor couldn't be more simple. Just point it to the ``lemur`` e
   autostart=true
   autorestart=true
   redirect_stderr=true
-  stdout_logfile syslog
-  stderr_logfile syslog
+  stdout_logfile=syslog
+  stderr_logfile=syslog
 
 See :ref:`Using Supervisor <UsingSupervisor>` for more details on using Supervisor.
 
@@ -251,13 +298,14 @@ See :ref:`Using Supervisor <UsingSupervisor>` for more details on using Supervis
 Syncing
 -------
 
-Lemur uses periodic sync tasks to make sure it is up-to-date with its environment. As always, things can change outside of Lemur, but we do our best to reconcile those changes, for example, using Cron:
+Lemur uses periodic sync tasks to make sure it is up-to-date with its environment. Things change outside of Lemur we do our best to reconcile those changes. The recommended method is to use CRON:
 
 .. code-block:: bash
 
-  $ crontab -e
-  * 3 * * * lemur sync --all
-  * 3 * * * lemur check_revoked
+  crontab -e
+  */15 * * * * lemur sync -s all
+  0 22 * * * lemur check_revoked
+  0 22 * * * lemur notify
 
 
 Additional Utilities
@@ -280,6 +328,12 @@ unlock
 Decrypts sensitive key material - used to decrypt the secrets stored in source during deployment.
 
 
+Automated celery tasks
+~~~~~~~~~~~~~~~~~~~~~~
+
+Please refer to :ref:`Periodic Tasks <PeriodicTasks>` to learn more about task scheduling in Lemur.
+
+
 What's Next?
 ------------
 

docs/requirements.txt

@@ -1,29 +0,0 @@
-Jinja2>=2.3
-Pygments>=1.2
-Sphinx>=1.3
-docutils>=0.7
-markupsafe
-sphinxcontrib-httpdomain
-Flask==0.10.1
-Flask-RESTful==0.3.3
-Flask-SQLAlchemy==2.1
-Flask-Script==2.0.5
-Flask-Migrate==1.6.0
-Flask-Bcrypt==0.7.1
-Flask-Principal==0.4.0
-Flask-Mail==0.9.1
-SQLAlchemy-Utils==0.31.3
-BeautifulSoup4
-requests==2.8.1
-psycopg2==2.6.1
-arrow==0.7.0
-boto==2.38.0  # we might make this optional
-six==1.10.0
-gunicorn==19.4.1
-pycrypto==2.6.1
-cryptography==1.1.1
-pyopenssl==0.15.1
-pyjwt==1.4.0
-xmltodict==0.9.2
-lockfile==0.12.2
-future==0.15.2

docs/security.rst

@@ -22,7 +22,7 @@ Supported Versions
 ------------------
 
 At any given time, we will provide security support for the `master`_ branch
-as well as the 2 most recent releases.
+as well as the most recent release.
 
 Disclosure Process
 ------------------
@@ -30,20 +30,15 @@ Disclosure Process
 Our process for taking a security issue from private discussion to public
 disclosure involves multiple steps.
 
-Approximately one week before full public disclosure, we will send advance
-notification of the issue to a list of people and organizations, primarily
-composed of operating-system vendors and other distributors of
-``lemur``.  This notification will consist of an email message
-containing:
+Approximately one week before full public disclosure, we will provide advanced notification that a security issue exists. Depending on the severity of the issue, we may choose to either send a targeted email to known Lemur users and contributors or post an issue to the Lemur repository.  In either case, the notification should contain the following.
 
-* A full description of the issue and the affected versions of
-  ``lemur``.
+* A description of the potential impact
+* The affected versions of ``lemur``.
 * The steps we will be taking to remedy the issue.
-* The patches, if any, that will be applied to ``lemur``.
 * The date on which the ``lemur`` team will apply these patches, issue
   new releases, and publicly disclose the issue.
 
-Simultaneously, the reporter of the issue will receive notification of the date
+If the issue was disclosed to us, the reporter will receive notification of the date
 on which we plan to make the issue public.
 
 On the day of disclosure, we will take the following steps:
@@ -52,7 +47,7 @@ On the day of disclosure, we will take the following steps:
   messages for these patches will indicate that they are for security issues,
   but will not describe the issue in any detail; instead, they will warn of
   upcoming disclosure.
-* Issue the relevant releases.
+* Issue an updated release.
 
 If a reported issue is believed to be particularly time-sensitive – due to a
 known exploit in the wild, for example – the time between advance notification
@@ -60,7 +55,7 @@ and public disclosure may be shortened considerably.
 
 The list of people and organizations who receives advanced notification of
 security issues is not, and will not, be made public. This list generally
-consists of high profile downstream distributors and is entirely at the
+consists of high-profile downstream distributors and is entirely at the
 discretion of the ``lemur`` team.
 
 .. _`master`: https://github.com/Netflix/lemur

gulp/build.js

@@ -1,14 +1,13 @@
+'use strict';
+
 var gulp = require('gulp'),
   minifycss = require('gulp-minify-css'),
   concat = require('gulp-concat'),
   less = require('gulp-less'),
   gulpif = require('gulp-if'),
-  order = require('gulp-order'),
   gutil = require('gulp-util'),
-  rename = require('gulp-rename'),
   foreach = require('gulp-foreach'),
-  debug = require('gulp-debug'),
-  path =require('path'),
+  path = require('path'),
   merge = require('merge-stream'),
   del = require('del'),
   size = require('gulp-size'),
@@ -22,85 +21,109 @@ var gulp = require('gulp'),
   useref = require('gulp-useref'),
   filter = require('gulp-filter'),
   rev = require('gulp-rev'),
-  revReplace = require('gulp-rev-replace'),
   imagemin = require('gulp-imagemin'),
   minifyHtml = require('gulp-minify-html'),
   bowerFiles = require('main-bower-files'),
   karma = require('karma'),
-  replace = require('gulp-replace');
+  replace = require('gulp-replace'),
+  argv = require('yargs').argv;
 
-gulp.task('default', ['clean'], function () {
+// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
+function escapeRegExp(string) {
+  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
+}
+
+function replaceAll(string, find, replace) {
+  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
+}
+
+function stringSrc(filename, string) {
+  let src = require('stream').Readable({objectMode: true});
+  src._read = function () {
+    this.push(new gutil.File({cwd: '', base: '', path: filename, contents: Buffer.from(string)}));
+    this.push(null);
+  };
+  return src;
+}
+
+gulp.task('clean', function (done) {
+  del(['.tmp', 'lemur/static/dist'], done);
+  done();
+});
+
+gulp.task('default', gulp.series(['clean'], function () {
   gulp.start('fonts', 'styles');
-});
-
-gulp.task('clean', function (cb) {
-  del(['.tmp', 'lemur/static/dist'], cb);
-});
+}));
 
 gulp.task('test', function (done) {
   new karma.Server({
     configFile: __dirname + '/karma.conf.js',
     singleRun: true
-  }, function() {
-    done();
+  }, function (err) {
+    if (err === 0) {
+      done();
+    } else {
+      // if karma server failed to start raise error
+      done(new gutil.PluginError('karma', {
+        message: 'Karma Tests failed'
+      }));
+    }
   }).start();
 });
 
 gulp.task('dev:fonts', function () {
-  var fileList = [
+  let fileList = [
     'bower_components/bootstrap/dist/fonts/*',
     'bower_components/fontawesome/fonts/*'
   ];
 
   return gulp.src(fileList)
-    .pipe(gulp.dest('.tmp/fonts'));
+    .pipe(gulp.dest('.tmp/fonts')); // returns a stream making it async
 });
 
 gulp.task('dev:styles', function () {
-  var baseContent = '@import "bower_components/bootstrap/less/bootstrap.less";@import "bower_components/bootswatch/$theme$/variables.less";@import "bower_components/bootswatch/$theme$/bootswatch.less";@import "bower_components/bootstrap/less/utilities.less";';
-  var isBootswatchFile = function (file) {
+  let baseContent = '@import "bower_components/bootstrap/less/bootstrap.less";@import "bower_components/bootswatch/$theme$/variables.less";@import "bower_components/bootswatch/$theme$/bootswatch.less";@import "bower_components/bootstrap/less/utilities.less";';
+  let isBootswatchFile = function (file) {
 
-    var suffix = 'bootswatch.less';
+    let suffix = 'bootswatch.less';
     return file.path.indexOf(suffix, file.path.length - suffix.length) !== -1;
   };
 
-  var isBootstrapFile = function (file) {
-    var suffix = 'bootstrap-',
+  let isBootstrapFile = function (file) {
+    let suffix = 'bootstrap-',
       fileName = path.basename(file.path);
 
     return fileName.indexOf(suffix) === 0;
   };
 
-  var fileList = [
+  let fileList = [
     'bower_components/bootswatch/sandstone/bootswatch.less',
     'bower_components/fontawesome/css/font-awesome.css',
-    'bower_components/angular-spinkit/src/angular-spinkit.css',
     'bower_components/angular-chart.js/dist/angular-chart.css',
     'bower_components/angular-loading-bar/src/loading-bar.css',
     'bower_components/angular-ui-switch/angular-ui-switch.css',
     'bower_components/angular-wizard/dist/angular-wizard.css',
-    'bower_components/ng-table/ng-table.css',
+    'bower_components/ng-table/dist/ng-table.css',
     'bower_components/angularjs-toaster/toaster.css',
+    'bower_components/angular-ui-select/dist/select.css',
     'lemur/static/app/styles/lemur.css'
   ];
 
   return gulp.src(fileList)
     .pipe(gulpif(isBootswatchFile, foreach(function (stream, file) {
-      var themeName = path.basename(path.dirname(file.path)),
-        content = replaceAll(baseContent, '$theme$', themeName),
-        file = string_src('bootstrap-' +  themeName + '.less', content);
-
-      return file;
+      let themeName = path.basename(path.dirname(file.path)),
+        content = replaceAll(baseContent, '$theme$', themeName);
+      return stringSrc('bootstrap-' + themeName + '.less', content);
     })))
     .pipe(less())
     .pipe(gulpif(isBootstrapFile, foreach(function (stream, file) {
-      var fileName = path.basename(file.path),
+      let fileName = path.basename(file.path),
         themeName = fileName.substring(fileName.indexOf('-') + 1, fileName.indexOf('.'));
 
       // http://stackoverflow.com/questions/21719833/gulp-how-to-add-src-files-in-the-middle-of-a-pipe
       // https://github.com/gulpjs/gulp/blob/master/docs/recipes/using-multiple-sources-in-one-task.md
-      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css']))
-        .pipe(concat('style-' + themeName + ".css"));
+      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css'], {allowEmpty: true}))
+        .pipe(concat('style-' + themeName + '.css'));
     })))
     .pipe(plumber())
     .pipe(concat('styles.css'))
@@ -110,24 +133,6 @@ gulp.task('dev:styles', function () {
     .pipe(size());
 });
 
-// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
-function escapeRegExp(string) {
-  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
-}
-
-function replaceAll(string, find, replace) {
-  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
-}
-
-function string_src(filename, string) {
-  var src = require('stream').Readable({ objectMode: true });
-  src._read = function () {
-    this.push(new gutil.File({ cwd: "", base: "", path: filename, contents: new Buffer(string) }));
-    this.push(null);
-  };
-  return src;
-}
-
 gulp.task('dev:scripts', function () {
   return gulp.src(['lemur/static/app/angular/**/*.js'])
     .pipe(jshint())
@@ -143,49 +148,35 @@ gulp.task('build:extras', function () {
 function injectHtml(isDev) {
   return gulp.src('lemur/static/app/index.html')
     .pipe(
-    inject(gulp.src(bowerFiles({ base: 'app' }), {
-      read: false
-    }), {
-      starttag: '<!-- inject:bower:{{ext}} -->',
-      addRootSlash: false,
-      ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
-    })
-  )
-    .pipe(inject(gulp.src(['lemur/static/app/angular/**/*.js'], {
-      read: false
-    }), {
-      read: false,
+      inject(gulp.src(bowerFiles({base: 'app'})), {
+        starttag: '<!-- inject:bower:{{ext}} -->',
+        addRootSlash: false,
+        ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
+      })
+    )
+    .pipe(inject(gulp.src(['lemur/static/app/angular/**/*.js']), {
       starttag: '<!-- inject:{{ext}} -->',
       addRootSlash: false,
       ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
     }))
-    .pipe(inject(gulp.src(['.tmp/styles/**/*.css'], {
-      read: false
-    }), {
-      read: false,
+    .pipe(inject(gulp.src(['.tmp/styles/**/*.css']), {
       starttag: '<!-- inject:{{ext}} -->',
       addRootSlash: false,
       ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
     }))
     .pipe(
-    gulpif(!isDev,
-      inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js'), {
-        read: false,
-        starttag: '<!-- inject:ngviews -->',
-        addRootSlash: false
-      })
-    )
-  )
-    .pipe(gulp.dest('.tmp/'));
+      gulpif(!isDev,
+        inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js', {allowEmpty: true}), {
+          starttag: '<!-- inject:ngviews -->',
+          addRootSlash: false
+        })
+      )
+    ).pipe(gulp.dest('.tmp/'));
 }
 
-gulp.task('dev:inject', ['dev:styles', 'dev:scripts'], function () {
+gulp.task('dev:inject', gulp.series(['dev:styles', 'dev:scripts'], function () {
   return injectHtml(true);
-});
-
-gulp.task('build:inject', ['dev:styles', 'dev:scripts', 'build:ngviews'], function () {
-  return injectHtml(false);
-});
+}));
 
 gulp.task('build:ngviews', function () {
   return gulp.src(['lemur/static/app/angular/**/*.html'])
@@ -198,32 +189,30 @@ gulp.task('build:ngviews', function () {
     .pipe(size());
 });
 
-gulp.task('build:html', ['dev:styles', 'dev:scripts', 'build:ngviews', 'build:inject'], function () {
-  var jsFilter = filter('**/*.js');
-  var cssFilter = filter('**/*.css');
+gulp.task('build:inject', gulp.series(['dev:styles', 'dev:scripts', 'build:ngviews'], function () {
+  return injectHtml(false);
+}));
 
-  var assets = useref.assets();
+gulp.task('build:html', gulp.series(['build:inject'], function () {
+  let jsFilter = filter(['**/*.js'], {'restore': true});
+  let cssFilter = filter(['**/*.css'], {'restore': true});
 
   return gulp.src('.tmp/index.html')
-    .pipe(assets)
-    .pipe(rev())
     .pipe(jsFilter)
     .pipe(ngAnnotate())
-    .pipe(jsFilter.restore())
+    .pipe(jsFilter.restore)
     .pipe(cssFilter)
     .pipe(csso())
-    .pipe(cssFilter.restore())
-    .pipe(assets.restore())
+    .pipe(cssFilter.restore)
     .pipe(useref())
-    .pipe(revReplace())
     .pipe(gulp.dest('lemur/static/dist'))
     .pipe(size());
-});
+}));
 
-gulp.task('build:fonts', ['dev:fonts'], function () {
+gulp.task('build:fonts', gulp.series(['dev:fonts'], function () {
   return gulp.src('.tmp/fonts/**/*')
     .pipe(gulp.dest('lemur/static/dist/fonts'));
-});
+}));
 
 gulp.task('build:images', function () {
   return gulp.src('lemur/static/app/images/**/*')
@@ -241,10 +230,32 @@ gulp.task('package:strip', function () {
     .pipe(replace('http:\/\/localhost:3000', ''))
     .pipe(replace('http:\/\/localhost:8000', ''))
     .pipe(useref())
-    .pipe(revReplace())
     .pipe(gulp.dest('lemur/static/dist/scripts'))
     .pipe(size());
 });
 
-gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']);
-gulp.task('package', ['package:strip']);
+gulp.task('addUrlContextPath:revision', function () {
+  return gulp.src(['lemur/static/dist/**/*.css', 'lemur/static/dist/**/*.js'])
+    .pipe(rev())
+    .pipe(gulp.dest('lemur/static/dist'))
+    .pipe(rev.manifest())
+    .pipe(gulp.dest('lemur/static/dist'));
+});
+
+gulp.task('addUrlContextPath:revreplace', gulp.series(['addUrlContextPath:revision'], function () {
+  return gulp.src('lemur/static/dist/index.html')
+    .pipe(gulp.dest('lemur/static/dist'));
+}));
+
+gulp.task('addUrlContextPath', gulp.series(['addUrlContextPath:revreplace'], function () {
+  let urlContextPathExists = !!argv.urlContextPath;
+  return gulp.src(['lemur/static/dist/scripts/main*.js', 'lemur/static/dist/angular/**/*.html'])
+    .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
+    .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/')))
+    .pipe(gulp.dest(function (file) {
+      return file.base;
+    }));
+}));
+
+gulp.task('build', gulp.series(['build:images', 'build:fonts', 'build:html', 'build:extras']));
+gulp.task('package', gulp.series(['addUrlContextPath', 'package:strip']));

gulp/karma.conf.js

@@ -1,12 +1,14 @@
 // Contents of: config/karma.conf.js
+'use strict';
+
 module.exports = function (config) {
   config.set({
-    basePath : '../',
+    basePath: '../',
 
     // Fix for "JASMINE is not supported anymore" warning
-    frameworks : ["jasmine"],
+    frameworks: ['jasmine'],
 
-    files : [
+    files: [
       'app/lib/angular/angular.js',
       'app/lib/angular/angular-*.js',
       'test/lib/angular/angular-mocks.js',
@@ -14,14 +16,22 @@ module.exports = function (config) {
       'test/unit/**/*.js'
     ],
 
-    autoWatch : true,
+    autoWatch: true,
 
-    browsers : ['Chrome'],
+    browsers: [process.env.TRAVIS ? 'Chrome_travis_ci' : 'Chrome'],
+    customLaunchers: {
+      'Chrome_travis_ci': {
+        base: 'Chrome',
+        flags: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu',],
+      },
+    },
 
-    junitReporter : {
-      outputFile : 'test_out/unit.xml',
-      suite      : 'unit'
+    junitReporter: {
+      outputFile: 'test_out/unit.xml',
+      suite: 'unit'
       //...
-    }
+    },
+
+    failOnEmptyTestSuite: false,
   });
-};
+};

gulp/server.js

@@ -1,36 +1,37 @@
 'use strict';
 
 var gulp = require('gulp');
+const watch = require('./watch')
 
 var browserSync = require('browser-sync');
 var httpProxy = require('http-proxy');
 
 /* This configuration allow you to configure browser sync to proxy your backend */
-/*
- var proxyTarget = 'http://localhost/context/'; // The location of your backend
- var proxyApiPrefix = 'api'; // The element in the URL which differentiate between API request and static file request
+
+ var proxyTarget = 'http://localhost:8000/'; // The location of your backend
+ var proxyApiPrefix = '/api/'; // The element in the URL which differentiate between API request and static file request
  var proxy = httpProxy.createProxyServer({
- target: proxyTarget
+     target: proxyTarget
  });
  function proxyMiddleware(req, res, next) {
- if (req.url.indexOf(proxyApiPrefix) !== -1) {
- proxy.web(req, res);
- } else {
- next();
+     if (req.url.indexOf(proxyApiPrefix) !== -1) {
+         proxy.web(req, res);
+     } else {
+         next();
+     }
  }
- }
- */
 
 function browserSyncInit(baseDir, files, browser) {
   browser = browser === undefined ? 'default' : browser;
 
   browserSync.instance = browserSync.init(files, {
     startPath: '/index.html',
-    server: {
-      baseDir: baseDir,
-      routes: {
-        '/bower_components': './bower_components'
-      }
+      server: {
+          middleware: [proxyMiddleware],
+          baseDir: baseDir,
+          routes: {
+              '/bower_components': './bower_components'
+          }
     },
     browser: browser,
     ghostMode: false
@@ -38,7 +39,7 @@ function browserSyncInit(baseDir, files, browser) {
 
 }
 
-gulp.task('serve', ['watch'], function () {
+gulp.task('serve', gulp.series(['watch'], function (done) {
   browserSyncInit([
     '.tmp',
     'lemur/static/app'
@@ -51,9 +52,12 @@ gulp.task('serve', ['watch'], function () {
     'lemur/static/app/angular/**/*',
     'lemur/static/app/index.html'
   ]);
-});
+
+  done();
+}));
 
 
-gulp.task('serve:dist', ['build'], function () {
+gulp.task('serve:dist', gulp.series(['build'], function (done) {
   browserSyncInit('lemur/static/dist');
-});
+  done();
+}));

gulp/watch.js

@@ -3,10 +3,13 @@
 var gulp = require('gulp');
 
 
-gulp.task('watch', ['dev:styles', 'dev:scripts', 'dev:inject', 'dev:fonts'] ,function () {
-  gulp.watch('app/styles/**/*.less', ['dev:styles']);
-  gulp.watch('app/styles/**/*.css', ['dev:styles']);
-  gulp.watch('app/**/*.js', ['dev:scripts']);
-  gulp.watch('app/images/**/*', ['build:images']);
-  gulp.watch('bower.json', ['dev:inject']);
-});
+const watch = gulp.task('watch', gulp.series(['dev:inject', 'dev:fonts'] ,function (done) {
+  gulp.watch('app/styles/**/*.less', gulp.series('dev:styles'));
+  gulp.watch('app/styles/**/*.css', gulp.series('dev:styles'));
+  gulp.watch('app/**/*.js', gulp.series('dev:scripts'));
+  gulp.watch('app/images/**/*', gulp.series('build:images'));
+  gulp.watch('bower.json', gulp.series('dev:inject'));
+  done();
+}));
+
+module.exports = {watch:watch}

hooks/pre-commit

@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-
-import glob
-import os
-import sys
-
-os.environ['PYFLAKES_NODOCTEST'] = '1'
-
-# pep8.py uses sys.argv to find setup.cfg
-sys.argv = [os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)]
-
-# git usurbs your bin path for hooks and will always run system python
-if 'VIRTUAL_ENV' in os.environ:
-    site_packages = glob.glob(
-        '%s/lib/*/site-packages' % os.environ['VIRTUAL_ENV'])[0]
-    sys.path.insert(0, site_packages)
-
-
-def py_lint(files_modified):
-    from flake8.main import DEFAULT_CONFIG
-    from flake8.engine import get_style_guide
-
-    # remove non-py files and files which no longer exist
-    files_modified = filter(lambda x: x.endswith('.py'), files_modified)
-
-    flake8_style = get_style_guide(parse_argv=True, config_file=DEFAULT_CONFIG)
-    report = flake8_style.check_files(files_modified)
-
-    return report.total_errors != 0
-
-
-def main():
-    from flake8.hooks import run
-
-    gitcmd = "git diff-index --cached --name-only HEAD"
-
-    _, files_modified, _ = run(gitcmd)
-
-    files_modified = filter(lambda x: os.path.exists(x), files_modified)
-
-    if py_lint(files_modified):
-        return 1
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

lemur/__about__.py

@@ -1,18 +1,24 @@
 from __future__ import absolute_import, division, print_function
 
 __all__ = [
-    "__title__", "__summary__", "__uri__", "__version__", "__author__",
-    "__email__", "__license__", "__copyright__",
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
 ]
 
 __title__ = "lemur"
-__summary__ = ("Certificate management and orchestration service")
+__summary__ = "Certificate management and orchestration service"
 __uri__ = "https://github.com/Netflix/lemur"
 
-__version__ = "0.2"
+__version__ = "develop"
 
 __author__ = "The Lemur developers"
 __email__ = "security@netflix.com"
 
 __license__ = "Apache License, Version 2.0"
-__copyright__ = "Copyright 2015 {0}".format(__author__)
+__copyright__ = "Copyright 2018 {0}".format(__author__)

lemur/__init__.py

@@ -1,16 +1,19 @@
 """
 .. module: lemur
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-
+.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
+.. moduleauthor:: Hossein Shafagh <hshafagh@netflix.com>
 
 """
-from __future__ import absolute_import, division, print_function
+import time
+from flask import g, request
 
 from lemur import factory
+from lemur.extensions import metrics
 
 from lemur.users.views import mod as users_bp
 from lemur.roles.views import mod as roles_bp
@@ -23,16 +26,33 @@ from lemur.defaults.views import mod as defaults_bp
 from lemur.plugins.views import mod as plugins_bp
 from lemur.notifications.views import mod as notifications_bp
 from lemur.sources.views import mod as sources_bp
+from lemur.endpoints.views import mod as endpoints_bp
+from lemur.logs.views import mod as logs_bp
+from lemur.api_keys.views import mod as api_key_bp
+from lemur.pending_certificates.views import mod as pending_certificates_bp
+from lemur.dns_providers.views import mod as dns_providers_bp
 
 from lemur.__about__ import (
-    __author__, __copyright__, __email__, __license__, __summary__, __title__,
-    __uri__, __version__
+    __author__,
+    __copyright__,
+    __email__,
+    __license__,
+    __summary__,
+    __title__,
+    __uri__,
+    __version__,
 )
 
 
 __all__ = [
-    "__title__", "__summary__", "__uri__", "__version__", "__author__",
-    "__email__", "__license__", "__copyright__",
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
 ]
 
 LEMUR_BLUEPRINTS = (
@@ -46,12 +66,19 @@ LEMUR_BLUEPRINTS = (
     defaults_bp,
     plugins_bp,
     notifications_bp,
-    sources_bp
+    sources_bp,
+    endpoints_bp,
+    logs_bp,
+    api_key_bp,
+    pending_certificates_bp,
+    dns_providers_bp,
 )
 
 
-def create_app(config=None):
-    app = factory.create_app(app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config)
+def create_app(config_path=None):
+    app = factory.create_app(
+        app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config_path
+    )
     configure_hook(app)
     return app
 
@@ -62,16 +89,40 @@ def configure_hook(app):
     :param app:
     :return:
     """
-    from flask.ext.principal import PermissionDenied
-    from lemur.decorators import crossdomain
-    if app.config.get('CORS'):
-        @app.after_request
-        @crossdomain(origin=u"http://localhost:3000", methods=['PUT', 'HEAD', 'GET', 'POST', 'OPTIONS', 'DELETE'])
-        def after(response):
+    from flask import jsonify
+    from werkzeug.exceptions import HTTPException
+
+    @app.errorhandler(Exception)
+    def handle_error(e):
+        code = 500
+        if isinstance(e, HTTPException):
+            code = e.code
+
+        app.logger.exception(e)
+        return jsonify(error=str(e)), code
+
+    @app.before_request
+    def before_request():
+        g.request_start_time = time.time()
+
+    @app.after_request
+    def after_request(response):
+        # Return early if we don't have the start time
+        if not hasattr(g, "request_start_time"):
             return response
 
-    @app.errorhandler(PermissionDenied)
-    def handle_invalid_usage(error):
-        response = {'message': 'You are not allow to access this resource'}
-        response.status_code = 403
+        # Get elapsed time in milliseconds
+        elapsed = time.time() - g.request_start_time
+        elapsed = int(round(1000 * elapsed))
+
+        # Collect request/response tags
+        tags = {
+            "endpoint": request.endpoint,
+            "request_method": request.method.lower(),
+            "status_code": response.status_code,
+        }
+
+        # Record our response time metric
+        metrics.send("response_time", "TIMER", elapsed, metric_tags=tags)
+        metrics.send("status_code_{}".format(response.status_code), "counter", 1)
         return response

lemur/acme_providers/cli.py

@@ -0,0 +1,191 @@
+import time
+import json
+import arrow
+
+from flask_script import Manager
+from flask import current_app
+
+from lemur.extensions import sentry
+from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.plugins import plugins
+from lemur.plugins.lemur_acme.plugin import AcmeHandler
+from lemur.plugins.lemur_aws import s3
+
+manager = Manager(
+    usage="Handles all ACME related tasks"
+)
+
+
+@manager.option(
+    "-d",
+    "--domain",
+    dest="domain",
+    required=True,
+    help="Name of the Domain to store to (ex. \"_acme-chall.test.com\".",
+)
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    required=True,
+    help="Value of the Token to store in DNS as content.",
+)
+def dnstest(domain, token):
+    """
+    Create, verify, and delete DNS TXT records using an autodetected provider.
+    """
+    print("[+] Starting ACME Tests.")
+    change_id = (domain, token)
+
+    acme_handler = AcmeHandler()
+    acme_handler.autodetect_dns_providers(domain)
+    if not acme_handler.dns_providers_for_domain[domain]:
+        raise Exception(f"No DNS providers found for domain: {format(domain)}.")
+
+    # Create TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        print(f"[+] Creating TXT Record in `{dns_provider.name}` provider")
+        change_id = dns_provider_plugin.create_txt_record(domain, token, account_number)
+
+    print("[+] Verifying TXT Record has propagated to DNS.")
+    print("[+] This step could take a while...")
+    time.sleep(10)
+
+    # Verify TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        try:
+            dns_provider_plugin.wait_for_dns_change(change_id, account_number)
+            print(f"[+] Verified TXT Record in `{dns_provider.name}` provider")
+        except Exception:
+            sentry.captureException()
+            current_app.logger.debug(
+                f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: "
+                f"{account_number}",
+                exc_info=True,
+            )
+            print(f"[+] Unable to Verify TXT Record in `{dns_provider.name}` provider")
+
+    time.sleep(10)
+
+    # Delete TXT Records
+    for dns_provider in acme_handler.dns_providers_for_domain[domain]:
+        dns_provider_plugin = acme_handler.get_dns_provider(dns_provider.provider_type)
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+
+        # TODO(csine@: Add Exception Handling
+        dns_provider_plugin.delete_txt_record(change_id, account_number, domain, token)
+        print(f"[+] Deleted TXT Record in `{dns_provider.name}` provider")
+
+    status = SUCCESS_METRIC_STATUS
+    print("[+] Done with ACME Tests.")
+
+
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="Value of the Token",
+)
+@manager.option(
+    "-n",
+    "--token_name",
+    dest="token_name",
+    default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="path",
+)
+@manager.option(
+    "-p",
+    "--prefix",
+    dest="prefix",
+    default="test/",
+    required=False,
+    help="S3 bucket prefix",
+)
+@manager.option(
+    "-a",
+    "--account_number",
+    dest="account_number",
+    required=True,
+    help="AWS Account",
+)
+@manager.option(
+    "-b",
+    "--bucket_name",
+    dest="bucket_name",
+    required=True,
+    help="Bucket Name",
+)
+def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
+    """
+    This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
+    It mainly serves for testing purposes.
+    :param token:
+    :param token_name:
+    :param prefix:
+    :param account_number:
+    :param bucket_name:
+    :return:
+    """
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket_name,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account_number,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    p = plugins.get("aws-s3")
+    p.upload_acme_token(token_name, token, additional_options)
+
+    if not prefix.endswith("/"):
+        prefix + "/"
+
+    token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
+    assert(token_res == token)
+    s3.delete(bucket_name, prefix + token_name, account_number=account_number)

lemur/analyze/service.py

@@ -1,64 +0,0 @@
-"""
-.. module: lemur.analyze.service
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-"""
-# def analyze(endpoints, truststores):
-#    results = {"headings": ["Endpoint"],
-#               "results": [],
-#               "time": datetime.now().strftime("#Y%m%d %H:%M:%S")}
-#
-#    for store in truststores:
-#        results['headings'].append(os.path.basename(store))
-#
-#    for endpoint in endpoints:
-#        result_row = [endpoint]
-#        for store in truststores:
-#            result = {'details': []}
-#
-#            tests = []
-#            for region, ip in REGIONS.items():
-#                try:
-#                    domain = dns.name.from_text(endpoint)
-#                    if not domain.is_absolute():
-#                        domain = domain.concatenate(dns.name.root)
-#
-#                    my_resolver = dns.resolver.Resolver()
-#                    my_resolver.nameservers = [ip]
-#                    answer = my_resolver.query(domain)
-#
-#                    #force the testing of regional enpoints by changing the dns server
-#                    response = requests.get('https://' + str(answer[0]), verify=store)
-#                    tests.append('pass')
-#                    result['details'].append("{}: SSL testing completed without errors".format(region))
-#
-#                except SSLError as e:
-#                    log.debug(e)
-#                    if 'hostname' in str(e):
-#                        tests.append('pass')
-#                        result['details'].append(
-#                              "{}: This test passed ssl negotiation but failed hostname verification because \
-#                                   the hostname is not included in the certificate".format(region))
-#                    elif 'certificate verify failed' in str(e):
-#                        tests.append('fail')
-#                        result['details'].append("{}: This test failed to verify the SSL certificate".format(region))
-#                    else:
-#                        tests.append('fail')
-#                        result['details'].append("{}: {}".format(region, str(e)))
-#
-#                except Exception as e:
-#                    log.debug(e)
-#                    tests.append('fail')
-#                    result['details'].append("{}: {}".format(region, str(e)))
-#
-#            #any failing tests fails the whole endpoint
-#            if 'fail' in tests:
-#                result['test'] = 'fail'
-#            else:
-#                result['test'] = 'pass'
-#
-#            result_row.append(result)
-#        results['results'].append(result_row)
-#    return results
-#

lemur/api_keys/cli.py

@@ -0,0 +1,50 @@
+"""
+.. module: lemur.api_keys.cli
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from flask_script import Manager
+from lemur.api_keys import service as api_key_service
+from lemur.auth.service import create_token
+
+from datetime import datetime
+
+manager = Manager(usage="Handles all api key related tasks.")
+
+
+@manager.option(
+    "-u", "--user-id", dest="uid", help="The User ID this access key belongs too."
+)
+@manager.option("-n", "--name", dest="name", help="The name of this API Key.")
+@manager.option(
+    "-t", "--ttl", dest="ttl", help="The TTL of this API Key. -1 for forever."
+)
+def create(uid, name, ttl):
+    """
+    Create a new api key for a user.
+    :return:
+    """
+    print("[+] Creating a new api key.")
+    key = api_key_service.create(
+        user_id=uid,
+        name=name,
+        ttl=ttl,
+        issued_at=int(datetime.utcnow().timestamp()),
+        revoked=False,
+    )
+    print("[+] Successfully created a new api key. Generating a JWT...")
+    jwt = create_token(uid, key.id, key.ttl)
+    print("[+] Your JWT is: {jwt}".format(jwt=jwt))
+
+
+@manager.option("-a", "--api-key-id", dest="aid", help="The API Key ID to revoke.")
+def revoke(aid):
+    """
+    Revokes an api key for a user.
+    :return:
+    """
+    print("[-] Revoking the API Key api key.")
+    api_key_service.revoke(aid=aid)
+    print("[+] Successfully revoked the api key")

lemur/api_keys/models.py

@@ -0,0 +1,30 @@
+"""
+.. module: lemur.api_keys.models
+    :platform: Unix
+    :synopsis: This module contains all of the models need to create an api key within Lemur.
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from sqlalchemy import BigInteger, Boolean, Column, ForeignKey, Integer, String
+
+from lemur.database import db
+
+
+class ApiKey(db.Model):
+    __tablename__ = "api_keys"
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    user_id = Column(Integer, ForeignKey("users.id"))
+    ttl = Column(BigInteger)
+    issued_at = Column(BigInteger)
+    revoked = Column(Boolean)
+
+    def __repr__(self):
+        return "ApiKey(name={name}, user_id={user_id}, ttl={ttl}, issued_at={iat}, revoked={revoked})".format(
+            user_id=self.user_id,
+            name=self.name,
+            ttl=self.ttl,
+            iat=self.issued_at,
+            revoked=self.revoked,
+        )

lemur/api_keys/schemas.py

@@ -0,0 +1,63 @@
+"""
+.. module: lemur.api_keys.schemas
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from flask import g
+from marshmallow import fields
+
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.users.schemas import UserNestedOutputSchema, UserInputSchema
+
+
+def current_user_id():
+    return {
+        "id": g.current_user.id,
+        "email": g.current_user.email,
+        "username": g.current_user.username,
+    }
+
+
+class ApiKeyInputSchema(LemurInputSchema):
+    name = fields.String(required=False)
+    user = fields.Nested(
+        UserInputSchema, missing=current_user_id, default=current_user_id
+    )
+    ttl = fields.Integer()
+
+
+class ApiKeyRevokeSchema(LemurInputSchema):
+    id = fields.Integer(required=True)
+    name = fields.String()
+    user = fields.Nested(UserInputSchema, required=True)
+    revoked = fields.Boolean()
+    ttl = fields.Integer()
+    issued_at = fields.Integer(required=False)
+
+
+class UserApiKeyInputSchema(LemurInputSchema):
+    name = fields.String(required=False)
+    ttl = fields.Integer()
+
+
+class ApiKeyOutputSchema(LemurOutputSchema):
+    jwt = fields.String()
+
+
+class ApiKeyDescribedOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    name = fields.String()
+    user = fields.Nested(UserNestedOutputSchema)
+    ttl = fields.Integer()
+    issued_at = fields.Integer()
+    revoked = fields.Boolean()
+
+
+api_key_input_schema = ApiKeyInputSchema()
+api_key_revoke_schema = ApiKeyRevokeSchema()
+api_key_output_schema = ApiKeyOutputSchema()
+api_keys_output_schema = ApiKeyDescribedOutputSchema(many=True)
+api_key_described_output_schema = ApiKeyDescribedOutputSchema()
+user_api_key_input_schema = UserApiKeyInputSchema()

lemur/api_keys/service.py

@@ -0,0 +1,104 @@
+"""
+.. module: lemur.api_keys.service
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from lemur import database
+from lemur.api_keys.models import ApiKey
+from lemur.logs import service as log_service
+
+
+def get(aid):
+    """
+    Retrieves an api key by its ID.
+    :param aid: The access key id to get.
+    :return:
+    """
+    return database.get(ApiKey, aid)
+
+
+def delete(access_key):
+    """
+    Delete an access key. This is one way to remove a key, though you probably should just set revoked.
+    :param access_key:
+    :return:
+    """
+    log_service.audit_log("delete_api_key", access_key.name, "Deleting the API key")
+    database.delete(access_key)
+
+
+def revoke(aid):
+    """
+    Revokes an api key.
+    :param aid:
+    :return:
+    """
+    api_key = get(aid)
+    setattr(api_key, "revoked", True)
+
+    log_service.audit_log("revoke_api_key", api_key.name, "Revoking API key")
+    return database.update(api_key)
+
+
+def get_all_api_keys():
+    """
+    Retrieves all Api Keys.
+    :return:
+    """
+    return ApiKey.query.all()
+
+
+def create(**kwargs):
+    """
+    Creates a new API Key.
+
+    :param kwargs:
+    :return:
+    """
+    api_key = ApiKey(**kwargs)
+    # this logs only metadata about the api key
+    log_service.audit_log("create_api_key", api_key.name, f"Creating the API key {api_key}")
+
+    database.create(api_key)
+    return api_key
+
+
+def update(api_key, **kwargs):
+    """
+    Updates an api key.
+    :param api_key:
+    :param kwargs:
+    :return:
+    """
+    for key, value in kwargs.items():
+        setattr(api_key, key, value)
+
+    log_service.audit_log("update_api_key", api_key.name, f"Update summary - {kwargs}")
+    return database.update(api_key)
+
+
+def render(args):
+    """
+    Helper to parse REST Api requests
+
+    :param args:
+    :return:
+    """
+    query = database.session_query(ApiKey)
+    user_id = args.pop("user_id", None)
+    aid = args.pop("id", None)
+    has_permission = args.pop("has_permission", False)
+    requesting_user_id = args.pop("requesting_user_id")
+
+    if user_id:
+        query = query.filter(ApiKey.user_id == user_id)
+
+    if aid:
+        query = query.filter(ApiKey.id == aid)
+
+    if not has_permission:
+        query = query.filter(ApiKey.user_id == requesting_user_id)
+
+    return database.sort_and_page(query, ApiKey, args)

lemur/api_keys/views.py

@@ -0,0 +1,625 @@
+"""
+.. module: lemur.api_keys.views
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+
+"""
+from datetime import datetime
+
+from flask import Blueprint, g
+from flask_restful import reqparse, Api
+
+from lemur.api_keys import service
+from lemur.auth.service import AuthenticatedResource, create_token
+from lemur.auth.permissions import ApiKeyCreatorPermission
+
+from lemur.common.schema import validate_schema
+from lemur.common.utils import paginated_parser
+
+from lemur.api_keys.schemas import (
+    api_key_input_schema,
+    api_key_revoke_schema,
+    api_key_output_schema,
+    api_keys_output_schema,
+    api_key_described_output_schema,
+    user_api_key_input_schema,
+)
+
+mod = Blueprint("api_keys", __name__)
+api = Api(mod)
+
+
+class ApiKeyList(AuthenticatedResource):
+    """ Defines the 'api_keys' endpoint """
+
+    def __init__(self):
+        super(ApiKeyList, self).__init__()
+
+    @validate_schema(None, api_keys_output_schema)
+    def get(self):
+        """
+        .. http:get:: /keys
+
+           The current list of api keys, that you can see.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [
+                    {
+                      "id": 1,
+                      "name": "custom name",
+                      "user_id": 1,
+                      "ttl": -1,
+                      "issued_at": 12,
+                      "revoked": false
+                    }
+                ],
+                "total": 1
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int default is 1
+           :query count: count number. default is 10
+           :query user_id: a user to filter by.
+           :query id: an access key to filter by.
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        args["has_permission"] = ApiKeyCreatorPermission().can()
+        args["requesting_user_id"] = g.current_user.id
+        return service.render(args)
+
+    @validate_schema(api_key_input_schema, api_key_output_schema)
+    def post(self, data=None):
+        """
+        .. http:post:: /keys
+
+           Creates an API Key.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              POST /keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                "name": "my custom name",
+                "user_id": 1,
+                "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+
+                "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if not ApiKeyCreatorPermission().can():
+            if data["user"]["id"] != g.current_user.id:
+                return (
+                    dict(
+                        message="You are not authorized to create tokens for: {0}".format(
+                            data["user"]["username"]
+                        )
+                    ),
+                    403,
+                )
+
+        access_token = service.create(
+            name=data["name"],
+            user_id=data["user"]["id"],
+            ttl=data["ttl"],
+            revoked=False,
+            issued_at=int(datetime.utcnow().timestamp()),
+        )
+        return dict(
+            jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
+        )
+
+
+class ApiKeyUserList(AuthenticatedResource):
+    """ Defines the 'keys' endpoint on the 'users' endpoint. """
+
+    def __init__(self):
+        super(ApiKeyUserList, self).__init__()
+
+    @validate_schema(None, api_keys_output_schema)
+    def get(self, user_id):
+        """
+        .. http:get:: /users/:user_id/keys
+
+           The current list of api keys for a user, that you can see.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /users/1/keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [
+                    {
+                      "id": 1,
+                      "name": "custom name",
+                      "user_id": 1,
+                      "ttl": -1,
+                      "issued_at": 12,
+                      "revoked": false
+                    }
+                ],
+                "total": 1
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int default is 1
+           :query count: count number. default is 10
+           :query id: an access key to filter by.
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        args["has_permission"] = ApiKeyCreatorPermission().can()
+        args["requesting_user_id"] = g.current_user.id
+        args["user_id"] = user_id
+        return service.render(args)
+
+    @validate_schema(user_api_key_input_schema, api_key_output_schema)
+    def post(self, user_id, data=None):
+        """
+        .. http:post:: /users/:user_id/keys
+
+           Creates an API Key for a user.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              POST /users/1/keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                "name": "my custom name"
+                "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+
+                "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if not ApiKeyCreatorPermission().can():
+            if user_id != g.current_user.id:
+                return (
+                    dict(
+                        message="You are not authorized to create tokens for: {0}".format(
+                            user_id
+                        )
+                    ),
+                    403,
+                )
+
+        access_token = service.create(
+            name=data["name"],
+            user_id=user_id,
+            ttl=data["ttl"],
+            revoked=False,
+            issued_at=int(datetime.utcnow().timestamp()),
+        )
+        return dict(
+            jwt=create_token(access_token.user_id, access_token.id, access_token.ttl)
+        )
+
+
+class ApiKeys(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(ApiKeys, self).__init__()
+
+    @validate_schema(None, api_key_output_schema)
+    def get(self, aid):
+        """
+        .. http:get:: /keys/1
+
+           Fetch one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    @validate_schema(api_key_revoke_schema, api_key_output_schema)
+    def put(self, aid, data=None):
+        """
+        .. http:put:: /keys/1
+
+           update one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              PUT /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                  "name": "new_name",
+                  "revoked": false,
+                  "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to update this token!"), 403
+
+        service.update(
+            access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
+        )
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    def delete(self, aid):
+        """
+        .. http:delete:: /keys/1
+
+           deletes one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              DELETE /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "result": true
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to delete this token!"), 403
+
+        service.delete(access_key)
+        return {"result": True}
+
+
+class UserApiKeys(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(UserApiKeys, self).__init__()
+
+    @validate_schema(None, api_key_output_schema)
+    def get(self, uid, aid):
+        """
+        .. http:get:: /users/1/keys/1
+
+           Fetch one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /users/1/api_keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if uid != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        access_key = service.get(aid)
+
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != uid:
+            return dict(message="You are not authorized to view this token!"), 403
+
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    @validate_schema(api_key_revoke_schema, api_key_output_schema)
+    def put(self, uid, aid, data=None):
+        """
+        .. http:put:: /users/1/keys/1
+
+           update one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              PUT /users/1/keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
+
+              {
+                  "name": "new_name",
+                  "revoked": false,
+                  "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if uid != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != uid:
+            return dict(message="You are not authorized to update this token!"), 403
+
+        service.update(
+            access_key, name=data["name"], revoked=data["revoked"], ttl=data["ttl"]
+        )
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    def delete(self, uid, aid):
+        """
+        .. http:delete:: /users/1/keys/1
+
+           deletes one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              DELETE /users/1/keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "result": true
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if uid != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != uid:
+            return dict(message="You are not authorized to delete this token!"), 403
+
+        service.delete(access_key)
+        return {"result": True}
+
+
+class ApiKeysDescribed(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(ApiKeysDescribed, self).__init__()
+
+    @validate_schema(None, api_key_described_output_schema)
+    def get(self, aid):
+        """
+        .. http:get:: /keys/1/described
+
+           Fetch one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "id": 2,
+                  "name": "hoi",
+                  "user_id": 2,
+                  "ttl": -1,
+                  "issued_at": 1222222,
+                  "revoked": false
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        return access_key
+
+
+api.add_resource(ApiKeyList, "/keys", endpoint="api_keys")
+api.add_resource(ApiKeys, "/keys/<int:aid>", endpoint="api_key")
+api.add_resource(
+    ApiKeysDescribed, "/keys/<int:aid>/described", endpoint="api_key_described"
+)
+api.add_resource(ApiKeyUserList, "/users/<int:user_id>/keys", endpoint="user_api_keys")
+api.add_resource(
+    UserApiKeys, "/users/<int:uid>/keys/<int:aid>", endpoint="user_api_key"
+)

lemur/auth/ldap.py

@@ -0,0 +1,230 @@
+"""
+.. module: lemur.auth.ldap
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Ian Stahnke <ian.stahnke@myob.com>
+"""
+import ldap
+
+from flask import current_app
+
+from lemur.users import service as user_service
+from lemur.roles import service as role_service
+from lemur.common.utils import validate_conf, get_psuedo_random_string
+
+
+class LdapPrincipal:
+    """
+    Provides methods for authenticating against an LDAP server.
+    """
+
+    def __init__(self, args):
+        self._ldap_validate_conf()
+        # setup ldap config
+        if not args["username"]:
+            raise Exception("missing ldap username")
+        if not args["password"]:
+            self.error_message = "missing ldap password"
+            raise Exception("missing ldap password")
+        self.ldap_principal = args["username"]
+        self.ldap_email_domain = current_app.config.get("LDAP_EMAIL_DOMAIN", None)
+        if "@" not in self.ldap_principal:
+            self.ldap_principal = "%s@%s" % (
+                self.ldap_principal,
+                self.ldap_email_domain,
+            )
+        self.ldap_username = args["username"]
+        if "@" in self.ldap_username:
+            self.ldap_username = args["username"].split("@")[0]
+        self.ldap_password = args["password"]
+        self.ldap_server = current_app.config.get("LDAP_BIND_URI", None)
+        self.ldap_base_dn = current_app.config.get("LDAP_BASE_DN", None)
+        self.ldap_use_tls = current_app.config.get("LDAP_USE_TLS", False)
+        self.ldap_cacert_file = current_app.config.get("LDAP_CACERT_FILE", None)
+        self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None)
+        self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None)
+        self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None)
+        self.ldap_is_active_directory = current_app.config.get(
+            "LDAP_IS_ACTIVE_DIRECTORY", False
+        )
+        self.ldap_attrs = ["memberOf"]
+        self.ldap_client = None
+        self.ldap_groups = None
+
+    def _update_user(self, roles):
+        """
+        create or update a local user instance.
+        """
+        # try to get user from local database
+        user = user_service.get_by_email(self.ldap_principal)
+
+        # create them a local account
+        if not user:
+            user = user_service.create(
+                self.ldap_username,
+                get_psuedo_random_string(),
+                self.ldap_principal,
+                True,
+                "",  # thumbnailPhotoUrl
+                list(roles),
+            )
+        else:
+            # we add 'lemur' specific roles, so they do not get marked as removed
+            for ur in user.roles:
+                if not ur.third_party:
+                    roles.add(ur)
+
+            # update any changes to the user
+            user_service.update(
+                user.id,
+                self.ldap_username,
+                self.ldap_principal,
+                user.active,
+                user.profile_picture,
+                list(roles),
+            )
+        return user
+
+    def _authorize(self):
+        """
+        check groups and roles to confirm access.
+        return a list of roles if ok.
+        raise an exception on error.
+        """
+        if not self.ldap_principal:
+            return None
+
+        if self.ldap_required_group:
+            # ensure the user has the required group in their group list
+            if self.ldap_required_group not in self.ldap_groups:
+                return None
+
+        roles = set()
+        if self.ldap_default_role:
+            role = role_service.get_by_name(self.ldap_default_role)
+            if role:
+                if not role.third_party:
+                    role = role_service.set_third_party(role.id, third_party_status=True)
+                roles.add(role)
+
+        # update their 'roles'
+        role = role_service.get_by_name(self.ldap_principal)
+        if not role:
+            description = "auto generated role based on owner: {0}".format(
+                self.ldap_principal
+            )
+            role = role_service.create(
+                self.ldap_principal, description=description, third_party=True
+            )
+        if not role.third_party:
+            role = role_service.set_third_party(role.id, third_party_status=True)
+        roles.add(role)
+        if not self.ldap_groups_to_roles:
+            return roles
+
+        for ldap_group_name, role_name in self.ldap_groups_to_roles.items():
+            role = role_service.get_by_name(role_name)
+            if role:
+                if ldap_group_name in self.ldap_groups:
+                    current_app.logger.debug(
+                        "assigning role {0} to ldap user {1}".format(
+                            self.ldap_principal, role
+                        )
+                    )
+                    if not role.third_party:
+                        role = role_service.set_third_party(
+                            role.id, third_party_status=True
+                        )
+                    roles.add(role)
+        return roles
+
+    def authenticate(self):
+        """
+        orchestrate the ldap login.
+        raise an exception on error.
+        """
+        self._bind()
+        roles = self._authorize()
+        if not roles:
+            raise Exception("ldap authorization failed")
+        return self._update_user(roles)
+
+    def _bind(self):
+        """
+        authenticate an ldap user.
+        list groups for a user.
+        raise an exception on error.
+        """
+        if "@" not in self.ldap_principal:
+            self.ldap_principal = "%s@%s" % (
+                self.ldap_principal,
+                self.ldap_email_domain,
+            )
+        ldap_filter = "userPrincipalName=%s" % self.ldap_principal
+
+        # query ldap for auth
+        try:
+            # build a client
+            if not self.ldap_client:
+                self.ldap_client = ldap.initialize(self.ldap_server)
+            # perform a synchronous bind
+            self.ldap_client.set_option(ldap.OPT_REFERRALS, 0)
+            if self.ldap_use_tls:
+                ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+                self.ldap_client.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
+                self.ldap_client.set_option(ldap.OPT_X_TLS, ldap.OPT_X_TLS_DEMAND)
+                self.ldap_client.set_option(ldap.OPT_X_TLS_DEMAND, True)
+                self.ldap_client.set_option(ldap.OPT_DEBUG_LEVEL, 255)
+            if self.ldap_cacert_file:
+                self.ldap_client.set_option(
+                    ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file
+                )
+            self.ldap_client.simple_bind_s(self.ldap_principal, self.ldap_password)
+        except ldap.INVALID_CREDENTIALS:
+            self.ldap_client.unbind()
+            raise Exception("The supplied ldap credentials are invalid")
+        except ldap.SERVER_DOWN:
+            raise Exception("ldap server unavailable")
+        except ldap.LDAPError as e:
+            raise Exception("ldap error: {0}".format(e))
+
+        if self.ldap_is_active_directory:
+            # Lookup user DN, needed to search for group membership
+            userdn = self.ldap_client.search_s(
+                self.ldap_base_dn,
+                ldap.SCOPE_SUBTREE,
+                ldap_filter,
+                ["distinguishedName"],
+            )[0][1]["distinguishedName"][0]
+            userdn = userdn.decode("utf-8")
+            # Search all groups that have the userDN as a member
+            groupfilter = "(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))".format(
+                userdn
+            )
+            lgroups = self.ldap_client.search_s(
+                self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ["cn"]
+            )
+
+            # Create a list of group CN's from the result
+            self.ldap_groups = []
+            for group in lgroups:
+                (dn, values) = group
+                if type(values) == dict:
+                    self.ldap_groups.append(values["cn"][0].decode("utf-8"))
+        else:
+            lgroups = self.ldap_client.search_s(
+                self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs
+            )[0][1]["memberOf"]
+            # lgroups is a list of utf-8 encoded strings
+            # convert to a single string of groups to allow matching
+            self.ldap_groups = b"".join(lgroups).decode("ascii")
+
+        self.ldap_client.unbind()
+
+    def _ldap_validate_conf(self):
+        """
+        Confirms required ldap config settings exist.
+        """
+        required_vars = ["LDAP_BIND_URI", "LDAP_BASE_DN", "LDAP_EMAIL_DOMAIN"]
+        validate_conf(current_app, required_vars)

lemur/auth/permissions.py

@@ -2,55 +2,73 @@
 .. module: lemur.auth.permissions
     :platform: Unix
     :synopsis: This module defines all the permission used within Lemur
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from functools import partial
 from collections import namedtuple
 
-from flask.ext.principal import Permission, RoleNeed
+from flask import current_app
+from flask_principal import Permission, RoleNeed
 
 # Permissions
-operator_permission = Permission(RoleNeed('operator'))
-admin_permission = Permission(RoleNeed('admin'))
+operator_permission = Permission(RoleNeed("operator"))
+admin_permission = Permission(RoleNeed("admin"))
 
-CertificateCreator = namedtuple('certificate', ['method', 'value'])
-CertificateCreatorNeed = partial(CertificateCreator, 'key')
+CertificateOwner = namedtuple("certificate", ["method", "value"])
+CertificateOwnerNeed = partial(CertificateOwner, "role")
 
 
-class ViewKeyPermission(Permission):
-    def __init__(self, certificate_id, owner):
-        c_need = CertificateCreatorNeed(certificate_id)
-        super(ViewKeyPermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
+class SensitiveDomainPermission(Permission):
+    def __init__(self):
+        needs = [RoleNeed("admin")]
+        sensitive_domain_roles = current_app.config.get("SENSITIVE_DOMAIN_ROLES", [])
+
+        if sensitive_domain_roles:
+            for role in sensitive_domain_roles:
+                needs.append(RoleNeed(role))
+
+        super(SensitiveDomainPermission, self).__init__(*needs)
 
 
-class UpdateCertificatePermission(Permission):
-    def __init__(self, certificate_id, owner):
-        c_need = CertificateCreatorNeed(certificate_id)
-        super(UpdateCertificatePermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
+class CertificatePermission(Permission):
+    def __init__(self, owner, roles):
+        needs = [RoleNeed("admin"), RoleNeed(owner), RoleNeed("creator")]
+        for r in roles:
+            needs.append(CertificateOwnerNeed(str(r)))
+            # Backwards compatibility with mixed-case role names
+            if str(r) != str(r).lower():
+                needs.append(CertificateOwnerNeed(str(r).lower()))
+
+        super(CertificatePermission, self).__init__(*needs)
 
 
-RoleUser = namedtuple('role', ['method', 'value'])
-ViewRoleCredentialsNeed = partial(RoleUser, 'roleView')
+class ApiKeyCreatorPermission(Permission):
+    def __init__(self):
+        super(ApiKeyCreatorPermission, self).__init__(RoleNeed("admin"))
 
 
-class ViewRoleCredentialsPermission(Permission):
+RoleMember = namedtuple("role", ["method", "value"])
+RoleMemberNeed = partial(RoleMember, "member")
+
+
+class RoleMemberPermission(Permission):
     def __init__(self, role_id):
-        need = ViewRoleCredentialsNeed(role_id)
-        super(ViewRoleCredentialsPermission, self).__init__(need, RoleNeed('admin'))
+        needs = [RoleNeed("admin"), RoleMemberNeed(role_id)]
+        super(RoleMemberPermission, self).__init__(*needs)
 
 
-AuthorityCreator = namedtuple('authority', ['method', 'value'])
-AuthorityCreatorNeed = partial(AuthorityCreator, 'authorityUse')
+AuthorityCreator = namedtuple("authority", ["method", "value"])
+AuthorityCreatorNeed = partial(AuthorityCreator, "authorityUse")
 
-AuthorityOwner = namedtuple('authority', ['method', 'value'])
-AuthorityOwnerNeed = partial(AuthorityOwner, 'role')
+AuthorityOwner = namedtuple("authority", ["method", "value"])
+AuthorityOwnerNeed = partial(AuthorityOwner, "role")
 
 
 class AuthorityPermission(Permission):
     def __init__(self, authority_id, roles):
-        needs = [RoleNeed('admin'), AuthorityCreatorNeed(str(authority_id))]
+        needs = [RoleNeed("admin"), AuthorityCreatorNeed(str(authority_id))]
         for r in roles:
             needs.append(AuthorityOwnerNeed(str(r)))
 

lemur/auth/service.py

@@ -3,16 +3,13 @@
     :platform: Unix
     :synopsis: This module contains all of the authentication duties for
     lemur
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from __future__ import unicode_literals
-from builtins import bytes
 import jwt
 import json
-import base64
 import binascii
 
 from functools import wraps
@@ -20,31 +17,18 @@ from datetime import datetime, timedelta
 
 from flask import g, current_app, jsonify, request
 
-from flask.ext.restful import Resource
-from flask.ext.principal import identity_loaded, RoleNeed, UserNeed
+from flask_restful import Resource
+from flask_principal import identity_loaded, RoleNeed, UserNeed
 
-from flask.ext.principal import Identity, identity_changed
+from flask_principal import Identity, identity_changed
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicNumbers
 
 from lemur.users import service as user_service
-from lemur.auth.permissions import CertificateCreatorNeed, \
-    AuthorityCreatorNeed, ViewRoleCredentialsNeed
-
-
-def base64url_decode(data):
-    rem = len(data) % 4
-
-    if rem > 0:
-        data += '=' * (4 - rem)
-
-    return base64.urlsafe_b64decode(bytes(data.encode('latin-1')))
-
-
-def base64url_encode(data):
-    return base64.urlsafe_b64encode(data).replace('=', '')
+from lemur.api_keys import service as api_key_service
+from lemur.auth.permissions import AuthorityCreatorNeed, RoleMemberNeed
 
 
 def get_rsa_public_key(n, e):
@@ -55,68 +39,102 @@ def get_rsa_public_key(n, e):
     :param e:
     :return: a RSA Public Key in PEM format
     """
-    n = int(binascii.hexlify(base64url_decode(n)), 16)
-    e = int(binascii.hexlify(base64url_decode(e)), 16)
+    n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, "utf-8"))), 16)
+    e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, "utf-8"))), 16)
+
     pub = RSAPublicNumbers(e, n).public_key(default_backend())
     return pub.public_bytes(
         encoding=serialization.Encoding.PEM,
-        format=serialization.PublicFormat.SubjectPublicKeyInfo
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
     )
 
 
-def create_token(user):
+def create_token(user, aid=None, ttl=None):
     """
-    Create a valid JWT for a given user, this token is then used to authenticate
+    Create a valid JWT for a given user/api key, this token is then used to authenticate
     sessions until the token expires.
 
     :param user:
     :return:
     """
-    expiration_delta = timedelta(days=int(current_app.config.get('LEMUR_TOKEN_EXPIRATION', 1)))
-    payload = {
-        'sub': user.id,
-        'iat': datetime.now(),
-        'exp': datetime.now() + expiration_delta
-    }
-    token = jwt.encode(payload, current_app.config['LEMUR_TOKEN_SECRET'])
-    return token.decode('unicode_escape')
+    expiration_delta = timedelta(
+        days=int(current_app.config.get("LEMUR_TOKEN_EXPIRATION", 1))
+    )
+    payload = {"iat": datetime.utcnow(), "exp": datetime.utcnow() + expiration_delta}
+
+    # Handle Just a User ID & User Object.
+    if isinstance(user, int):
+        payload["sub"] = user
+    else:
+        payload["sub"] = user.id
+    if aid is not None:
+        payload["aid"] = aid
+    # Custom TTLs are only supported on Access Keys.
+    if ttl is not None and aid is not None:
+        # Tokens that are forever until revoked.
+        if ttl == -1:
+            del payload["exp"]
+        else:
+            payload["exp"] = datetime.utcnow() + timedelta(days=ttl)
+    token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"])
+    return token
 
 
 def login_required(f):
     """
-    Validates the JWT and ensures that is has not expired.
+    Validates the JWT and ensures that is has not expired and the user is still active.
 
     :param f:
     :return:
     """
+
     @wraps(f)
     def decorated_function(*args, **kwargs):
-        if not request.headers.get('Authorization'):
-            response = jsonify(message='Missing authorization header')
+        if not request.headers.get("Authorization"):
+            response = jsonify(message="Missing authorization header")
             response.status_code = 401
             return response
 
         try:
-            token = request.headers.get('Authorization').split()[1]
+            token = request.headers.get("Authorization").split()[1]
         except Exception as e:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
 
         try:
-            payload = jwt.decode(token, current_app.config['LEMUR_TOKEN_SECRET'])
+            header_data = fetch_token_header(token)
+            payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"], algorithms=[header_data["alg"]])
         except jwt.DecodeError:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
         except jwt.ExpiredSignatureError:
-            return dict(message='Token has expired'), 403
+            return dict(message="Token has expired"), 403
         except jwt.InvalidTokenError:
-            return dict(message='Token is invalid'), 403
+            return dict(message="Token is invalid"), 403
 
-        g.current_user = user_service.get(payload['sub'])
+        if "aid" in payload:
+            access_key = api_key_service.get(payload["aid"])
+            if access_key.revoked:
+                return dict(message="Token has been revoked"), 403
+            if access_key.ttl != -1:
+                current_time = datetime.utcnow()
+                # API key uses days
+                expired_time = datetime.fromtimestamp(access_key.issued_at) + timedelta(days=access_key.ttl)
+                if current_time >= expired_time:
+                    return dict(message="Token has expired"), 403
+
+        user = user_service.get(payload["sub"])
+
+        if not user.active:
+            return dict(message="User is not currently active"), 403
+
+        g.current_user = user
 
         if not g.current_user:
-            return dict(message='You are not logged in'), 403
+            return dict(message="You are not logged in"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(g.current_user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(g.current_user.id)
+        )
 
         return f(*args, **kwargs)
 
@@ -130,21 +148,18 @@ def fetch_token_header(token):
     :param token:
     :return: :raise jwt.DecodeError:
     """
-    token = token.encode('utf-8')
+    token = token.encode("utf-8")
     try:
-        signing_input, crypto_segment = token.rsplit(b'.', 1)
-        header_segment, payload_segment = signing_input.split(b'.', 1)
+        signing_input, crypto_segment = token.rsplit(b".", 1)
+        header_segment, payload_segment = signing_input.split(b".", 1)
     except ValueError:
-        raise jwt.DecodeError('Not enough segments')
+        raise jwt.DecodeError("Not enough segments")
 
     try:
-        return json.loads(base64url_decode(header_segment))
+        return json.loads(jwt.utils.base64url_decode(header_segment).decode("utf-8"))
     except TypeError as e:
         current_app.logger.exception(e)
-        raise jwt.DecodeError('Invalid header padding')
-    except binascii.Error as e:
-        current_app.logger.exception(e)
-        raise jwt.DecodeError('Invalid header padding')
+        raise jwt.DecodeError("Invalid header padding")
 
 
 @identity_loaded.connect
@@ -163,21 +178,16 @@ def on_identity_loaded(sender, identity):
     identity.provides.add(UserNeed(identity.id))
 
     # identity with the roles that the user provides
-    if hasattr(user, 'roles'):
+    if hasattr(user, "roles"):
         for role in user.roles:
-            identity.provides.add(ViewRoleCredentialsNeed(role.id))
             identity.provides.add(RoleNeed(role.name))
+            identity.provides.add(RoleMemberNeed(role.id))
 
     # apply ownership for authorities
-    if hasattr(user, 'authorities'):
+    if hasattr(user, "authorities"):
         for authority in user.authorities:
             identity.provides.add(AuthorityCreatorNeed(authority.id))
 
-    # apply ownership of certificates
-    if hasattr(user, 'certificates'):
-        for certificate in user.certificates:
-            identity.provides.add(CertificateCreatorNeed(certificate.id))
-
     g.user = user
 
 
@@ -185,6 +195,7 @@ class AuthenticatedResource(Resource):
     """
     Inherited by all resources that need to be protected by authentication.
     """
+
     method_decorators = [login_required]
 
     def __init__(self):

lemur/auth/views.py

@@ -1,30 +1,281 @@
 """
 .. module: lemur.auth.views
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import json
+
 import jwt
 import base64
 import requests
 
-from flask import g, Blueprint, current_app
+from flask import Blueprint, current_app
 
-from flask.ext.restful import reqparse, Resource, Api
-from flask.ext.principal import Identity, identity_changed
+from flask_restful import reqparse, Resource, Api
+from flask_principal import Identity, identity_changed
 
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
+from lemur.extensions import metrics
 from lemur.common.utils import get_psuedo_random_string
 
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
+from lemur.logs import service as log_service
 from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key
+from lemur.auth import ldap
+from lemur.plugins.base import plugins
 
-
-mod = Blueprint('auth', __name__)
+mod = Blueprint("auth", __name__)
 api = Api(mod)
 
 
+def exchange_for_access_token(
+    code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True
+):
+    """
+    Exchanges authorization code for access token.
+
+    :param code:
+    :param redirect_uri:
+    :param client_id:
+    :param secret:
+    :param access_token_url:
+    :param verify_cert:
+    :return:
+    :return:
+    """
+    # take the information we have received from the provider to create a new request
+    params = {
+        "grant_type": "authorization_code",
+        "scope": "openid email profile address",
+        "code": code,
+        "redirect_uri": redirect_uri,
+        "client_id": client_id,
+    }
+
+    # the secret and cliendId will be given to you when you signup for the provider
+    token = "{0}:{1}".format(client_id, secret)
+
+    basic = base64.b64encode(bytes(token, "utf-8"))
+    headers = {
+        "Content-Type": "application/x-www-form-urlencoded",
+        "authorization": "basic {0}".format(basic.decode("utf-8")),
+    }
+
+    # exchange authorization code for access token.
+    r = requests.post(
+        access_token_url, headers=headers, params=params, verify=verify_cert
+    )
+    if r.status_code == 400:
+        r = requests.post(
+            access_token_url, headers=headers, data=params, verify=verify_cert
+        )
+    id_token = r.json()["id_token"]
+    access_token = r.json()["access_token"]
+
+    return id_token, access_token
+
+
+def validate_id_token(id_token, client_id, jwks_url):
+    """
+    Ensures that the token we receive is valid.
+
+    :param id_token:
+    :param client_id:
+    :param jwks_url:
+    :return:
+    """
+    # fetch token public key
+    header_data = fetch_token_header(id_token)
+
+    # retrieve the key material as specified by the token header
+    r = requests.get(jwks_url)
+    for key in r.json()["keys"]:
+        if key["kid"] == header_data["kid"]:
+            secret = get_rsa_public_key(key["n"], key["e"])
+            algo = header_data["alg"]
+            break
+    else:
+        return dict(message="Key not found"), 401
+
+    # validate your token based on the key it was signed with
+    try:
+        jwt.decode(
+            id_token, secret.decode("utf-8"), algorithms=[algo], audience=client_id
+        )
+    except jwt.DecodeError:
+        return dict(message="Token is invalid"), 401
+    except jwt.ExpiredSignatureError:
+        return dict(message="Token has expired"), 401
+    except jwt.InvalidTokenError:
+        return dict(message="Token is invalid"), 401
+
+
+def retrieve_user(user_api_url, access_token):
+    """
+    Fetch user information from provided user api_url.
+
+    :param user_api_url:
+    :param access_token:
+    :return:
+    """
+    user_params = dict(access_token=access_token, schema="profile")
+
+    headers = {}
+
+    if current_app.config.get("PING_INCLUDE_BEARER_TOKEN"):
+        headers = {"Authorization": f"Bearer {access_token}"}
+
+    # retrieve information about the current user.
+    r = requests.get(user_api_url, params=user_params, headers=headers)
+    # Some IDPs, like "Keycloak", require a POST instead of a GET
+    if r.status_code == 400:
+        r = requests.post(user_api_url, data=user_params, headers=headers)
+
+    profile = r.json()
+
+    user = user_service.get_by_email(profile["email"])
+    return user, profile
+
+
+def retrieve_user_memberships(user_api_url, user_membership_api_url, access_token):
+    user, profile = retrieve_user(user_api_url, access_token)
+
+    if user_membership_api_url is None:
+        return user, profile
+    """
+    Potentially, below code can be made more generic i.e., plugin driven. Unaware of the usage of this
+    code across the community, current implementation is config driven. Without user_membership_api_url
+    configured, it is backward compatible.
+    """
+    tls_provider = plugins.get(current_app.config.get("PING_USER_MEMBERSHIP_TLS_PROVIDER"))
+
+    # put user id in url
+    user_membership_api_url = user_membership_api_url.replace("%user_id%", profile["userId"])
+
+    session = tls_provider.session(current_app.config.get("PING_USER_MEMBERSHIP_SERVICE"))
+    headers = {"Content-Type": "application/json"}
+    data = {"relation": "DIRECT_ONLY", "groupFilter": {"type": "GOOGLE"}, "size": 500}
+    user_membership = {"email": profile["email"],
+                       "thumbnailPhotoUrl": profile["thumbnailPhotoUrl"],
+                       "googleGroups": []}
+    while True:
+        # retrieve information about the current user memberships
+        r = session.post(user_membership_api_url, data=json.dumps(data), headers=headers)
+
+        if r.status_code == 200:
+            response = r.json()
+            membership_details = response["data"]
+            for membership in membership_details:
+                user_membership["googleGroups"].append(membership["membership"]["name"])
+
+            if "nextPageToken" in response and response["nextPageToken"]:
+                data["nextPageToken"] = response["nextPageToken"]
+            else:
+                break
+        else:
+            current_app.logger.error(f"Response Code:{r.status_code} {r.text}")
+            break
+    return user, user_membership
+
+
+def create_user_roles(profile):
+    """Creates new roles based on profile information.
+
+    :param profile:
+    :return:
+    """
+    roles = []
+
+    # update their google 'roles'
+    if "googleGroups" in profile:
+        for group in profile["googleGroups"]:
+            role = role_service.get_by_name(group)
+            if not role:
+                role = role_service.create(
+                    group,
+                    description="This is a google group based role created by Lemur",
+                    third_party=True,
+                )
+            if (group != 'admin') and (not role.third_party):
+                role = role_service.set_third_party(role.id, third_party_status=True)
+            roles.append(role)
+    else:
+        current_app.logger.warning(
+            "'googleGroups' not sent by identity provider, no specific roles will assigned to the user."
+        )
+
+    role = role_service.get_by_name(profile["email"])
+
+    if not role:
+        role = role_service.create(
+            profile["email"],
+            description="This is a user specific role",
+            third_party=True,
+        )
+    if not role.third_party:
+        role = role_service.set_third_party(role.id, third_party_status=True)
+
+    roles.append(role)
+
+    # every user is an operator (tied to a default role)
+    if current_app.config.get("LEMUR_DEFAULT_ROLE"):
+        default = role_service.get_by_name(current_app.config["LEMUR_DEFAULT_ROLE"])
+        if not default:
+            default = role_service.create(
+                current_app.config["LEMUR_DEFAULT_ROLE"],
+                description="This is the default Lemur role.",
+            )
+        if not default.third_party:
+            role_service.set_third_party(default.id, third_party_status=True)
+        roles.append(default)
+
+    return roles
+
+
+def update_user(user, profile, roles):
+    """Updates user with current profile information and associated roles.
+
+    :param user:
+    :param profile:
+    :param roles:
+    """
+    # if we get an sso user create them an account
+    if not user:
+        user = user_service.create(
+            profile["email"],
+            get_psuedo_random_string(),
+            profile["email"],
+            True,
+            profile.get("thumbnailPhotoUrl"),
+            roles,
+        )
+
+    else:
+        # we add 'lemur' specific roles, so they do not get marked as removed
+        removed_roles = []
+        for ur in user.roles:
+            if not ur.third_party:
+                roles.append(ur)
+            elif ur not in roles:
+                # This is a role assigned in lemur, but not returned by sso during current login
+                removed_roles.append(ur.name)
+
+        if removed_roles:
+            log_service.audit_log("unassign_role", user.username, f"Un-assigning roles {removed_roles}")
+        # update any changes to the user
+        user_service.update(
+            user.id,
+            profile["email"],
+            profile["email"],
+            True,
+            profile.get("thumbnailPhotoUrl"),  # profile isn't google+ enabled
+            roles,
+        )
+
+
 class Login(Resource):
     """
     Provides an endpoint for Lemur's basic authentication. It takes a username and password
@@ -42,6 +293,7 @@ class Login(Resource):
     on your uses cases but. It is important to not that there is currently no build in method to revoke a users token \
     and force re-authentication.
     """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Login, self).__init__()
@@ -59,6 +311,7 @@ class Login(Resource):
               POST /auth/login HTTP/1.1
               Host: example.com
               Accept: application/json, text/javascript
+              Content-Type: application/json;charset=UTF-8
 
               {
                 "username": "test",
@@ -82,153 +335,303 @@ class Login(Resource):
            :statuscode 401: invalid credentials
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('username', type=str, required=True, location='json')
-        self.reqparse.add_argument('password', type=str, required=True, location='json')
+        self.reqparse.add_argument("username", type=str, required=True, location="json")
+        self.reqparse.add_argument("password", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
-        if '@' in args['username']:
-            user = user_service.get_by_email(args['username'])
+        if "@" in args["username"]:
+            user = user_service.get_by_email(args["username"])
         else:
-            user = user_service.get_by_username(args['username'])
+            user = user_service.get_by_username(args["username"])
 
-        if user and user.check_password(args['password']):
+        # default to local authentication
+        if user and user.check_password(args["password"]) and user.active:
             # Tell Flask-Principal the identity changed
-            identity_changed.send(current_app._get_current_object(),
-                                  identity=Identity(user.id))
+            identity_changed.send(
+                current_app._get_current_object(), identity=Identity(user.id)
+            )
+
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+            )
             return dict(token=create_token(user))
 
-        return dict(message='The supplied credentials are invalid'), 401
+        # try ldap login
+        if current_app.config.get("LDAP_AUTH"):
+            try:
+                ldap_principal = ldap.LdapPrincipal(args)
+                user = ldap_principal.authenticate()
+                if user and user.active:
+                    # Tell Flask-Principal the identity changed
+                    identity_changed.send(
+                        current_app._get_current_object(), identity=Identity(user.id)
+                    )
+                    metrics.send(
+                        "login",
+                        "counter",
+                        1,
+                        metric_tags={"status": SUCCESS_METRIC_STATUS},
+                    )
+                    return dict(token=create_token(user))
+            except Exception as e:
+                current_app.logger.error("ldap error: {0}".format(e))
+                ldap_message = "ldap error: %s" % e
+                metrics.send(
+                    "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+                )
+                return dict(message=ldap_message), 403
 
-    def get(self):
-        return {'username': g.current_user.username, 'roles': [r.name for r in g.current_user.roles]}
+        # if not valid user - no certificates for you
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+        )
+        return dict(message="The supplied credentials are invalid"), 403
 
 
 class Ping(Resource):
     """
     This class serves as an example of how one might implement an SSO provider for use with Lemur. In
-    this example we use a OpenIDConnect authentication flow, that is essentially OAuth2 underneath. If you have an
+    this example we use an OpenIDConnect authentication flow, that is essentially OAuth2 underneath. If you have an
     OAuth2 provider you want to use Lemur there would be two steps:
 
-    1. Define your own class that inherits from :class:`flask.ext.restful.Resource` and create the HTTP methods the \
-    provider uses for it's callbacks.
+    1. Define your own class that inherits from :class:`flask_restful.Resource` and create the HTTP methods the \
+    provider uses for its callbacks.
     2. Add or change the Lemur AngularJS Configuration to point to your new provider
     """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Ping, self).__init__()
 
+    def get(self):
+        return "Redirecting..."
+
     def post(self):
-        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
-        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
-        self.reqparse.add_argument('code', type=str, required=True, location='json')
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
 
         args = self.reqparse.parse_args()
 
-        # take the information we have received from the provider to create a new request
-        params = {
-            'client_id': args['clientId'],
-            'grant_type': 'authorization_code',
-            'scope': 'openid email profile address',
-            'redirect_uri': args['redirectUri'],
-            'code': args['code']
-        }
-
         # you can either discover these dynamically or simply configure them
-        access_token_url = current_app.config.get('PING_ACCESS_TOKEN_URL')
-        user_api_url = current_app.config.get('PING_USER_API_URL')
+        access_token_url = current_app.config.get("PING_ACCESS_TOKEN_URL")
 
-        # the secret and cliendId will be given to you when you signup for the provider
-        basic = base64.b64encode('{0}:{1}'.format(args['clientId'], current_app.config.get("PING_SECRET")))
-        headers = {'Authorization': 'Basic {0}'.format(basic)}
+        secret = current_app.config.get("PING_SECRET")
 
-        # exchange authorization code for access token.
+        id_token, access_token = exchange_for_access_token(
+            args["code"],
+            args["redirectUri"],
+            args["clientId"],
+            secret,
+            access_token_url=access_token_url,
+        )
 
-        r = requests.post(access_token_url, headers=headers, params=params)
-        id_token = r.json()['id_token']
-        access_token = r.json()['access_token']
+        jwks_url = current_app.config.get("PING_JWKS_URL")
+        error_code = validate_id_token(id_token, args["clientId"], jwks_url)
+        if error_code:
+            return error_code
 
-        # fetch token public key
-        header_data = fetch_token_header(id_token)
-        jwks_url = current_app.config.get('PING_JWKS_URL')
+        user, profile = retrieve_user_memberships(
+            current_app.config.get("PING_USER_API_URL"),
+            current_app.config.get("PING_USER_MEMBERSHIP_URL"),
+            access_token
+        )
+        roles = create_user_roles(profile)
+        update_user(user, profile, roles)
 
-        # retrieve the key material as specified by the token header
-        r = requests.get(jwks_url)
-        for key in r.json()['keys']:
-            if key['kid'] == header_data['kid']:
-                secret = get_rsa_public_key(key['n'], key['e'])
-                algo = header_data['alg']
-                break
-        else:
-            return dict(message='Key not found'), 403
-
-        # validate your token based on the key it was signed with
-        try:
-            jwt.decode(id_token, secret, algorithms=[algo], audience=args['clientId'])
-        except jwt.DecodeError:
-            return dict(message='Token is invalid'), 403
-        except jwt.ExpiredSignatureError:
-            return dict(message='Token has expired'), 403
-        except jwt.InvalidTokenError:
-            return dict(message='Token is invalid'), 403
-
-        user_params = dict(access_token=access_token, schema='profile')
-
-        # retrieve information about the current user.
-        r = requests.get(user_api_url, params=user_params)
-        profile = r.json()
-
-        user = user_service.get_by_email(profile['email'])
-
-        # update their google 'roles'
-        roles = []
-
-        for group in profile['googleGroups']:
-            role = role_service.get_by_name(group)
-            if not role:
-                role = role_service.create(group, description='This is a google group based role created by Lemur')
-            roles.append(role)
-
-        # if we get an sso user create them an account
-        # we still pick a random password in case sso is down
-        if not user:
-
-            # every user is an operator (tied to a default role)
-            if current_app.config.get('LEMUR_DEFAULT_ROLE'):
-                v = role_service.get_by_name(current_app.config.get('LEMUR_DEFAULT_ROLE'))
-                if v:
-                    roles.append(v)
-
-            user = user_service.create(
-                profile['email'],
-                get_psuedo_random_string(),
-                profile['email'],
-                True,
-                profile.get('thumbnailPhotoUrl'),
-                roles
-            )
-
-        else:
-            # we add 'lemur' specific roles, so they do not get marked as removed
-            for ur in user.roles:
-                if ur.authority_id:
-                    roles.append(ur)
-
-            # update any changes to the user
-            user_service.update(
-                user.id,
-                profile['email'],
-                profile['email'],
-                True,
-                profile.get('thumbnailPhotoUrl'),  # incase profile isn't google+ enabled
-                roles
+        if not user or not user.active:
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
             )
+            return dict(message="The supplied credentials are invalid"), 403
 
         # Tell Flask-Principal the identity changed
-        identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(user.id)
+        )
+
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+        )
+        return dict(token=create_token(user))
+
+
+class OAuth2(Resource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(OAuth2, self).__init__()
+
+    def get(self):
+        return "Redirecting..."
+
+    def post(self):
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
+
+        args = self.reqparse.parse_args()
+
+        # you can either discover these dynamically or simply configure them
+        access_token_url = current_app.config.get("OAUTH2_ACCESS_TOKEN_URL")
+        user_api_url = current_app.config.get("OAUTH2_USER_API_URL")
+        verify_cert = current_app.config.get("OAUTH2_VERIFY_CERT")
+
+        secret = current_app.config.get("OAUTH2_SECRET")
+
+        id_token, access_token = exchange_for_access_token(
+            args["code"],
+            args["redirectUri"],
+            args["clientId"],
+            secret,
+            access_token_url=access_token_url,
+            verify_cert=verify_cert,
+        )
+
+        jwks_url = current_app.config.get("OAUTH2_JWKS_URL")
+        error_code = validate_id_token(id_token, args["clientId"], jwks_url)
+        if error_code:
+            return error_code
+
+        user, profile = retrieve_user(user_api_url, access_token)
+        roles = create_user_roles(profile)
+        update_user(user, profile, roles)
+
+        if not user.active:
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid"), 403
+
+        # Tell Flask-Principal the identity changed
+        identity_changed.send(
+            current_app._get_current_object(), identity=Identity(user.id)
+        )
+
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+        )
 
         return dict(token=create_token(user))
 
 
-api.add_resource(Login, '/auth/login', endpoint='login')
-api.add_resource(Ping, '/auth/ping', endpoint='ping')
+class Google(Resource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(Google, self).__init__()
+
+    def post(self):
+        access_token_url = "https://accounts.google.com/o/oauth2/token"
+        people_api_url = "https://www.googleapis.com/plus/v1/people/me/openIdConnect"
+
+        self.reqparse.add_argument("clientId", type=str, required=True, location="json")
+        self.reqparse.add_argument(
+            "redirectUri", type=str, required=True, location="json"
+        )
+        self.reqparse.add_argument("code", type=str, required=True, location="json")
+
+        args = self.reqparse.parse_args()
+
+        # Step 1. Exchange authorization code for access token
+        payload = {
+            "client_id": args["clientId"],
+            "grant_type": "authorization_code",
+            "redirect_uri": args["redirectUri"],
+            "code": args["code"],
+            "client_secret": current_app.config.get("GOOGLE_SECRET"),
+        }
+
+        r = requests.post(access_token_url, data=payload)
+        token = r.json()
+
+        # Step 2. Retrieve information about the current user
+        headers = {"Authorization": "Bearer {0}".format(token["access_token"])}
+
+        r = requests.get(people_api_url, headers=headers)
+        profile = r.json()
+
+        user = user_service.get_by_email(profile["email"])
+
+        if not (user and user.active):
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+            )
+            return dict(message="The supplied credentials are invalid."), 403
+
+        if user:
+            metrics.send(
+                "login", "counter", 1, metric_tags={"status": SUCCESS_METRIC_STATUS}
+            )
+            return dict(token=create_token(user))
+
+        metrics.send(
+            "login", "counter", 1, metric_tags={"status": FAILURE_METRIC_STATUS}
+        )
+
+
+class Providers(Resource):
+    def get(self):
+        active_providers = []
+
+        for provider in current_app.config.get("ACTIVE_PROVIDERS", []):
+            provider = provider.lower()
+
+            if provider == "google":
+                active_providers.append(
+                    {
+                        "name": "google",
+                        "clientId": current_app.config.get("GOOGLE_CLIENT_ID"),
+                        "url": api.url_for(Google),
+                    }
+                )
+
+            elif provider == "ping":
+                active_providers.append(
+                    {
+                        "name": current_app.config.get("PING_NAME"),
+                        "url": current_app.config.get("PING_REDIRECT_URI"),
+                        "redirectUri": current_app.config.get("PING_REDIRECT_URI"),
+                        "clientId": current_app.config.get("PING_CLIENT_ID"),
+                        "responseType": "code",
+                        "scope": ["openid", "email", "profile", "address"],
+                        "scopeDelimiter": " ",
+                        "authorizationEndpoint": current_app.config.get(
+                            "PING_AUTH_ENDPOINT"
+                        ),
+                        "requiredUrlParams": ["scope"],
+                        "type": "2.0",
+                    }
+                )
+
+            elif provider == "oauth2":
+                active_providers.append(
+                    {
+                        "name": current_app.config.get("OAUTH2_NAME"),
+                        "url": current_app.config.get("OAUTH2_REDIRECT_URI"),
+                        "redirectUri": current_app.config.get("OAUTH2_REDIRECT_URI"),
+                        "clientId": current_app.config.get("OAUTH2_CLIENT_ID"),
+                        "responseType": "code",
+                        "scope": ["openid", "email", "profile", "groups"],
+                        "scopeDelimiter": " ",
+                        "authorizationEndpoint": current_app.config.get(
+                            "OAUTH2_AUTH_ENDPOINT"
+                        ),
+                        "requiredUrlParams": ["scope", "state", "nonce"],
+                        "state": "STATE",
+                        "nonce": get_psuedo_random_string(),
+                        "type": "2.0",
+                    }
+                )
+
+        return active_providers
+
+
+api.add_resource(Login, "/auth/login", endpoint="login")
+api.add_resource(Ping, "/auth/ping", endpoint="ping")
+api.add_resource(Google, "/auth/google", endpoint="google")
+api.add_resource(OAuth2, "/auth/oauth2", endpoint="oauth2")
+api.add_resource(Providers, "/auth/providers", endpoint="providers")

lemur/authorities/models.py

@@ -1,58 +1,117 @@
 """
 .. module: lemur.authorities.models
     :platform: unix
-    :synopsis: This module contains all of the models need to create a authority within Lemur.
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :synopsis: This module contains all of the models need to create an authority within Lemur.
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
+import json
 
+from flask import current_app
 from sqlalchemy.orm import relationship
-from sqlalchemy import Column, Integer, String, Text, func, ForeignKey, DateTime, PassiveDefault, Boolean
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    Text,
+    func,
+    ForeignKey,
+    DateTime,
+    DefaultClause,
+    Boolean,
+)
 from sqlalchemy.dialects.postgresql import JSON
 
 from lemur.database import db
-from lemur.certificates.models import get_cn, get_not_after, get_not_before
+from lemur.plugins.base import plugins
+from lemur.models import roles_authorities
 
 
 class Authority(db.Model):
-    __tablename__ = 'authorities'
+    __tablename__ = "authorities"
     id = Column(Integer, primary_key=True)
-    owner = Column(String(128))
+    owner = Column(String(128), nullable=False)
     name = Column(String(128), unique=True)
     body = Column(Text())
     chain = Column(Text())
-    bits = Column(Integer())
-    cn = Column(String(128))
-    not_before = Column(DateTime)
-    not_after = Column(DateTime)
     active = Column(Boolean, default=True)
-    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
     plugin_name = Column(String(64))
     description = Column(Text)
     options = Column(JSON)
-    roles = relationship('Role', backref=db.backref('authority'), lazy='dynamic')
-    user_id = Column(Integer, ForeignKey('users.id'))
-    certificates = relationship("Certificate", backref='authority')
+    date_created = Column(DateTime, DefaultClause(func.now()), nullable=False)
+    roles = relationship(
+        "Role",
+        secondary=roles_authorities,
+        passive_deletes=True,
+        backref=db.backref("authority"),
+        lazy="dynamic",
+    )
+    user_id = Column(Integer, ForeignKey("users.id"))
+    authority_certificate = relationship(
+        "Certificate",
+        backref="root_authority",
+        uselist=False,
+        foreign_keys="Certificate.root_authority_id",
+    )
+    certificates = relationship(
+        "Certificate", backref="authority", foreign_keys="Certificate.authority_id"
+    )
 
-    def __init__(self, name, owner, plugin_name, body, roles=None, chain=None, description=None):
-        self.name = name
-        self.body = body
-        self.chain = chain
-        self.owner = owner
-        self.plugin_name = plugin_name
-        cert = x509.load_pem_x509_certificate(str(body), default_backend())
-        self.cn = get_cn(cert)
-        self.not_before = get_not_before(cert)
-        self.not_after = get_not_after(cert)
-        self.roles = roles
-        self.description = description
+    authority_pending_certificate = relationship(
+        "PendingCertificate",
+        backref="root_authority",
+        uselist=False,
+        foreign_keys="PendingCertificate.root_authority_id",
+    )
+    pending_certificates = relationship(
+        "PendingCertificate",
+        backref="authority",
+        foreign_keys="PendingCertificate.authority_id",
+    )
 
-    def as_dict(self):
-        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+    def __init__(self, **kwargs):
+        self.owner = kwargs["owner"]
+        self.roles = kwargs.get("roles", [])
+        self.name = kwargs.get("name")
+        self.description = kwargs.get("description")
+        self.authority_certificate = kwargs["authority_certificate"]
+        self.plugin_name = kwargs["plugin"]["slug"]
+        self.options = kwargs.get("options")
 
-    def serialize(self):
-        blob = self.as_dict()
-        return blob
+    @property
+    def plugin(self):
+        return plugins.get(self.plugin_name)
+
+    @property
+    def is_cab_compliant(self):
+        """
+        Parse the options to find whether authority is CAB Forum Compliant,
+        i.e., adhering to the CA/Browser Forum Baseline Requirements.
+        Returns None if option is not available
+        """
+        if not self.options:
+            return None
+
+        options_array = json.loads(self.options)
+        if isinstance(options_array, list):
+            for option in options_array:
+                if "name" in option and option["name"] == 'cab_compliant':
+                    return option["value"]
+
+        return None
+
+    @property
+    def max_issuance_days(self):
+        if self.is_cab_compliant:
+            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
+
+    @property
+    def default_validity_days(self):
+        if self.is_cab_compliant:
+            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
+
+        return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365)  # 1 year default
+
+    def __repr__(self):
+        return "Authority(name={name})".format(name=self.name)

lemur/authorities/schemas.py

@@ -0,0 +1,157 @@
+"""
+.. module: lemur.authorities.schemas
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import current_app
+
+from marshmallow import fields, validates_schema, pre_load, post_dump
+from marshmallow import validate
+from marshmallow.exceptions import ValidationError
+
+from lemur.schemas import (
+    PluginInputSchema,
+    PluginOutputSchema,
+    ExtensionSchema,
+    AssociatedAuthoritySchema,
+    AssociatedRoleSchema,
+)
+from lemur.users.schemas import UserNestedOutputSchema
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.common import validators, missing
+
+from lemur.common.fields import ArrowDateTime
+from lemur.constants import CERTIFICATE_KEY_TYPES
+from lemur.plugins.base import plugins
+
+
+class AuthorityInputSchema(LemurInputSchema):
+    name = fields.String(required=True)
+    owner = fields.Email(required=True)
+    description = fields.String()
+    common_name = fields.String(required=True, validate=validators.common_name)
+
+    validity_start = ArrowDateTime()
+    validity_end = ArrowDateTime()
+    validity_years = fields.Integer()
+
+    # certificate body fields
+    organizational_unit = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATIONAL_UNIT")
+    )
+    organization = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_ORGANIZATION")
+    )
+    location = fields.String()
+    country = fields.String(
+        missing=lambda: current_app.config.get("LEMUR_DEFAULT_COUNTRY")
+    )
+    state = fields.String(missing=lambda: current_app.config.get("LEMUR_DEFAULT_STATE"))
+    # Creating a String field instead of Email to allow empty value
+    email = fields.String()
+
+    plugin = fields.Nested(PluginInputSchema)
+
+    # signing related options
+    type = fields.String(validate=validate.OneOf(["root", "subca"]), missing="root")
+    parent = fields.Nested(AssociatedAuthoritySchema)
+    signing_algorithm = fields.String(
+        validate=validate.OneOf(["sha256WithRSA", "sha1WithRSA",
+                                 "sha256WithECDSA", "SHA384withECDSA", "SHA512withECDSA"]),
+        missing="sha256WithRSA",
+    )
+    key_type = fields.String(
+        validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing="RSA2048"
+    )
+    key_name = fields.String()
+    sensitivity = fields.String(
+        validate=validate.OneOf(["medium", "high"]), missing="medium"
+    )
+    serial_number = fields.Integer()
+    first_serial = fields.Integer(missing=1)
+
+    extensions = fields.Nested(ExtensionSchema)
+
+    roles = fields.Nested(AssociatedRoleSchema(many=True))
+
+    @validates_schema
+    def validate_dates(self, data):
+        validators.dates(data)
+
+    @validates_schema
+    def validate_subca(self, data):
+        if data["type"] == "subca":
+            if not data.get("parent"):
+                raise ValidationError(
+                    "If generating a subca, parent 'authority' must be specified."
+                )
+
+    @pre_load
+    def ensure_dates(self, data):
+        return missing.convert_validity_years(data)
+
+
+class AuthorityUpdateSchema(LemurInputSchema):
+    owner = fields.Email(required=True)
+    description = fields.String()
+    active = fields.Boolean(missing=True)
+    roles = fields.Nested(AssociatedRoleSchema(many=True))
+
+
+class RootAuthorityCertificateOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    active = fields.Boolean()
+    bits = fields.Integer()
+    body = fields.String()
+    chain = fields.String()
+    description = fields.String()
+    name = fields.String()
+    cn = fields.String()
+    not_after = fields.DateTime()
+    not_before = fields.DateTime()
+    owner = fields.Email()
+    status = fields.Boolean()
+    user = fields.Nested(UserNestedOutputSchema)
+
+
+class AuthorityOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    plugin = fields.Nested(PluginOutputSchema)
+    active = fields.Boolean()
+    options = fields.Dict()
+    roles = fields.List(fields.Nested(AssociatedRoleSchema))
+    max_issuance_days = fields.Integer()
+    default_validity_days = fields.Integer()
+    authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema)
+
+    @post_dump
+    def handle_auth_certificate(self, cert):
+        # Plugins may need to modify the cert object before returning it to the user
+        plugin = plugins.get(cert['plugin']['slug'])
+        plugin.wrap_auth_certificate(cert['authority_certificate'])
+
+
+class AuthorityNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    plugin = fields.Nested(PluginOutputSchema)
+    active = fields.Boolean()
+    authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["not_after", "not_before"])
+    is_cab_compliant = fields.Boolean()
+    max_issuance_days = fields.Integer()
+    default_validity_days = fields.Integer()
+
+
+authority_update_schema = AuthorityUpdateSchema()
+authority_input_schema = AuthorityInputSchema()
+authority_output_schema = AuthorityOutputSchema()
+authorities_output_schema = AuthorityOutputSchema(many=True)

lemur/authorities/service.py

@@ -3,108 +3,151 @@
     :platform: Unix
     :synopsis: This module contains all of the services level functions used to
     administer authorities in Lemur
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from flask import g
-from flask import current_app
+
+import json
 
 from lemur import database
+from lemur.common.utils import truthiness
+from lemur.extensions import metrics
 from lemur.authorities.models import Authority
-from lemur.roles import service as role_service
-from lemur.notifications import service as notification_service
-
-from lemur.roles.models import Role
 from lemur.certificates.models import Certificate
+from lemur.roles import service as role_service
 
-from lemur.plugins.base import plugins
+from lemur.certificates.service import upload
 
 
-def update(authority_id, description=None, owner=None, active=None, roles=None):
+def update(authority_id, description, owner, active, roles):
     """
-    Update a an authority with new values.
+    Update an authority with new values.
 
     :param authority_id:
     :param roles: roles that are allowed to use this authority
-    :rtype : Authority
     :return:
     """
     authority = get(authority_id)
-    if roles:
-        authority = database.update_list(authority, 'roles', Role, roles)
-
-    if active:
-        authority.active = active
 
+    authority.roles = roles
+    authority.active = active
     authority.description = description
     authority.owner = owner
+
     return database.update(authority)
 
 
-def create(kwargs):
+def update_options(authority_id, options):
     """
-    Create a new authority.
+    Update an authority with new options.
 
-    :rtype : Authority
+    :param authority_id:
+    :param options: the new options to be saved into the authority
     :return:
     """
 
-    issuer = plugins.get(kwargs.get('pluginName'))
+    authority = get(authority_id)
 
-    kwargs['creator'] = g.current_user.email
-    cert_body, intermediate, issuer_roles = issuer.create_authority(kwargs)
+    authority.options = options
 
-    cert = Certificate(cert_body, chain=intermediate)
-    cert.owner = kwargs['ownerEmail']
+    return database.update(authority)
 
-    if kwargs['caType'] == 'subca':
-        cert.description = "This is the ROOT certificate for the {0} sub certificate authority the parent \
-                                authority is {1}.".format(kwargs.get('caName'), kwargs.get('caParent'))
-    else:
-        cert.description = "This is the ROOT certificate for the {0} certificate authority.".format(
-            kwargs.get('caName')
-        )
 
-    cert.user = g.current_user
+def mint(**kwargs):
+    """
+    Creates the authority based on the plugin provided.
+    """
+    issuer = kwargs["plugin"]["plugin_object"]
+    values = issuer.create_authority(kwargs)
 
-    cert.notifications = notification_service.create_default_expiration_notifications(
-        'DEFAULT_SECURITY',
-        current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
+    # support older plugins
+    if len(values) == 3:
+        body, chain, roles = values
+        private_key = None
+    elif len(values) == 4:
+        body, private_key, chain, roles = values
+
+    roles = create_authority_roles(
+        roles,
+        kwargs["owner"],
+        kwargs["plugin"]["plugin_object"].title,
+        kwargs["creator"],
     )
+    return body, private_key, chain, roles
 
-    # we create and attach any roles that the issuer gives us
+
+def create_authority_roles(roles, owner, plugin_title, creator):
+    """
+    Creates all of the necessary authority roles.
+    :param creator:
+    :param roles:
+    :return:
+    """
     role_objs = []
-    for r in issuer_roles:
-
-        role = role_service.create(
-            r['name'],
-            password=r['password'],
-            description="{0} auto generated role".format(kwargs.get('pluginName')),
-            username=r['username'])
+    for r in roles:
+        role = role_service.get_by_name(r["name"])
+        if not role:
+            role = role_service.create(
+                r["name"],
+                password=r["password"],
+                description="Auto generated role for {0}".format(plugin_title),
+                username=r["username"],
+            )
 
         # the user creating the authority should be able to administer it
-        if role.username == 'admin':
-            g.current_user.roles.append(role)
+        if role.username == "admin":
+            creator.roles.append(role)
 
         role_objs.append(role)
 
-    authority = Authority(
-        kwargs.get('caName'),
-        kwargs['ownerEmail'],
-        kwargs['pluginName'],
-        cert_body,
-        description=kwargs['caDescription'],
-        chain=intermediate,
-        roles=role_objs
-    )
+    # create an role for the owner and assign it
+    owner_role = role_service.get_by_name(owner)
+    if not owner_role:
+        owner_role = role_service.create(
+            owner, description="Auto generated role based on owner: {0}".format(owner)
+        )
 
-    database.update(cert)
+    role_objs.append(owner_role)
+    return role_objs
+
+
+def create(**kwargs):
+    """
+    Creates a new authority.
+    """
+    ca_name = kwargs.get("name")
+    if get_by_name(ca_name):
+        raise Exception(f"Authority with name {ca_name} already exists")
+    if role_service.get_by_name(f"{ca_name}_admin") or role_service.get_by_name(f"{ca_name}_operator"):
+        raise Exception(f"Admin and/or operator roles for authority {ca_name} already exist")
+
+    body, private_key, chain, roles = mint(**kwargs)
+
+    kwargs["creator"].roles = list(set(list(kwargs["creator"].roles) + roles))
+
+    kwargs["body"] = body
+    kwargs["private_key"] = private_key
+    kwargs["chain"] = chain
+
+    if kwargs.get("roles"):
+        kwargs["roles"] += roles
+    else:
+        kwargs["roles"] = roles
+
+    cert = upload(**kwargs)
+    kwargs["authority_certificate"] = cert
+    if kwargs.get("plugin", {}).get("plugin_options", []):
+        kwargs["options"] = json.dumps(kwargs["plugin"]["plugin_options"])
+
+    authority = Authority(**kwargs)
     authority = database.create(authority)
+    kwargs["creator"].authorities.append(authority)
 
-    g.current_user.authorities.append(authority)
-
+    metrics.send(
+        "authority_created", "counter", 1, metric_tags=dict(owner=authority.owner)
+    )
     return authority
 
 
@@ -123,7 +166,6 @@ def get(authority_id):
     """
     Retrieves an authority given it's ID
 
-    :rtype : Authority
     :param authority_id:
     :return:
     """
@@ -135,28 +177,22 @@ def get_by_name(authority_name):
     Retrieves an authority given it's name.
 
     :param authority_name:
-    :rtype : Authority
     :return:
     """
-    return database.get(Authority, authority_name, field='name')
+    return database.get(Authority, authority_name, field="name")
 
 
-def get_authority_role(ca_name):
+def get_authority_role(ca_name, creator=None):
     """
     Attempts to get the authority role for a given ca uses current_user
     as a basis for accomplishing that.
 
     :param ca_name:
     """
-    if g.current_user.is_admin:
-        authority = get_by_name(ca_name)
-        # TODO we should pick admin ca roles for admin
-        return authority.roles[0]
-    else:
-        for role in g.current_user.roles:
-            if role.authority:
-                if role.authority.name == ca_name:
-                    return role
+    if creator:
+        if creator.is_admin:
+            return role_service.get_by_name("{0}_admin".format(ca_name))
+    return role_service.get_by_name("{0}_operator".format(ca_name))
 
 
 def render(args):
@@ -166,30 +202,33 @@ def render(args):
     :return:
     """
     query = database.session_query(Authority)
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
-    filt = args.pop('filter')
+    filt = args.pop("filter")
 
     if filt:
-        terms = filt.split(';')
-        if 'active' in filt:  # this is really weird but strcmp seems to not work here??
-            query = query.filter(Authority.active == terms[1])
+        terms = filt.split(";")
+        if "active" in filt:
+            query = query.filter(Authority.active == truthiness(terms[1]))
+        elif "cn" in filt:
+            term = "%{0}%".format(terms[1])
+            sub_query = (
+                database.session_query(Certificate.root_authority_id)
+                .filter(Certificate.cn.ilike(term))
+                .subquery()
+            )
+
+            query = query.filter(Authority.id.in_(sub_query))
         else:
             query = database.filter(query, Authority, terms)
 
-    # we make sure that a user can only use an authority they either own are are a member of - admins can see all
-    if not g.current_user.is_admin:
+    # we make sure that a user can only use an authority they either own are a member of - admins can see all
+    if not args["user"].is_admin:
         authority_ids = []
-        for role in g.current_user.roles:
-            if role.authority:
-                authority_ids.append(role.authority.id)
+        for authority in args["user"].authorities:
+            authority_ids.append(authority.id)
+
+        for role in args["user"].roles:
+            for authority in role.authorities:
+                authority_ids.append(authority.id)
         query = query.filter(Authority.id.in_(authority_ids))
 
-    query = database.find_all(query, Authority, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Authority, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)
+    return database.sort_and_page(query, Authority, args)