Merge pull request #1372 from castrapel/acme_validation_dns_provider_option

R53: Extend only TXT records

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+include = lemur/*.py
+omit = lemur/migrations/*
+       lemur/tests/*
+

.gitignore

@@ -1,3 +1,4 @@
+/.cache
 .coverage
 .tox
 .DS_Store
@@ -12,6 +13,7 @@
 MANIFEST
 test.conf
 pip-log.txt
+package-lock.json
 /htmlcov
 /cover
 /build
@@ -26,5 +28,7 @@ pip-log.txt
 docs/_build
 .editorconfig
 .idea
-test.conf
+lemur/tests/tmp
-lemur/tests/tmp
+
+/lemur/plugins/lemur_email/tests/expiration-rendered.html
+/lemur/plugins/lemur_email/tests/rotation-rendered.html

.jshintrc

@@ -8,7 +8,7 @@
   "eqeqeq": true,
   "immed": true,
   "indent": 2,
-  "latedef": true,
+  "latedef": false,
   "newcap": false,
   "noarg": true,
   "quotmark": "single",
@@ -22,6 +22,8 @@
     "angular": false,
     "moment": false,
     "toaster": false,
+    "d3": false,
+    "self": false,
     "_": false
   }
 }

.pre-commit-config.yaml

@@ -0,0 +1,10 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: v0.9.1
+    hooks:
+    -   id: trailing-whitespace
+    -   id: flake8
+    -   id: check-merge-conflict
+-   repo: git://github.com/pre-commit/mirrors-jshint
+    sha: v2.9.5
+    hooks:
+    -   id: jshint

.travis.yml

@@ -1,18 +1,17 @@
-sudo: false
-
 language: python
+sudo: required
+dist: trusty
+
+node_js:
+  - "6.2.0"
 
 addons:
   postgresql: "9.4"
 
 matrix:
   include:
-    - python: "2.7"
+    - python: "3.5"
-      env: TOXENV=py27
+      env: TOXENV=py35
-    - python: "3.3"
-      env: TOXENV=py33
-    - python: "3.4"
-      env: TOXENV=py34
 
 cache:
   directories:
@@ -22,14 +21,27 @@ cache:
 env:
   global:
     - PIP_DOWNLOAD_CACHE=".pip_download_cache"
+    # do not load /etc/boto.cfg with Python 3 incompatible plugin
+    # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
+    - BOTO_CONFIG=/doesnotexist
 
 before_script:
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
+  - npm config set registry https://registry.npmjs.org
   - npm install -g bower
+  - pip install --upgrade setuptools
+
+install:
+  - pip install coveralls
+  - pip install bandit
 
 script:
   - make test
+  - bandit -r . -ll -ii -x lemur/tests/,docs
+
+after_success:
+  - coveralls
 
 notifications:
   email:

CHANGELOG.rst

@@ -1,10 +1,206 @@
 Changelog
 =========
 
-0.2.2 - `master` _
+
+0.7 - `2018-05-07`
+~~~~~~~~~~~~~~
+
+This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality.
+The linux_dst plugin will also be deprecated and removed.
+
+The pending_dns_authorizations and dns_providers tables were created. New columns
+were added to the certificates and pending_certificates tables, (For the DNS provider ID), and authorities (For options).
+Please run a database migration when upgrading.
+
+The Let's Encrypt flow will run asynchronously. When a certificate is requested through the acme-issuer, a pending certificate
+will be created. A cron needs to be defined to run `lemur pending_certs fetch_all_acme`. This command will iterate through all of the pending
+certificates, request a DNS challenge token from Let's Encrypt, and set the appropriate _acme-challenge TXT entry. It will
+then iterate through and resolve the challenges before requesting a certificate for each pending certificate. If a certificate
+is successfully obtained, the pending_certificate will be moved to the certificates table with the appropriate properties.
+
+Special thanks to all who helped with this release, notably:
+
+- The folks at Cloudflare
+- dmitryzykov
+- jchuong
+- seils
+- titouanc
+
+
+Upgrading
+---------
+
+.. note:: This release will need a migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+0.6 - `2018-01-02`
 ~~~~~~~~~~~~~~~~~~
 
-.. note:: This version not yet released and is under active development
+Happy Holidays! This is a big release with lots of bug fixes and features. Below are the highlights and are not exhaustive.
+
+
+Features:
+
+* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates.
+is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used.
+* Adds per-user API Keys, allows users to issue multiple long-lived API tokens with the same permission as the user creating them.
+* Adds the ability to revoke certificates from the Lemur UI/API, this is currently only supported for the digicert CIS and cfssl plugins.
+* Allow destinations to support an export function. Useful for file system destinations e.g. S3 to specify the export plugin you wish to run before being sent to the destination.
+* Adds support for uploading certificates to Cloudfront.
+* Re-worked certificate metadata pane for improved readability.
+* Adds support for LDAP user authentication
+
+Bugs:
+
+* Closed `#767 <https://github.com/Netflix/lemur/issues/767>`_ - Fixed issue with login redirect loop.
+* Closed `#792 <https://github.com/Netflix/lemur/issues/792>`_ - Fixed an issue with a unique constraint was violated when replacing certificates.
+* Closed `#752 <https://github.com/Netflix/lemur/issues/752>`_ - Fixed an internal server error when validating notification units.
+* Closed `#684 <https://github.com/Netflix/lemur/issues/684>`_ - Fixed migration failure when null values encountered.
+* Closes `#661 <https://github.com/Netflix/lemur/issues/661>`_ - Fixed an issue where default values were missing during clone operations.
+
+
+Special thanks to all who helped with this release, notably:
+
+- intgr
+- SecurityInsanity
+- johanneslange
+- RickB17
+- pr8kerl
+- bunjiboys
+
+See the full list of issues closed in `0.6 <https://github.com/Netflix/lemur/milestone/5>`_.
+
+Upgrading
+---------
+
+.. note:: This release will need a migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+
+0.5 - `2016-04-08`
+~~~~~~~~~~~~~~~~~~
+
+This release is most notable for dropping support for python2.7. All Lemur versions >0.4 will now support python3.5 only.
+
+Big thanks to neilschelly for quite a lot of improvements to the `lemur-cryptography` plugin.
+
+Other Highlights:
+
+* Closed `#501 <https://github.com/Netflix/lemur/issues/501>`_ - Endpoint resource as now kept in sync via an
+expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never
+removed from Lemur.
+* Closed `#551 <https://github.com/Netflix/lemur/pull/551>`_ - Added the ability to create a 4096 bit key during certificate
+creation. Closed `#528 <https://github.com/Netflix/lemur/pull/528>`_ to ensure that issuer plugins supported the new 4096 bit keys.
+* Closed `#566 <https://github.com/Netflix/lemur/issues/566>`_ - Fixed an issue changing the notification status for  certificates
+without private keys.
+* Closed `#594 <https://github.com/Netflix/lemur/issues/594>`_ - Added `replaced` field indicating if a certificate has been superseded.
+* Closed `#602 <https://github.com/Netflix/lemur/issues/602>`_ - AWS plugin added support for ALBs for endpoint tracking.
+
+
+Special thanks to all who helped with this release, notably:
+
+- RcRonco
+- harmw
+- jeremyguarini
+
+See the full list of issues closed in `0.5 <https://github.com/Netflix/lemur/milestone/4>`_.
+
+Upgrading
+---------
+
+.. note:: This release will need a slight migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+0.4 - `2016-11-17`
+~~~~~~~~~~~~~~~~~~
+
+There have been quite a few issues closed in this release. Some notables:
+
+* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated
+AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for
+future enhancements of Lemur's certificate 'deployment' capabilities.
+
+* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability
+to restrict certificate expiration dates to weekdays.
+
+Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!)
+
+This will most likely be the last release to support python2.7 moving Lemur to target python3 exclusively. Please comment
+on issue #340 if this negatively affects your usage of Lemur.
+
+See the full list of issues closed in `0.4 <https://github.com/Netflix/lemur/milestone/3>`_.
+
+Upgrading
+---------
+
+.. note:: This release will need a slight migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+0.3.0 - `2016-06-06`
+~~~~~~~~~~~~~~~~~~~~
+
+This is quite a large upgrade, it is highly advised you backup your database before attempting to upgrade as this release
+requires the migration of database structure as well as data.
+
+
+Upgrading
+---------
+
+Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+Source Plugin Owners
+--------------------
+
+The dictionary returned from a source plugin has changed keys from `public_certificate` to `body` and `intermediate_certificate` to chain.
+
+
+Issuer Plugin Owners
+--------------------
+
+This release may break your plugins, the keys in `issuer_options` have been changed from `camelCase` to `under_score`.
+This change was made to break an undue reliance on downstream options maintains a more pythonic naming convention. Renaming
+these keys should be fairly trivial, additionally pull requests have been submitted to affected plugins to help ease the transition.
+
+.. note:: This change only affects issuer plugins and does not affect any other types of plugins.
+
+
+* Closed `#63 <https://github.com/Netflix/lemur/issues/63>`_ - Validates all endpoints with Marshmallow schemas, this allows for
+    stricter input validation and better error messages when validation fails.
+* Closed `#146 <https://github.com/Netflix/lemur/issues/146>`_ - Moved authority type to first pane of authority creation wizard.
+* Closed `#147 <https://github.com/Netflix/lemur/issues/147>`_ - Added and refactored the relationship between authorities and their
+    root certificates. Displays the certificates (and chains) next to the authority in question.
+* Closed `#199 <https://github.com/Netflix/lemur/issues/199>`_ - Ensures that the dates submitted to Lemur during authority and
+    certificate creation are actually dates.
+* Closed `#230 <https://github.com/Netflix/lemur/issues/230>`_ - Migrated authority dropdown to an ui-select based dropdown, this
+    should be easier to determine what authorities are available and when an authority has actually been selected.
+* Closed `#254 <https://github.com/Netflix/lemur/issues/254>`_ - Forces certificate names to be generally unique. If a certificate name
+    (generated or otherwise) is found to be a duplicate we increment by appending a counter.
+* Closed `#254 <https://github.com/Netflix/lemur/issues/275>`_ - Switched to using Fernet generated passphrases for exported items.
+    These are more sounds that pseudo random passphrases generated before and have the nice property of being in base64.
+* Closed `#278 <https://github.com/Netflix/lemur/issues/278>`_ - Added ability to specify a custom name to certificate creation, previously
+    this was only available in the certificate import wizard.
+* Closed `#281 <https://github.com/Netflix/lemur/issues/281>`_ - Fixed an issue where notifications could not be removed from a certificate
+    via the UI.
+* Closed `#289 <https://github.com/Netflix/lemur/issues/289>`_ - Fixed and issue where intermediates were not being properly exported.
+* Closed `#315 <https://github.com/Netflix/lemur/issues/315>`_ - Made how roles are associated with certificates and authorities much more
+    explicit, including adding the ability to add roles directly to certificates and authorities on creation.
+
+
+
+0.2.2 - 2016-02-05
+~~~~~~~~~~~~~~~~~~
+
+* Closed `#234 <https://github.com/Netflix/lemur/issues/234>`_ - Allows export plugins to define whether they need
+    private key material (default is True)
+* Closed `#231 <https://github.com/Netflix/lemur/issues/231>`_ - Authorities were not respecting 'owning' roles and their
+    users
+* Closed `#228 <https://github.com/Netflix/lemur/issues/228>`_ - Fixed documentation with correct filter values
+* Closed `#226 <https://github.com/Netflix/lemur/issues/226>`_ - Fixes issue were `import_certificate` was requiring
+    replacement certificates to be specified
+* Closed `#224 <https://github.com/Netflix/lemur/issues/224>`_ - Fixed an issue where NPM might not be globally available (thanks AlexClineBB!)
+* Closed `#221 <https://github.com/Netflix/lemur/issues/234>`_ - Fixes several reported issues where older migration scripts were
+    missing tables, this change removes pre 0.2 migration scripts
+* Closed `#218 <https://github.com/Netflix/lemur/issues/234>`_ - Fixed an issue where export passphrases would not validate
 
 
 0.2.1 - 2015-12-14
@@ -19,7 +215,7 @@ Changelog
 
 
 0.2.0 - 2015-12-02
-~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 * Closed #120 - Error messages not displaying long enough
 * Closed #121 - Certificate create form should not be valid until a Certificate Authority object is available
@@ -35,8 +231,8 @@ Changelog
 
 
 0.1.5 - 2015-10-26
-~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
-* **SECURITY ISSUE**: Switched from use a AES static key to Fernet encryption.
+* **SECURITY ISSUE**: Switched from use an AES static key to Fernet encryption.
   Affects all versions prior to 0.1.5. If upgrading this will require a data migration.
-  see: `Upgrading Lemur <https://lemur.readthedocs.com/adminstration#UpgradingLemur>`_
+  see: `Upgrading Lemur <https://lemur.readthedocs.io/administration#UpgradingLemur>`_

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.5
+RUN apt-get update
+RUN apt-get install -y make python-software-properties curl
+RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
+RUN apt-get update
+RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN pip install -U setuptools
+RUN pip install coveralls bandit
+WORKDIR /app
+COPY . /app/
+RUN pip install -e .
+RUN pip install "file://`pwd`#egg=lemur[dev]"
+RUN pip install "file://`pwd`#egg=lemur[tests]"

LICENSE

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -187,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2014 Netflix, Inc.
+   Copyright 2018 Netflix, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,16 +1,38 @@
 NPM_ROOT = ./node_modules
 STATIC_DIR = src/lemur/static/app
+SHELL=/bin/bash
+USER := $(shell whoami)
 
 develop: update-submodules setup-git
 	@echo "--> Installing dependencies"
+ifeq ($(USER), root)
+	@echo "WARNING: It looks like you are installing Lemur as root. This is not generally advised."
+	npm install --unsafe-perm
+else
 	npm install
+endif
 	pip install "setuptools>=0.9.8"
 	# order matters here, base package must install first
 	pip install -e .
 	pip install "file://`pwd`#egg=lemur[dev]"
 	pip install "file://`pwd`#egg=lemur[tests]"
 	node_modules/.bin/gulp build
-	node_modules/.bin/gulp package
+	node_modules/.bin/gulp package --urlContextPath=$(urlContextPath)
+	@echo ""
+
+release:
+	@echo "--> Installing dependencies"
+ifeq ($(USER), root)
+	@echo "WARNING: It looks like you are installing Lemur as root. This is not generally advised."
+	npm install --unsafe-perm
+else
+	npm install
+endif
+	pip install "setuptools>=0.9.8"
+	# order matters here, base package must install first
+	pip install -e .
+	node_modules/.bin/gulp build
+	node_modules/.bin/gulp package --urlContextPath=$(urlContextPath)
 	@echo ""
 
 dev-docs:
@@ -41,7 +63,7 @@ test: develop lint test-python
 
 testloop: develop
 	pip install pytest-xdist
-	py.test tests -f
+	coverage run --source lemur -m py.test
 
 test-cli:
 	@echo "--> Testing CLI"
@@ -60,7 +82,7 @@ test-js:
 
 test-python:
 	@echo "--> Running Python tests"
-	py.test lemur/tests || exit 1
+	coverage run --source lemur -m py.test
 	@echo ""
 
 lint: lint-python lint-js
@@ -82,4 +104,24 @@ coverage: develop
 publish:
 	python setup.py sdist bdist_wheel upload
 
-.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish
+up-reqs:
+ifndef VIRTUAL_ENV
+    $(error Please activate virtualenv first)
+endif
+	@echo "--> Updating Python requirements"
+	pip install --upgrade pip
+	pip install --upgrade pip-tools
+	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
+	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
+	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
+	pip-compile --output-file requirements.txt requirements.in -U --no-index
+	@echo "--> Done updating Python requirements"
+	@echo "--> Removing python-ldap from requirements-docs.txt"
+	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt
+	@echo "--> Installing new dependencies"
+	pip install -e .
+	@echo "--> Done installing new dependencies"
+	@echo ""
+
+
+.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish release

README.rst

@@ -5,36 +5,31 @@ Lemur
    :alt: Join the chat at https://gitter.im/Netflix/lemur
    :target: https://gitter.im/Netflix/lemur?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: https://img.shields.io/pypi/v/lemur.svg
-    :target: https://pypi.python.org/pypi/lemur/
-    :alt: Latest Version
-
 .. image:: https://readthedocs.org/projects/lemur/badge/?version=latest
-    :target: https://lemur.readthedocs.org
+    :target: https://lemur.readthedocs.io
     :alt: Latest Docs
 
+.. image:: https://img.shields.io/badge/NetflixOSS-active-brightgreen.svg
+
 .. image:: https://travis-ci.org/Netflix/lemur.svg
     :target: https://travis-ci.org/Netflix/lemur
 
-.. image:: https://requires.io/github/Netflix/lemur/requirements.svg?branch=master
+.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=master
-    :target: https://requires.io/github/Netflix/lemur/requirements/?branch=master
+    :target: https://coveralls.io/github/Netflix/lemur?branch=master
-    :alt: Requirements Status
+
 
-.. image:: https://badge.waffle.io/Netflix/lemur.png?label=ready&title=Ready
-    :target: https://waffle.io/Netflix/lemur
-    :alt: 'Stories in Ready'
 
 Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs
 and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults.
 
-It works on CPython 2.7, 3.3, 3.4. We deploy on Ubuntu and develop on OS X.
+It works on CPython 3.5. We deploy on Ubuntu and develop on OS X.
 
 
 Project resources
 =================
 
 - `Lemur Blog Post <http://techblog.netflix.com/2015/09/introducing-lemur.html>`_
-- `Documentation <http://lemur.readthedocs.org/>`_
+- `Documentation <http://lemur.readthedocs.io/>`_
 - `Source code <https://github.com/netflix/lemur>`_
 - `Issue tracker <https://github.com/netflix/lemur/issues>`_
 - `Docker <https://github.com/Netflix/lemur-docker>`_

bower.json

@@ -6,43 +6,45 @@
   },
   "private": true,
   "dependencies": {
-    "angular": "1.3",
+    "jquery": "~2.2.0",
-    "json3": "~3.3",
-    "es5-shim": "~4.0",
-    "jquery": "~2.1",
-    "angular-resource": "1.2.15",
-    "angular-cookies": "1.2.15",
-    "angular-sanitize": "1.2.15",
-    "angular-route": "1.2.15",
-    "angular-strap": "~2.0.2",
-    "restangular": "~1.4.0",
-    "ng-table": "~0.5.4",
-    "ngAnimate": "*",
-    "moment": "~2.6.0",
-    "angular-animate": "~1.4.0",
-    "angular-loading-bar": "~0.6.0",
-    "fontawesome": "~4.2.0",
     "angular-wizard": "~0.4.0",
-    "bootswatch": "3.3.1+2",
+    "angular": "1.4.9",
-    "angular-spinkit": "~0.3.3",
+    "json3": "~3.3",
-    "angular-bootstrap": "~0.12.0",
+    "es5-shim": "~4.5.0",
-    "angular-ui-switch": "~0.1.0",
+    "bootstrap": "~3.3.6",
-    "angular-chart.js": "~0.7.1",
+    "angular-bootstrap": "~1.1.1",
-    "satellizer": "~0.9.4",
+    "angular-animate": "~1.4.9",
-    "angularjs-toaster": "~0.4.14",
+    "restangular": "~1.5.1",
-    "ngletteravatar": "~3.0.1",
+    "ng-table": "~0.8.3",
+    "moment": "~2.11.1",
+    "angular-loading-bar": "~0.8.0",
+    "angular-moment": "~0.10.3",
+    "moment-range": "~2.1.0",
+    "angular-clipboard": "~1.3.0",
+    "angularjs-toaster": "~1.0.0",
+    "angular-chart.js": "~0.8.8",
+    "ngletteravatar": "~4.0.0",
+    "bootswatch": "~3.3.6",
+    "fontawesome": "~4.5.0",
+    "satellizer": "~0.13.4",
     "angular-ui-router": "~0.2.15",
-    "angular-clipboard": "~1.1.1",
+    "font-awesome": "~4.5.0",
-    "angular-file-saver": "~1.0.1"
+    "lodash": "~4.0.1",
-  },
+    "underscore": "~1.8.3",
-  "devDependencies": {
+    "angular-smart-table": "2.1.8",
-    "angular-mocks": "~1.3",
+    "angular-strap": ">= 2.2.2",
-    "angular-scenario": "~1.3",
+    "angular-underscore": "^0.5.0",
-    "ngletteravatar": "~3.0.1"
+    "angular-translate": "^2.9.0",
+    "angular-ui-switch": "~0.1.0",
+    "angular-sanitize": "~1.5.0",
+    "angular-file-saver": "~1.0.1",
+    "angular-ui-select": "~0.17.1",
+    "d3": "^3.5.17"
   },
   "resolutions": {
-    "bootstrap": "~3.3.1",
+    "moment": ">=2.8.0 <2.11.0",
-    "angular": "1.3"
+    "lodash": ">=1.3.0 <2.5.0",
+    "angular": "1.4.9"
   },
   "ignore": [
     "**/.*",

docker-compose.yml

@@ -0,0 +1,19 @@
+---
+version: '2.0'
+services:
+  test:
+    build: .
+    volumes:
+      - ".:/app"
+    links:
+      - postgres
+    command: make test
+    environment:
+      SQLALCHEMY_DATABASE_URI: postgresql://lemur:lemur@postgres:5432/lemur
+      VIRTUAL_ENV: 'true'
+
+  postgres:
+    image: postgres:9.4
+    environment:
+      POSTGRES_USER: lemur
+      POSTGRES_PASSWORD: lemur

docs/administration.rst

@@ -7,6 +7,10 @@ Configuration
     that allow you to encrypt files at rest and decrypt them when it's time for deployment. See :ref:`Credential Management <CredentialManagement>`
     for more information.
 
+.. note::
+    All configuration values are python strings unless otherwise noted.
+
+
 Basic Configuration
 -------------------
 
@@ -24,16 +28,14 @@ Basic Configuration
 
         LOG_FILE = "/logs/lemur/lemur-test.log"
 
-
+.. data:: DEBUG
-.. data:: debug
     :noindex:
 
     Sets the flask debug flag to true (if supported by the webserver)
 
     ::
 
-        debug = False
+        DEBUG = False
-
 
     .. warning::
         This should never be used in a production environment as it exposes Lemur to
@@ -53,7 +55,7 @@ Basic Configuration
         CORS = False
 
 
-.. data:: SQLACHEMY_DATABASE_URI
+.. data:: SQLALCHEMY_DATABASE_URI
     :noindex:
 
         If you have ever used sqlalchemy before this is the standard connection string used. Lemur uses a postgres database and the connection string would look something like:
@@ -63,11 +65,59 @@ Basic Configuration
         SQLALCHEMY_DATABASE_URI = 'postgresql://<user>:<password>@<hostname>:5432/lemur'
 
 
-.. data:: LEMUR_RESTRICTED_DOMAINS
+.. data:: SQLALCHEMY_POOL_SIZE
+:noindex:
+
+            The default connection pool size is 5 for sqlalchemy managed connections.   Depending on the number of Lemur instances,
+            please specify per instance connection pool size.  Below is an example to set connection pool size to 10.
+
+        ::
+
+        SQLALCHEMY_POOL_SIZE = 10
+
+
+    .. warning::
+This is an optional setting but important to review and set for optimal database connection usage and for overall database performance.
+
+.. data:: SQLALCHEMY_MAX_OVERFLOW
+:noindex:
+
+        This setting allows to create connections in addition to specified number of connections in pool size.   By default, sqlalchemy
+        allows 10 connections to create in addition to the pool size.  This is also an optional setting.  If `SQLALCHEMY_POOL_SIZE` and
+        `SQLALCHEMY_MAX_OVERFLOW` are not speficied then each Lemur instance may create maximum of 15 connections.
+
+    ::
+
+        SQLALCHECK_MAX_OVERFLOW = 0
+
+
+    .. note::
+Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size.
+
+
+.. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION
     :noindex:
 
-        This allows the administrator to mark a subset of domains or domains matching a particular regex as
+        Specifies whether to allow certificates created by Lemur to expire on weekends. Default is True.
-        *restricted*. This means that only an administrator is allows to issue the domains in question.
+
+.. data:: LEMUR_WHITELISTED_DOMAINS
+    :noindex:
+
+        List of regular expressions for domain restrictions; if the list is not empty, normal users can only issue
+        certificates for domain names matching at least one pattern on this list. Administrators are exempt from this
+        restriction.
+
+        Cerificate common name is matched against these rules *if* it does not contain a space. SubjectAltName DNS names
+        are always matched against these rules.
+
+        Take care to write patterns in such way to not allow the `*` wildcard character inadvertently. To match a `.`
+        character, it must be escaped (as `\.`).
+
+.. data:: LEMUR_OWNER_EMAIL_IN_SUBJECT
+    :noindex:
+
+        By default, Lemur will add the certificate owner's email address to certificate subject (for CAs that allow it).
+        Set this to `False` to disable this.
 
 .. data:: LEMUR_TOKEN_SECRET
     :noindex:
@@ -106,6 +156,12 @@ Basic Configuration
         LEMUR_ENCRYPTION_KEYS = ['1YeftooSbxCiX2zo8m1lXtpvQjy27smZcUUaGmffhMY=', 'LAfQt6yrkLqOK5lwpvQcT4jf2zdeTQJV1uYeh9coT5s=']
 
 
+.. data:: DEBUG_DUMP
+    :noindex:
+
+        Dump all imported or generated CSR and certificate details to stdout using OpenSSL. (default: `False`)
+
+
 Certificate Default Options
 ---------------------------
 
@@ -145,7 +201,7 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_ORGANIZATION = "Netflix"
 
 
-.. data:: LEMUR_DEFAULT_ORGANIZATION_UNIT
+.. data:: LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
     :noindex:
 
     ::
@@ -153,6 +209,22 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = "Operations"
 
 
+.. data:: LEMUR_DEFAULT_ISSUER_PLUGIN
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_ISSUER_PLUGIN = "verisign-issuer"
+
+
+.. data:: LEMUR_DEFAULT_AUTHORITY
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_AUTHORITY = "verisign"
+
+
 Notification Options
 --------------------
 
@@ -173,47 +245,329 @@ Lemur supports sending certification expiration notifications through SES and SM
 .. data:: LEMUR_EMAIL_SENDER
     :noindex:
 
-            Specifies which service will be delivering notification emails. Valid values are `SMTP` or `SES`
+    Specifies which service will be delivering notification emails. Valid values are `SMTP` or `SES`
 
-.. note::
+    .. note::
-    If using STMP as your provider you will need to define additional configuration options as specified by Flask-Mail.
+        If using SMTP as your provider you will need to define additional configuration options as specified by Flask-Mail.
-    See: `Flask-Mail <https://pythonhosted.org/Flask-Mail>`_
+        See: `Flask-Mail <https://pythonhosted.org/Flask-Mail>`_
 
-    If you are using SES the email specified by the `LEMUR_MAIL` configuration will need to be verified by AWS before
+        If you are using SES the email specified by the `LEMUR_MAIL` configuration will need to be verified by AWS before
-    you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
+        you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
 
-.. data:: LEMUR_MAIL
+
+.. data:: LEMUR_EMAIL
     :noindex:
 
-            Lemur sender's email
+        Lemur sender's email
 
         ::
 
-        LEMUR_MAIL = 'lemur.example.com'
+            LEMUR_EMAIL = 'lemur.example.com'
 
 
 .. data:: LEMUR_SECURITY_TEAM_EMAIL
     :noindex:
 
-            This is an email or list of emails that should be notified when a certificate is expiring. It is also the contact email address for any discovered certificate.
+        This is an email or list of emails that should be notified when a certificate is expiring. It is also the contact email address for any discovered certificate.
 
         ::
 
-        LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
+            LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
 
 
 .. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS
     :noindex:
 
-            Lemur notification intervals
+        Lemur notification intervals
 
         ::
 
-        LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
+            LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
 
 
-Authority Options
+Authentication Options
------------------
+----------------------
+Lemur currently supports Basic Authentication, LDAP Authentication, Ping OAuth2, and Google out of the box. Additional flows can be added relatively easily.
+
+LDAP Options
+~~~~~~~~~~~~
+
+Lemur supports the use of an LDAP server in conjunction with Basic Authentication. Lemur local users can still be defined and take precedence over LDAP users. If a local user does not exist, LDAP will be queried for authentication. Only simple ldap binding with or without TLS is supported.
+
+LDAP support requires the pyldap python library, which also depends on the following openldap packages.
+
+.. code-block:: bash
+
+      $ sudo apt-get update
+      $ sudo apt-get install libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+
+
+To configure the use of an LDAP server, a number of settings need to be configured in `lemur.conf.py`.
+
+Here is an example LDAP configuration stanza you can add to your config. Adjust to suit your environment of course.
+ 
+.. code-block:: python
+
+        LDAP_AUTH = True
+        LDAP_BIND_URI='ldaps://secure.evilcorp.net'
+        LDAP_BASE_DN='DC=users,DC=evilcorp,DC=net'
+        LDAP_EMAIL_DOMAIN='evilcorp.net'
+        LDAP_USE_TLS = True
+        LDAP_CACERT_FILE = '/opt/lemur/trusted.pem'
+        LDAP_REQUIRED_GROUP = 'certificate-management-access'
+        LDAP_GROUPS_TO_ROLES = {'certificate-management-admin': 'admin', 'certificate-management-read-only': 'read-only'}
+
+
+The lemur ldap module uses the `user principal name` (upn) of the authenticating user to bind. This is done once for each user at login time. The UPN is effectively the email address in AD/LDAP of the user. If the user doesn't provide the email address, it constructs one based on the username supplied (which should normally match the samAccountName) and the value provided by the config LDAP_EMAIL_DOMAIN.
+The config LDAP_BASE_DN tells lemur where to search within the AD/LDAP tree for the given UPN (user). If the bind with those credentials is successful - there is a valid user in AD with correct password.
+
+Each of the LDAP options are described below.
+
+.. data:: LDAP_AUTH
+    :noindex:
+
+        This enables the use of LDAP
+
+        ::
+
+            LDAP_AUTH = True
+
+.. data:: LDAP_BIND_URI
+    :noindex:
+
+        Specifies the LDAP server connection string
+
+        ::
+
+            LDAP_BIND_URI = 'ldaps://hostname'
+
+.. data:: LDAP_BASE_DN
+    :noindex:
+
+        Specifies the LDAP distinguished name location to search for users
+
+        ::
+
+            LDAP_BASE_DN = 'DC=Users,DC=Evilcorp,DC=com'
+
+.. data:: LDAP_EMAIL_DOMAIN
+    :noindex:
+
+        The email domain used by users in your directory. This is used to build the userPrincipalName to search with.
+
+        ::
+
+            LDAP_EMAIL_DOMAIN = 'evilcorp.com'
+
+The following LDAP options are not required, however TLS is always recommended.
+
+.. data:: LDAP_USE_TLS
+    :noindex:
+
+        Enables the use of TLS when connecting to the LDAP server. Ensure the LDAP_BIND_URI is using ldaps scheme.
+
+        ::
+
+            LDAP_USE_TLS = True
+
+.. data:: LDAP_CACERT_FILE
+    :noindex:
+
+        Specify a Certificate Authority file containing PEM encoded trusted issuer certificates. This can be used if your LDAP server is using certificates issued by a private CA.
+
+        ::
+
+            LDAP_CACERT_FILE = '/path/to/cacert/file'
+
+.. data:: LDAP_REQUIRED_GROUP
+    :noindex:
+
+        Lemur has pretty open permissions. You can define an LDAP group to specify who can access Lemur. Only members of this group will be able to login.
+
+        ::
+
+            LDAP_REQUIRED_GROUP = 'Lemur LDAP Group Name'
+
+.. data:: LDAP_GROUPS_TO_ROLES
+    :noindex:
+
+        You can also define a dictionary of ldap groups mapped to lemur roles. This allows you to use ldap groups to manage access to owner/creator roles in Lemur
+
+        ::
+
+            LDAP_GROUPS_TO_ROLES = {'lemur_admins': 'admin', 'Lemur Team DL Group': 'team@example.com'}
+
+
+Authentication Providers
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you are not using an authentication provider you do not need to configure any of these options.
+
+For more information about how to use social logins, see: `Satellizer <https://github.com/sahat/satellizer>`_
+
+.. data:: ACTIVE_PROVIDERS
+    :noindex:
+
+        ::
+
+            ACTIVE_PROVIDERS = ["ping", "google", "oauth2"]
+
+.. data:: PING_SECRET
+    :noindex:
+
+        ::
+
+            PING_SECRET = 'somethingsecret'
+
+.. data:: PING_ACCESS_TOKEN_URL
+    :noindex:
+
+        ::
+
+            PING_ACCESS_TOKEN_URL = "https://<yourpingserver>/as/token.oauth2"
+
+
+.. data:: PING_USER_API_URL
+    :noindex:
+
+        ::
+
+            PING_USER_API_URL = "https://<yourpingserver>/idp/userinfo.openid"
+
+.. data:: PING_JWKS_URL
+    :noindex:
+
+        ::
+
+            PING_JWKS_URL = "https://<yourpingserver>/pf/JWKS"
+
+.. data:: PING_NAME
+    :noindex:
+
+        ::
+
+            PING_NAME = "Example Oauth2 Provider"
+
+.. data:: PING_CLIENT_ID
+    :noindex:
+
+        ::
+
+            PING_CLIENT_ID = "client-id"
+
+.. data:: PING_REDIRECT_URI
+    :noindex:
+
+        ::
+
+            PING_REDIRECT_URI = "https://<yourlemurserver>/api/1/auth/ping"
+
+.. data:: PING_AUTH_ENDPOINT
+    :noindex:
+
+        ::
+
+            PING_AUTH_ENDPOINT = "https://<yourpingserver>/oauth2/authorize"
+
+.. data:: OAUTH2_SECRET
+    :noindex:
+
+        ::
+
+            OAUTH2_SECRET = 'somethingsecret'
+
+.. data:: OAUTH2_ACCESS_TOKEN_URL
+    :noindex:
+
+        ::
+
+            OAUTH2_ACCESS_TOKEN_URL = "https://<youroauthserver> /oauth2/v1/authorize"
+
+
+.. data:: OAUTH2_USER_API_URL
+    :noindex:
+
+        ::
+
+            OAUTH2_USER_API_URL = "https://<youroauthserver>/oauth2/v1/userinfo"
+
+.. data:: OAUTH2_JWKS_URL
+    :noindex:
+
+        ::
+
+            OAUTH2_JWKS_URL = "https://<youroauthserver>/oauth2/v1/keys"
+
+.. data:: OAUTH2_NAME
+    :noindex:
+
+        ::
+
+            OAUTH2_NAME = "Example Oauth2 Provider"
+
+.. data:: OAUTH2_CLIENT_ID
+    :noindex:
+
+        ::
+
+            OAUTH2_CLIENT_ID = "client-id"
+
+.. data:: OAUTH2_REDIRECT_URI
+    :noindex:
+
+        ::
+
+            OAUTH2_REDIRECT_URI = "https://<yourlemurserver>/api/1/auth/oauth2"
+
+.. data:: OAUTH2_AUTH_ENDPOINT
+    :noindex:
+
+        ::
+
+            OAUTH2_AUTH_ENDPOINT = "https://<youroauthserver>/oauth2/v1/authorize"
+
+.. data:: OAUTH2_VERIFY_CERT
+    :noindex:
+
+        ::
+
+            OAUTH2_VERIFY_CERT = True
+
+.. data:: GOOGLE_CLIENT_ID
+    :noindex:
+
+        ::
+
+            GOOGLE_CLIENT_ID = "client-id"
+
+.. data:: GOOGLE_SECRET
+    :noindex:
+
+        ::
+
+            GOOGLE_SECRET = "somethingsecret"
+
+
+Metric Providers
+~~~~~~~~~~~~~~~~
+
+If you are not using a metric provider you do not need to configure any of these options.
+
+.. data:: ACTIVE_PROVIDERS
+    :noindex:
+
+        A list of metric plugins slugs to be ativated.
+
+        ::
+
+            METRIC_PROVIDERS = ['atlas-metric']
+
+
+Plugin Specific Options
+-----------------------
+
+Verisign Issuer Plugin
+^^^^^^^^^^^^^^^^^^^^^^
 
 Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur,
 Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation
@@ -260,87 +614,84 @@ for those plugins.
         This is the root to be used for your CA chain
 
 
-Authentication
+Digicert Issuer Plugin
---------------
+~~~~~~~~~~~~~~~~~~~~~~
-Lemur currently supports Basic Authentication, Ping OAuth2, and Google out of the box. Additional flows can be added relatively easily.
-If you are not using an authentication provider you do not need to configure any of these options.
 
-For more information about how to use social logins, see: `Satellizer <https://github.com/sahat/satellizer>`_
+The following configuration properties are required to use the Digicert issuer plugin.
 
-.. data:: ACTIVE_PROVIDERS
+
+.. data:: DIGICERT_URL
     :noindex:
 
-    ::
+            This is the url for the Digicert API (e.g. https://www.digicert.com)
 
-        ACTIVE_PROVIDERS = ["ping", "google"]
 
-.. data:: PING_SECRET
+.. data:: DIGICERT_ORDER_TYPE
     :noindex:
 
-    ::
+            This is the type of certificate to order. (e.g. ssl_plus, ssl_ev_plus see: https://www.digicert.com/services/v2/documentation/order/overview-submit)
 
-        PING_SECRET = 'somethingsecret'
 
-.. data:: PING_ACCESS_TOKEN_URL
+.. data:: DIGICERT_API_KEY
     :noindex:
 
-    ::
+            This is the Digicert API key
-
-        PING_ACCESS_TOKEN_URL = "https://<yourpingserver>/as/token.oauth2"
 
 
-.. data:: PING_USER_API_URL
+.. data:: DIGICERT_ORG_ID
     :noindex:
 
-    ::
+            This is the Digicert organization ID tied to your API key
 
-        PING_USER_API_URL = "https://<yourpingserver>/idp/userinfo.openid"
 
-.. data:: PING_JWKS_URL
+.. data:: DIGICERT_ROOT
     :noindex:
 
-    ::
+            This is the root to be used for your CA chain
 
-        PING_JWKS_URL = "https://<yourpingserver>/pf/JWKS"
 
-.. data:: PING_NAME
+.. data:: DIGICERT_DEFAULT_VALIDITY
     :noindex:
 
-    ::
+            This is the default validity (in years), if no end date is specified. (Default: 1)
 
-        PING_NAME = "Example Oauth2 Provider"
 
-.. data:: PING_CLIENT_ID
+.. data:: DIGICERT_PRIVATE
     :noindex:
 
-    ::
+            This is whether or not to issue a private certificate. (Default: False)
 
-        PING_CLIENT_ID = "client-id"
 
-.. data:: GOOGLE_CLIENT_ID
+CFSSL Issuer Plugin
+^^^^^^^^^^^^^^^^^^^
+
+The following configuration properties are required to use the CFSSL issuer plugin.
+
+.. data:: CFSSL_URL
     :noindex:
 
-    ::
+        This is the URL for the CFSSL API
 
-        GOOGLE_CLIENT_ID = "client-id"
+.. data:: CFSSL_ROOT
-
-.. data:: GOOGLE_SECRET
     :noindex:
 
-    ::
+        This is the root to be used for your CA chain
 
-        GOOGLE_SECRET = "somethingsecret"
+.. data:: CFSSL_INTERMEDIATE
+    :noindex:
+
+        This is the intermediate to be used for your CA chain
 
 
-AWS Plugin Configuration
+AWS Source/Destination Plugin
-========================
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions.
 
 .. note:: AWS usage is completely optional. Lemur can upload, find and manage TLS certificates in AWS. But is not required to do so.
 
 Setting up IAM roles
---------------------
+""""""""""""""""""""
 
 Lemur's AWS plugin uses boto heavily to talk to all the AWS resources it manages. By default it uses the on-instance credentials to make the necessary calls.
 
@@ -388,7 +739,7 @@ STS-AssumeRole
 
 
 
-Next we will create the the Lemur IAM role.
+Next we will create the Lemur IAM role.
 
 .. note::
 
@@ -445,7 +796,8 @@ IAM-ServerCertificate
 
 
 Setting up STS access
----------------------
+"""""""""""""""""""""
+
 Once we have setup our accounts we need to ensure that we create a trust relationship so that LemurInstanceProfile can assume the Lemur role.
 
 In the AWS console select the Lemur IAM role and select the Trust Relationships tab and click Edit Trust Relationship
@@ -472,7 +824,7 @@ Below is an example policy:
 
 
 Adding N+1 accounts
--------------------
+"""""""""""""""""""
 
 To add another account we go to the new account and create a new Lemur IAM role with the same policy as above.
 
@@ -500,7 +852,7 @@ An example policy:
     }
 
 Setting up SES
---------------
+""""""""""""""
 
 Lemur has built in support for sending it's certificate notifications via Amazon's simple email service (SES). To force
 Lemur to use SES ensure you are the running as the IAM role defined above and that you have followed the steps outlined
@@ -515,23 +867,6 @@ Will be the sender of all notifications, so ensure that it is verified with AWS.
 SES if the default notification gateway and will be used unless SMTP settings are configured in the application configuration
 settings.
 
-Upgrading Lemur
-===============
-
-Lemur provides an easy way to upgrade between versions. Simply download the newest
-version of Lemur from pypi and then apply any schema changes with the following command.
-
-.. code-block:: bash
-
-    $ lemur db upgrade
-
-.. note:: Internally, this uses `Alembic <https://alembic.readthedocs.org/en/latest/>`_ to manage database migrations.
-
-.. note:: By default Alembic looks for the `migrations` folder in the current working directory. 
-The migrations folder is located under `<LEMUR_HOME>/lemur/migrations` if you are running the lemur command from any
-location besides `<LEMUR_HOME>/lemur` you will need to pass the `-d` flag to specify the absolute file path to the 
-`migrations` folder.
-
 .. _CommandLineInterface:
 
 Command Line Interface
@@ -601,24 +936,33 @@ All commands default to `~/.lemur/lemur.conf.py` if a configuration is not speci
 
     Traverses every certificate that Lemur is aware of and attempts to understand its validity.
     It utilizes both OCSP and CRL. If Lemur is unable to come to a conclusion about a certificates
-    validity its status is marked 'unknown'
+    validity its status is marked 'unknown'.
 
 
 .. data:: sync
 
     Sync attempts to discover certificates in the environment that were not created by Lemur. If you wish to only sync
-    a few sources you can pass a comma delimited list of sources to sync
+    a few sources you can pass a comma delimited list of sources to sync.
 
     ::
 
-        lemur sync source1,source2
+        lemur sync -s source1,source2
 
 
-    Additionally you can also list the available sources that Lemur can sync
+    Additionally you can also list the available sources that Lemur can sync.
 
     ::
 
-        lemur sync -list
+        lemur sync
+
+
+.. data:: notify
+
+    Will traverse all current notifications and see if any of them need to be triggered.
+
+    ::
+
+        lemur notify
 
 
 Sub-commands
@@ -641,37 +985,11 @@ and to get help on sub-commands
         lemur certificates --help
 
 
-Identity and Access Management
-==============================
-
-Lemur uses a Role Based Access Control (RBAC) mechanism to control which users have access to which resources. When a
-user is first created in Lemur they can be assigned one or more roles. These roles are typically dynamically created
-depending on a external identity provider (Google, LDAP, etc.,) or are hardcoded within Lemur and associated with special
-meaning.
-
-Within Lemur there are three main permissions: AdminPermission, CreatorPermission, OwnerPermission. Sub-permissions such
-as ViewPrivateKeyPermission are compositions of these three main Permissions.
-
-Lets take a look at how these permissions are used:
-
-Each `Authority` has a set of roles associated with it. If a user is also associated with the same roles
-that the `Authority` is associated with, Lemur allows that user to user/view/update that `Authority`.
-
-This RBAC is also used when determining which users can access which certificate private key. Lemur's current permission
-structure is setup such that if the user is a `Creator` or `Owner` of a given certificate they are allow to view that
-private key. Owners can also be a role name, such that any user with the same role as owner will be allowed to view the
-private key information.
-
-These permissions are applied to the user upon login and refreshed on every request.
-
-.. seealso::
-    `Flask-Principal <https://pythonhosted.org/Flask-Principal>`_
-
 
 Upgrading Lemur
 ===============
 
-To upgrade Lemur to the newest release you will need to ensure you have the lastest code and have run any needed
+To upgrade Lemur to the newest release you will need to ensure you have the latest code and have run any needed
 database migrations.
 
 To get the latest code from github run
@@ -697,3 +1015,231 @@ After you have the latest version of the Lemur code base you must run any needed
 
 
 This will ensure that any needed tables or columns are created or destroyed.
+
+.. note::
+    Internally, this uses `Alembic <http://alembic.zzzcomputing.com/en/latest/>`_ to manage database migrations.
+
+.. note::
+    By default Alembic looks for the `migrations` folder in the current working directory.The migrations folder is
+    located under `<LEMUR_HOME>/lemur/migrations` if you are running the lemur command from any location besides
+    `<LEMUR_HOME>/lemur` you will need to pass the `-d` flag to specify the absolute file path to the `migrations` folder.
+
+Plugins
+=======
+
+There are several interfaces currently available to extend Lemur. These are a work in
+progress and the API is not frozen.
+
+Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec.
+
+Verisign/Symantec
+-----------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for the VICE 2.0 API
+
+
+Cryptography
+------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Issuer
+:Description:
+    Toy certificate authority that creates self-signed certificate authorities.
+    Allows for the creation of arbitrary authorities and end-entity certificates.
+    This is *not* recommended for production use.
+
+
+Acme
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Issuer
+:Description:
+    Adds support for the ACME protocol (including LetsEncrypt) with domain validation being handled Route53.
+
+
+Atlas
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Metric
+:Description:
+    Adds basic support for the `Atlas <https://github.com/Netflix/atlas/wiki>`_ telemetry system.
+
+
+Email
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Notification
+:Description:
+    Adds support for basic email notifications via SES.
+
+
+Slack
+-----
+
+:Authors:
+    Harm Weites <harm@weites.com>
+:Type:
+    Notification
+:Description:
+    Adds support for slack notifications.
+
+
+AWS
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Source
+:Description:
+    Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment.
+
+
+AWS
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Destination
+:Description:
+    Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment.
+
+
+Kubernetes
+----------
+
+:Authors:
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Destination
+:Description:
+    Allows Lemur to upload generated certificates to the Kubernetes certificate store.
+
+
+Java
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Generates java compatible .jks keystores and truststores from Lemur managed certificates.
+
+
+Openssl
+-------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Leverages Openssl to support additional export formats (pkcs12)
+
+
+CFSSL
+-----
+
+:Authors:
+    Charles Hendrie <chad.hendrie@thomsonreuters.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for generating certificates from the private certificate authority CFSSL
+
+
+3rd Party Plugins
+=================
+
+The following plugins are available and maintained by members of the Lemur community:
+
+Digicert
+--------
+
+:Authors:
+    Chris Dorros
+:Type:
+    Issuer
+:Description:
+    Adds support for basic Digicert
+:Links:
+    https://github.com/opendns/lemur-digicert
+
+
+InfluxDB
+--------
+
+:Authors:
+    Titouan Christophe
+:Type:
+    Metric
+:Description:
+    Sends key metrics to InfluxDB
+:Links:
+    https://github.com/titouanc/lemur-influxdb
+
+Hashicorp Vault
+---------------
+
+:Authors:
+    Ron Cohen
+:Type:
+    Issuer
+:Description:
+    Adds support for basic Vault PKI secret backend.
+:Links:
+    https://github.com/RcRonco/lemur_vault
+
+
+Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
+get it added.
+
+Want to create your own extension? See :doc:`../developer/plugins/index` to get started.
+
+
+Identity and Access Management
+==============================
+
+Lemur uses a Role Based Access Control (RBAC) mechanism to control which users have access to which resources. When a
+user is first created in Lemur they can be assigned one or more roles. These roles are typically dynamically created
+depending on an external identity provider (Google, LDAP, etc.), or are hardcoded within Lemur and associated with special
+meaning.
+
+Within Lemur there are three main permissions: AdminPermission, CreatorPermission, OwnerPermission. Sub-permissions such
+as ViewPrivateKeyPermission are compositions of these three main Permissions.
+
+Lets take a look at how these permissions are used:
+
+Each `Authority` has a set of roles associated with it. If a user is also associated with the same roles
+that the `Authority` is associated with, Lemur allows that user to user/view/update that `Authority`.
+
+This RBAC is also used when determining which users can access which certificate private key. Lemur's current permission
+structure is setup such that if the user is a `Creator` or `Owner` of a given certificate they are allow to view that
+private key. Owners can also be a role name, such that any user with the same role as owner will be allowed to view the
+private key information.
+
+These permissions are applied to the user upon login and refreshed on every request.
+
+.. seealso::
+
+    `Flask-Principal <https://pythonhosted.org/Flask-Principal>`_

docs/conf.py

@@ -13,12 +13,24 @@
 # serve to show the default.
 import sys
 import os
+from unittest.mock import MagicMock
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('..'))
 
+# Mock packages that cannot be installed on rtd
+on_rtd = os.environ.get('READTHEDOCS') == 'True'
+if on_rtd:
+    class Mock(MagicMock):
+        @classmethod
+        def __getattr__(cls, name):
+            return MagicMock()
+
+    MOCK_MODULES = ['ldap']
+    sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -47,7 +59,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'lemur'
-copyright = u'2015, Netflix Inc.'
+copyright = u'2018, Netflix Inc.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -101,9 +113,13 @@ pygments_style = 'sphinx'
 
 # -- Options for HTML output ----------------------------------------------
 
-# The theme to use for HTML and HTML Help pages.  See the documentation for
+# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-# a list of builtin themes.
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-html_theme = 'default'
+
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/developer/index.rst

@@ -48,7 +48,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi
 * pip
 * virtualenv (ideally virtualenvwrapper)
 * node.js (for npm and building css/javascript)
-* (Optional) Potgresql
++* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
 
 Once you've got all that, the rest is simple:
 
@@ -77,6 +77,7 @@ Create a default Lemur configuration just as if this were a production instance:
 
 ::
 
+    lemur create_config
     lemur init
 
 You'll likely want to make some changes to the default configuration (we recommend developing against Postgres, for example). Once done, migrate your database using the following command:
@@ -86,7 +87,13 @@ You'll likely want to make some changes to the default configuration (we recomme
 	lemur upgrade
 
 
-.. note:: The ``upgrade`` shortcut is simply a shorcut to Alembic's upgrade command.
+.. note:: The ``upgrade`` shortcut is simply a shortcut to Alembic's upgrade command.
+
+
+Running tests with Docker and docker-compose
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Alternatively you can use Docker and docker-compose for running the tests with ``docker-compose run test``.
 
 
 Coding Standards
@@ -113,6 +120,12 @@ HTML:
   2 Spaces
 
 
+Git hooks
+~~~~~~~~~
+
+To help developers maintain the above standards, Lemur includes a configuration file for Yelp's `pre-commit <http://pre-commit.com/>`_. This is an optional dependency and is not required in order to contribute to Lemur.
+
+
 Running the Test Suite
 ----------------------
 
@@ -144,8 +157,19 @@ If you've made changes and need to compile them by hand for any reason, you can
 
 The minified and processed files should be committed alongside the unprocessed changes.
 
+It's also important to note that Lemur's frontend and API are not tied together. The API does not serve any of the static assets, we rely on nginx or some other file server to server all of the static assets.
+During development that means we need an additional server to serve those static files for the GUI.
+
+This is accomplished with a Gulp task:
+
+::
+
+    ./node_modules/.bin/gulp serve
+
+The gulp task compiles all the JS/CSS/HTML files and opens the Lemur welcome page in your default browsers. Additionally any changes to made to the JS/CSS/HTML with be reloaded in your browsers.
+
 Developing with Flask
-----------------------
+---------------------
 
 Because Lemur is just Flask, you can use all of the standard Flask functionality. The only difference is you'll be accessing commands that would normally go through manage.py using the ``lemur`` CLI helper instead.
 
@@ -164,7 +188,7 @@ Schema changes should always introduce the new schema in a commit, and then intr
 
 Removing columns and tables requires a slightly more painful flow, and should resemble the follow multi-commit flow:
 
-- Remove all references to the column or table (but dont remove the Model itself)
+- Remove all references to the column or table (but don't remove the Model itself)
 - Remove the model code
 - Remove the table or column
 
@@ -180,19 +204,116 @@ You can see a list of open pull requests (pending changes) by visiting https://g
 
 Pull requests should be against **master** and pass all TravisCI checks
 
-Plugins
+
-=======
+Writing a Plugin
+================
 
 .. toctree::
-    :maxdepth: 1
+    :maxdepth: 2
 
     plugins/index
 
+
+REST API
+========
+
+Lemur's front end is entirely API driven. Any action that you can accomplish via the UI can also be accomplished by the
+API. The following is documents and provides examples on how to make requests to the Lemur API.
+
+Authentication
+--------------
+
+.. automodule:: lemur.auth.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Destinations
+------------
+
+.. automodule:: lemur.destinations.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Notifications
+-------------
+
+.. automodule:: lemur.notifications.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Users
+-----
+
+.. automodule:: lemur.users.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Roles
+-----
+
+.. automodule:: lemur.roles.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Certificates
+------------
+
+.. automodule:: lemur.certificates.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Authorities
+-----------
+
+.. automodule:: lemur.authorities.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Domains
+-------
+
+.. automodule:: lemur.domains.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Endpoints
+---------
+
+.. automodule:: lemur.endpoints.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Logs
+----
+
+.. automodule:: lemur.logs.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Sources
+-------
+
+.. automodule:: lemur.sources.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
 Internals
 =========
 
 .. toctree::
-    :maxdepth: 1
+    :maxdepth: 2
 
     internals/lemur
-

docs/developer/internals/lemur.certificates.rst

@@ -1,15 +1,6 @@
 certificates Package
 ====================
 
-:mod:`exceptions` Module
-------------------------
-
-.. automodule:: lemur.certificates.exceptions
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`models` Module
 --------------------
 

docs/developer/internals/lemur.plugins.lemur_cfssl.rst

@@ -0,0 +1,20 @@
+lemur_cfssl Package
+===================
+
+:mod:`lemur_cfssl` Package
+--------------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.lemur_verisign.rst

@@ -10,15 +10,6 @@ lemur_verisign Package
     :undoc-members:
     :show-inheritance:
 
-:mod:`constants` Module
------------------------
-
-.. automodule:: lemur.plugins.lemur_verisign.constants
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 :mod:`plugin` Module
 --------------------
 

docs/developer/internals/lemur.plugins.rst

@@ -27,6 +27,6 @@ Subpackages
     lemur.plugins.base
     lemur.plugins.bases
     lemur.plugins.lemur_aws
-    lemur.plugins.lemur_cloudca
+    lemur.plugins.lemur_cfssl
     lemur.plugins.lemur_email
     lemur.plugins.lemur_verisign

docs/developer/internals/lemur.rst

@@ -96,5 +96,19 @@ Subpackages
     lemur.notifications
     lemur.plugins
     lemur.roles
-    lemur.status
     lemur.users
+    lemur.sources
+    lemur.logs
+    lemur.reporting
+    lemur.tests
+    lemur.deployment
+    lemur.endpoints
+    lemur.defaults
+    lemur.plugins.lemur_acme
+    lemur.plugins.lemur_atlas
+    lemur.plugins.lemur_cryptography
+    lemur.plugins.lemur_digicert
+    lemur.plugins.lemur_java
+    lemur.plugins.lemur_kubernetes
+    lemur.plugins.lemur_openssl
+    lemur.plugins.lemur_slack

docs/developer/internals/lemur.status.rst

@@ -1,11 +0,0 @@
-status Package
-==============
-
-:mod:`views` Module
--------------------
-
-.. automodule:: lemur.status.views
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:

docs/developer/plugins/index.rst

@@ -1,6 +1,3 @@
-Writing a Plugin
-================
-
 Several interfaces exist for extending Lemur:
 
 * Issuer (lemur.plugins.base.issuer)
@@ -28,7 +25,7 @@ if you want to pull the version using pkg_resources (which is what we recommend)
     try:
         VERSION = __import__('pkg_resources') \
             .get_distribution(__name__).version
-    except Exception, e:
+    except Exception as e:
         VERSION = 'unknown'
 
 Inside of ``plugin.py``, you'll declare your Plugin class::
@@ -73,10 +70,18 @@ at multiple plugins within your package::
         },
     )
 
+Once your plugin files are in place and the ``/www/lemur/setup.py`` file has been modified, you can load your plugin into your instance by reinstalling lemur:
+::
+
+    (lemur)$cd /www/lemur
+    (lemur)$pip install -e .
+
 That's it! Users will be able to install your plugin via ``pip install <package name>``.
 
 .. SeeAlso:: For more information about python packages see `Python Packaging <https://packaging.python.org/en/latest/distributing.html>`_
 
+.. SeeAlso:: For an example of a plugin operation outside of Lemur's core, see `lemur-digicert <https://github.com/opendns/lemur-digicert>`_
+
 .. _PluginInterfaces:
 
 Plugin Interfaces
@@ -95,10 +100,16 @@ If you have a third party or internal service that creates authorities (EJBCA, e
 it can treat any issuer plugin as both a source of creating new certificates as well as new authorities.
 
 
-The `IssuerPlugin` exposes two functions::
+The `IssuerPlugin` exposes four functions functions::
 
-    def create_certificate(self, options):
+    def create_certificate(self, csr, issuer_options):
         # requests.get('a third party')
+    def revoke_certificate(self, certificate, comments):
+        # requests.put('a third party')
+    def get_ordered_certificate(self, order_id):
+        # requests.get('already existing certificate')
+    def canceled_ordered_certificate(self, pending_cert, **kwargs):
+        # requests.put('cancel an order that has yet to be issued')
 
 Lemur will pass a dictionary of all possible options for certificate creation. Including a valid CSR, and the raw options associated with the request.
 
@@ -134,15 +145,34 @@ The `IssuerPlugin` doesn't have any options like Destination, Source, and Notifi
 any fields you might need to submit a request to a third party. If there are additional options you need
 in your plugin feel free to open an issue, or look into adding additional options to issuers yourself.
 
+Asynchronous Certificates
+^^^^^^^^^^^^^^^^^^^^^^^^^
+An issuer may take some time to actually issue a certificate for an order.  In this case, a `PendingCertificate` is returned, which holds information to recreate a `Certificate` object at a later time.  Then, `get_ordered_certificate()` should be run periodically via `python manage.py pending_certs fetch -i all` to attempt to retrieve an ordered certificate::
+
+    def get_ordered_ceriticate(self, order_id):
+        # order_id is the external id of the order, not the external_id of the certificate
+        # retrieve an order, and check if there is an issued certificate attached to it
+
+`cancel_ordered_certificate()` should be implemented to allow an ordered certificate to be canceled before it is issued::
+    def cancel_ordered_certificate(self, pending_cert, **kwargs):
+        # pending_cert should contain the necessary information to match an order
+        # kwargs can be given to provide information to the issuer for canceling
+
 Destination
 -----------
 
 Destination plugins allow you to propagate certificates managed by Lemur to additional third parties. This provides flexibility when
 different orchestration systems have their own way of manage certificates or there is an existing system you wish to integrate with Lemur.
 
+By default destination plugins have a private key requirement. If your plugin does not require a certificates private key mark `requires_key = False`
+in the plugins base class like so::
+
+    class MyDestinationPlugin(DestinationPlugin):
+        requires_key = False
+
 The DestinationPlugin requires only one function to be implemented::
 
-    def upload(self, cert, private_key, cert_chain, options, **kwargs):
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
         # request.post('a third party')
 
 Additionally the DestinationPlugin allows the plugin author to add additional options
@@ -151,25 +181,25 @@ that can be used to help define sub-destinations.
 For example, if we look at the aws-destination plugin we can see that it defines an `accountNumber` option::
 
     options = [
-      {
+        {
-          'name': 'accountNumber',
+            'name': 'accountNumber',
-          'type': 'int',
+            'type': 'int',
-          'required': True,
+            'required': True,
-          'validation': '/^[0-9]{12,12}$/',
+            'validation': '/^[0-9]{12,12}$/',
-          'helpMessage': 'Must be a valid AWS account number!',
+            'helpMessage': 'Must be a valid AWS account number!',
-      }
+        }
     ]
 
 By defining an `accountNumber` we can make this plugin handle many N number of AWS accounts instead of just one.
 
 The schema for defining plugin options are pretty straightforward:
 
-  - **Name**: name of the variable you wish to present the user, snake case (snakeCase) is preferrred as Lemur
+  - **Name**: name of the variable you wish to present the user, snake case (snakeCase) is preferred as Lemur
     will parse these and create pretty variable titles
   - **Type** there are currently four supported variable types
       - **Int** creates an html integer box for the user to enter integers into
       - **Str** creates a html text input box
-      - **Boolean** creates a checkbox for the user to signify truithyness
+      - **Boolean** creates a checkbox for the user to signify truthiness
       - **Select** creates a select box that gives the user a list of options
           - When used a `available` key must be provided with a list of selectable options
   - **Required** determines if this option is required, this **must be a boolean value**
@@ -185,7 +215,7 @@ Notification
 ------------
 
 Lemur includes the ability to create Email notifications by **default**. These notifications
-currently come in the form of expiration noticies. Lemur periodically checks certifications expiration dates and
+currently come in the form of expiration notices. Lemur periodically checks certifications expiration dates and
 determines if a given certificate is eligible for notification. There are currently only two parameters used to
 determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
 of days the current date (UTC) is from that expiration date.
@@ -196,10 +226,10 @@ are trying to create a new notification type (audit, failed logins, etc.) this w
 You would also then need to build additional code to trigger the new notification type.
 
 The second is `ExpirationNotificationPlugin`, this object inherits from `NotificationPlugin` object.
-You will most likely want to base your plugin on, if you want to add new channels for expiration notices (Slack, Hipcat, Jira, etc.). It adds default options that are required by
+You will most likely want to base your plugin on, if you want to add new channels for expiration notices (Slack, HipChat, Jira, etc.). It adds default options that are required by
-by all expiration notifications (interval, unit). This interface expects for the child to define the following function::
+all expiration notifications (interval, unit). This interface expects for the child to define the following function::
 
-    def send(self):
+    def send(self, notification_type, message, targets, options, **kwargs):
         #  request.post("some alerting infrastructure")
 
 
@@ -207,27 +237,27 @@ Source
 ------
 
 When building Lemur we realized that although it would be nice if every certificate went through Lemur to get issued, but this is not
-always be the case. Often times there are third parties that will issue certificates on your behalf and these can get deployed
+always be the case. Oftentimes there are third parties that will issue certificates on your behalf and these can get deployed
 to infrastructure without any interaction with Lemur. In an attempt to combat this and try to track every certificate, Lemur has a notion of
 certificate **Sources**. Lemur will contact the source at periodic intervals and attempt to **sync** against the source. This means downloading or discovering any
-certificate Lemur does not know about and adding the certificate to it's inventory to be tracked and alerted on.
+certificate Lemur does not know about and adding the certificate to its inventory to be tracked and alerted on.
 
 The `SourcePlugin` object has one default option of `pollRate`. This controls the number of seconds which to get new certificates.
 
- .. warning::
+.. warning::
-Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. A lock file is generated to guarantee that
+    Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. A lock file is generated to guarantee that
     only one sync is running at a time. It also means that the minimum resolution of a source plugin poll rate is effectively 15min. You can always specify a faster cron
     job if you need a higher resolution sync job.
 
 
 The `SourcePlugin` object requires implementation of one function::
 
-      def get_certificates(self, **kwargs):
+      def get_certificates(self, options, **kwargs):
           #  request.get("some source of certificates")
 
 
-.. Note::
+.. note::
-Often times to facilitate code re-use it makes sense put source and destination plugins into one package.
+    Oftentimes to facilitate code re-use it makes sense put source and destination plugins into one package.
 
 
 Export
@@ -247,9 +277,8 @@ The `ExportPlugin` object requires the implementation of one function::
         # return "extension", passphrase, raw
 
 
-.. Note::
+.. note::
-Support of various formats sometimes relies on external tools system calls. Always be mindful of sanitizing any input to
+    Support of various formats sometimes relies on external tools system calls. Always be mindful of sanitizing any input to these calls.
-these calls.
 
 
 Testing
@@ -268,9 +297,9 @@ Augment your setup.py to ensure at least the following:
 
    setup(
        # ...
-      install_requires=[
+       install_requires=[
           'lemur',
-      ]
+       ]
    )
 
 
@@ -281,11 +310,7 @@ The ``conftest.py`` file is our main entry-point for py.test. We need to configu
 
 .. code-block:: python
 
-   from __future__ import absolute_import
+   from lemur.tests.conftest import *  # noqa
-
-   pytest_plugins = [
-       'lemur.utils.pytest'
-   ]
 
 
 Test Cases
@@ -295,14 +320,18 @@ You can now inherit from Lemur's core test classes. These are Django-based and e
 
 .. code-block:: python
 
-   # test_myextension.py
+    import pytest
-   from __future__ import absolute_import
+    from lemur.tests.vectors import INTERNAL_CERTIFICATE_A_STR, INTERNAL_PRIVATE_KEY_A_STR
 
-   from lemur.testutils import TestCase
+    def test_export_keystore(app):
+        from lemur.plugins.base import plugins
+        p = plugins.get('java-keystore-jks')
+        options = [{'name': 'passphrase', 'value': 'test1234'}]
+        with pytest.raises(Exception):
+            p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
 
-   class MyExtensionTest(TestCase):
+        raw = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
-       def test_simple(self):
+        assert raw != b""
-          assert 1 != 2
 
 
 Running Tests
@@ -314,13 +343,14 @@ Running tests follows the py.test standard. As long as your test files and metho
 
     $ py.test -v
     ============================== test session starts ==============================
-    platform darwin -- Python 2.7.9 -- py-1.4.26 -- pytest-2.6.4/python2.7
+    platform darwin -- Python 2.7.10, pytest-2.8.5, py-1.4.30, pluggy-0.3.1
-    plugins: django
+    cachedir: .cache
-    collected 1 items
+    plugins: flask-0.10.0
+    collected 346 items
 
-    tests/test_myextension.py::MyExtensionTest::test_simple PASSED
+    lemur/plugins/lemur_acme/tests/test_acme.py::test_get_certificates PASSED
 
     =========================== 1 passed in 0.35 seconds ============================
 
 
-.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above. View the source: # TODO
+.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above.

docs/developer/rest.rst

@@ -1,66 +0,0 @@
-Lemur's front end is entirely API driven. Any action that you can accomplish via the UI can also be accomplished by the
-UI. The following is documents and provides examples on how to make requests to the Lemur API.
-
-Authentication
---------------
-
-.. automodule:: lemur.auth.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Destinations
-------------
-
-.. automodule:: lemur.destinations.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Notifications
--------------
-
-.. automodule:: lemur.notifications.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Users
------
-
-.. automodule:: lemur.users.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Roles
------
-
-.. automodule:: lemur.roles.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Certificates
-------------
-
-.. automodule:: lemur.certificates.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Authorities
------------
-
-.. automodule:: lemur.authorities.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Domains
--------
-
-.. automodule:: lemur.domains.views
-    :members:
-    :undoc-members:
-    :show-inheritance:

docs/faq.rst

@@ -6,7 +6,7 @@ Common Problems
 
 In my startup logs I see *'Aborting... Lemur cannot locate db encryption key, is LEMUR_ENCRYPTION_KEYS set?'*
   You likely have not correctly configured **LEMUR_ENCRYPTION_KEYS**. See
-  :doc:`administration/index` for more information.
+  :doc:`administration` for more information.
 
 
 I am seeing Lemur's javascript load in my browser but not the CSS.

docs/guide/index.rst

@@ -18,7 +18,7 @@ that Lemur can then manage.
 
 .. figure:: create_authority.png
 
-    Enter a authority name and short description about the authority. Enter an owner,
+    Enter an authority name and short description about the authority. Enter an owner,
     and certificate common name. Depending on the authority and the authority/issuer plugin
     these values may or may not be used.
 
@@ -56,7 +56,7 @@ Import an Existing Certificate
 
 .. figure:: upload_certificate.png
 
-    Enter a owner, short description and public certificate. If there are intermediates and private keys
+    Enter an owner, short description and public certificate. If there are intermediates and private keys
     Lemur will track them just as it does if the certificate were created through Lemur. Lemur generates
     a certificate name but you can override that by passing a value to the `Custom Name` field.
 

docs/index.rst

@@ -27,8 +27,7 @@ Administration
 .. toctree::
     :maxdepth: 2
 
-    administration/index
+    administration
-    plugins/index
 
 Developers
 ----------
@@ -38,17 +37,24 @@ Developers
 
     developer/index
 
-
+Security
-REST API
 --------
 
 .. toctree::
     :maxdepth: 2
 
-    developer/rest
+    security
+
+Doing a Release
+---------------
+
+.. toctree::
+    :maxdepth: 1
+
+    doing-a-release
 
 FAQ
-----
+---
 
 .. toctree::
     :maxdepth: 1

docs/plugins/index.rst

@@ -1,20 +0,0 @@
-Plugins
-=======
-
-There are several interfaces currently available to extend Lemur. These are a work in
-progress and the API is not frozen.
-
-Bundled Plugins
----------------
-
-Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec and CloudCA services.
-
-3rd Party Extensions
---------------------
-
-The following extensions are available and maintained by members of the Lemur community:
-
-Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
-get it added.
-
-Want to create your own extension? See :doc:`../developer/plugins/index` to get started.

docs/production/index.rst

@@ -37,20 +37,20 @@ Entropy
 -------
 
 Lemur generates private keys for the certificates it creates. This means that it is vitally important that Lemur has enough entropy to draw from. To generate private keys Lemur uses the python library `Cryptography <https://cryptography.io>`_. In turn Cryptography uses OpenSSL bindings to generate
-keys just like you might from the OpenSSL command line. OpenSSL draws it's initial entropy from system during startup and uses PRNGs to generate a stream of random bytes (as output by /dev/urandom) whenever it needs to do a cryptographic operation.
+keys just like you might from the OpenSSL command line. OpenSSL draws its initial entropy from system during startup and uses PRNGs to generate a stream of random bytes (as output by /dev/urandom) whenever it needs to do a cryptographic operation.
 
 What does all this mean? Well in order for the keys
 that Lemur generates to be strong, the system needs to interact with the outside world. This is typically accomplished through the systems hardware (thermal, sound, video user-input, etc.) since the physical world is much more "random" than the computer world.
 
-If you are running Lemur on its own server with its own hardware "bare metal" then the entropy of the system is typically "good enough" for generating keys. If however you are using an VM on shared hardware there is a potential that your initial seed data (data that was initially
+If you are running Lemur on its own server with its own hardware "bare metal" then the entropy of the system is typically "good enough" for generating keys. If however you are using a VM on shared hardware there is a potential that your initial seed data (data that was initially
-fed to the PRNG) is not very good. What's more VMs have been known to be unable to inject more entropy into the system once it has been started. This is because there is typically very little interaction with the server once it has been started.
+fed to the PRNG) is not very good. What's more, VMs have been known to be unable to inject more entropy into the system once it has been started. This is because there is typically very little interaction with the server once it has been started.
 
 The amount of effort you wish to expend ensuring that Lemur has good entropy to draw from is up to your specific risk tolerance and how Lemur is configured.
 
 If you wish to generate more entropy for your system we would suggest you take a look at the following resources:
 
 - `WES-entropy-client <https://github.com/WhitewoodCrypto/WES-entropy-client>`_
-- `haveaged <http://www.issihosts.com/haveged/>`_
+- `haveged <http://www.issihosts.com/haveged/>`_
 
 For additional information about OpenSSL entropy issues:
 
@@ -72,7 +72,7 @@ Nginx is a very popular choice to serve a Python project:
 Nginx doesn't run any Python process, it only serves requests from outside to
 the Python server.
 
-Therefore there are two steps:
+Therefore, there are two steps:
 
 - Run the Python process.
 - Run Nginx.
@@ -110,7 +110,7 @@ You can make some adjustments to get a better user experience::
        error_log   /var/log/nginx/log/lemur.error.log;
 
        location /api {
-            proxy_pass  http://127.0.0.1:5000;
+            proxy_pass  http://127.0.0.1:8000;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
             proxy_redirect off;
             proxy_buffering off;
@@ -176,7 +176,7 @@ sensitive nature of Lemur and what it controls makes this essential. This is a s
        resolver <IP DNS resolver>;
 
        location /api {
-            proxy_pass  http://127.0.0.1:5000;
+            proxy_pass  http://127.0.0.1:8000;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
             proxy_redirect off;
             proxy_buffering off;
@@ -217,13 +217,30 @@ An example apache config::
         # HSTS (mod_headers is required) (15768000 seconds = 6 months)
         Header always set Strict-Transport-Security "max-age=15768000"
         ...
+        
+     # Set the lemur DocumentRoot to static/dist
+     DocumentRoot /www/lemur/lemur/static/dist
+     
+     # Uncomment to force http 1.0 connections to proxy
+     # SetEnv force-proxy-request-1.0 1
+     
+     #Don't keep proxy connections alive
+     SetEnv proxy-nokeepalive 1
+     
+     # Only need to do reverse proxy
+     ProxyRequests Off
+     
+     # Proxy requests to the api to the lemur service (and sanitize redirects from it)
+     ProxyPass "/api" "http://127.0.0.1:8000/api"
+     ProxyPassReverse "/api" "http://127.0.0.1:8000/api"
+     
     </VirtualHost>
 
 Also included in the configurations above are several best practices when it comes to deploying TLS. Things like enabling
 HSTS, disabling vulnerable ciphers are all good ideas when it comes to deploying Lemur into a production environment.
 
 .. note::
-    This is a rather incomplete apache config for running Lemur (needs mod_wsgi etc.,), if you have a working apache config please let us know!
+    This is a rather incomplete apache config for running Lemur (needs mod_wsgi etc.), if you have a working apache config please let us know!
 
 .. seealso::
     `Mozilla SSL Configuration Generator <https://mozilla.github.io/server-side-tls/ssl-config-generator/>`_
@@ -240,10 +257,10 @@ most of the time), but here is a quick overview on how to use it.
 Create a configuration file named supervisor.ini::
 
     [unix_http_server]
-    file=/tmp/supervisor.sock;
+    file=/tmp/supervisor.sock
 
     [supervisorctl]
-    serverurl=unix:///tmp/supervisor.sock;
+    serverurl=unix:///tmp/supervisor.sock
 
     [rpcinterface:supervisor]
     supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
@@ -295,3 +312,25 @@ Then you can manage the process by running::
 It will start a shell from which you can start/stop/restart the service.
 
 You can read all errors that might occur from /tmp/lemur.log.
+
+
+Periodic Tasks
+==============
+
+Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
+a cron job that runs the commands.
+
+There are currently three commands that could/should be run on a periodic basis:
+
+- `notify`
+- `check_revoked`
+- `sync`
+
+How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
+`sync` is typically run every 15 minutes.
+
+Example cron entries::
+
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
+    */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked

docs/quickstart/index.rst

@@ -12,11 +12,11 @@ Dependencies
 Some basic prerequisites which you'll need in order to run Lemur:
 
 * A UNIX-based operating system (we test on Ubuntu, develop on OS X)
-* Python 2.7
+* Python 3.5 or greater
-* PostgreSQL
+* PostgreSQL 9.4 or greater
 * Nginx
 
-.. note:: Lemur was built with in AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
+.. note:: Lemur was built with in AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
 
 
 Installing Build Dependencies
@@ -27,10 +27,13 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack
 .. code-block:: bash
 
     $ sudo apt-get update
-    $ sudo apt-get install install nodejs-legacy python-pip python-dev libpq-dev build-essential libssl-dev libffi-dev nginx git supervisor npm postgresql
+    $ sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql
 
 .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver.  npm is needed if you're installing Lemur from the source (e.g., from git).
 
+.. note:: Installing node from a package manager may creat the nodejs bin at  /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following
+    $ sudo ln -s /user/bin/nodejs /usr/bin/node
+
 Now, install Python ``virtualenv`` package:
 
 .. code-block:: bash
@@ -52,6 +55,10 @@ Clone Lemur inside the just created directory and give yourself write permission
 
 .. code-block:: bash
 
+    $ sudo useradd lemur
+    $ sudo passwd lemur
+    $ sudo mkdir /home/lemur
+    $ sudo chown lemur:lemur /home/lemur
     $ sudo git clone https://github.com/Netflix/lemur
     $ sudo chown -R lemur lemur/
 
@@ -59,7 +66,8 @@ Create the virtual environment, activate it and enter the Lemur's directory:
 
 .. code-block:: bash
 
-    $ virtualenv lemur
+    $ su lemur
+    $ virtualenv -p python3 lemur
     $ source /www/lemur/bin/activate
     $ cd lemur
 
@@ -79,11 +87,23 @@ And then run:
 
 .. code-block:: bash
 
-  $ make develop
+  $ make release
 
 .. note:: This command will install npm dependencies as well as compile static assets.
 
 
+You may also run with the urlContextPath variable set. If this is set it will add the desired context path for subsequent calls back to lemur. This will only edit the front end code for calls back to the server, you will have to make sure the server knows about these routes.
+::
+
+  Example:
+    urlContextPath=lemur
+    /api/1/auth/providers -> /lemur/api/1/auth/providers
+
+.. code-block:: bash
+
+  $ make release urlContextPath={desired context path}
+
+
 Creating a configuration
 ------------------------
 
@@ -105,9 +125,24 @@ Update your configuration
 
 Once created, you will need to update the configuration file with information about your environment, such as which database to talk to, where keys are stored etc.
 
-.. note:: If you are unfamiliar with with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
+.. code-block:: bash
+
+    $ vi ~/.lemur/lemur.conf.py
+
+.. note:: If you are unfamiliar with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
       ``postgresql://userame:password@<database-fqdn>:<database-port>/<database-name>``
 
+Before Lemur will run you need to fill in a few required variables in the configuration file:
+
+.. code-block:: bash
+
+    LEMUR_SECURITY_TEAM_EMAIL
+    #/the e-mail address needs to be enclosed in quotes
+    LEMUR_DEFAULT_COUNTRY
+    LEMUR_DEFAULT_STATE
+    LEMUR_DEFAULT_LOCATION
+    LEMUR_DEFAULT_ORGANIZATION
+    LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
 
 Setup Postgres
 --------------
@@ -118,10 +153,9 @@ First, set a password for the postgres user.  For this guide, we will use ``lemu
 
 .. code-block:: bash
 
-    $ sudo -u postgres psql postgres
+    $ sudo -u postgres -i
-    # \password postgres
+    $ psql
-    Enter new password: lemur
+    postgres=# CREATE USER lemur WITH PASSWORD 'lemur';
-    Enter it again: lemur
 
 Once successful, type CTRL-D to exit the Postgres shell.
 
@@ -133,17 +167,11 @@ Next, we will create our new database:
 
 .. _InitializingLemur:
 
-Set a password for lemur user inside Postgres:
+.. note::
-
+    For this guide we assume you will use the `postgres` user to connect to your database, when deploying to a VM or container this is often all you will need. If you have a shared database it is recommend you give Lemur its own user.
-.. code-block:: bash
-
-    $ sudo -u postgres psql postgres
-    \password lemur
-    Enter new password: lemur
-    Enter it again: lemur
-
-Again, enter CTRL-D to exit the Postgres shell.
 
+.. note::
+    Postgres 9.4 or greater is required as Lemur relies advanced data columns (e.g. JSON Column type)
 
 Initializing Lemur
 ------------------
@@ -158,8 +186,10 @@ Additional notifications can be created through the UI or API.  See :ref:`Creati
 
 .. code-block:: bash
 
+    $ cd /www/lemur/lemur
     $ lemur init
 
+
 .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
 
@@ -177,7 +207,7 @@ You'll use the builtin ``HttpProxyModule`` within Nginx to handle proxying.  Edi
 ::
 
    location /api {
-        proxy_pass  http://127.0.0.1:5000;
+        proxy_pass  http://127.0.0.1:8000;
         proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
         proxy_redirect off;
         proxy_buffering off;
@@ -219,7 +249,7 @@ you can pass that via the ``--config`` option.
   # the correct host and port!
   lemur --config=/etc/lemur.conf.py start -b 127.0.0.1:8000
 
-You should now be able to test the web service by visiting ``http://localhost:5000/``.
+You should now be able to test the web service by visiting ``http://localhost:8000/``.
 
 
 Running Lemur as a Service
@@ -241,8 +271,8 @@ Configuring Supervisor couldn't be more simple. Just point it to the ``lemur`` e
   autostart=true
   autorestart=true
   redirect_stderr=true
-  stdout_logfile syslog
+  stdout_logfile=syslog
-  stderr_logfile syslog
+  stderr_logfile=syslog
 
 See :ref:`Using Supervisor <UsingSupervisor>` for more details on using Supervisor.
 
@@ -250,13 +280,14 @@ See :ref:`Using Supervisor <UsingSupervisor>` for more details on using Supervis
 Syncing
 -------
 
-Lemur uses periodic sync tasks to make sure it is up-to-date with its environment. As always, things can change outside of Lemur, but we do our best to reconcile those changes, for example, using Cron:
+Lemur uses periodic sync tasks to make sure it is up-to-date with its environment. Things change outside of Lemur we do our best to reconcile those changes. The recommended method is to use CRON:
 
 .. code-block:: bash
 
   $ crontab -e
-  * 3 * * * lemur sync --all
+  */15 * * * * lemur sync -s all
-  * 3 * * * lemur check_revoked
+  0 22 * * * lemur check_revoked
+  0 22 * * * lemur notify
 
 
 Additional Utilities

docs/requirements.txt

@@ -1,29 +0,0 @@
-Jinja2>=2.3
-Pygments>=1.2
-Sphinx>=1.3
-docutils>=0.7
-markupsafe
-sphinxcontrib-httpdomain
-Flask==0.10.1
-Flask-RESTful==0.3.3
-Flask-SQLAlchemy==2.1
-Flask-Script==2.0.5
-Flask-Migrate==1.6.0
-Flask-Bcrypt==0.7.1
-Flask-Principal==0.4.0
-Flask-Mail==0.9.1
-SQLAlchemy-Utils==0.31.3
-BeautifulSoup4
-requests==2.8.1
-psycopg2==2.6.1
-arrow==0.7.0
-boto==2.38.0  # we might make this optional
-six==1.10.0
-gunicorn==19.4.1
-pycrypto==2.6.1
-cryptography==1.1.1
-pyopenssl==0.15.1
-pyjwt==1.4.0
-xmltodict==0.9.2
-lockfile==0.12.2
-future==0.15.2

docs/security.rst

@@ -60,7 +60,7 @@ and public disclosure may be shortened considerably.
 
 The list of people and organizations who receives advanced notification of
 security issues is not, and will not, be made public. This list generally
-consists of high profile downstream distributors and is entirely at the
+consists of high-profile downstream distributors and is entirely at the
 discretion of the ``lemur`` team.
 
 .. _`master`: https://github.com/Netflix/lemur

gulp/build.js

@@ -1,13 +1,12 @@
+'use strict';
+
 var gulp = require('gulp'),
   minifycss = require('gulp-minify-css'),
   concat = require('gulp-concat'),
   less = require('gulp-less'),
   gulpif = require('gulp-if'),
-  order = require('gulp-order'),
   gutil = require('gulp-util'),
-  rename = require('gulp-rename'),
   foreach = require('gulp-foreach'),
-  debug = require('gulp-debug'),
   path =require('path'),
   merge = require('merge-stream'),
   del = require('del'),
@@ -27,7 +26,8 @@ var gulp = require('gulp'),
   minifyHtml = require('gulp-minify-html'),
   bowerFiles = require('main-bower-files'),
   karma = require('karma'),
-  replace = require('gulp-replace');
+  replace = require('gulp-replace'),
+  argv = require('yargs').argv;
 
 gulp.task('default', ['clean'], function () {
   gulp.start('fonts', 'styles');
@@ -79,8 +79,9 @@ gulp.task('dev:styles', function () {
     'bower_components/angular-loading-bar/src/loading-bar.css',
     'bower_components/angular-ui-switch/angular-ui-switch.css',
     'bower_components/angular-wizard/dist/angular-wizard.css',
-    'bower_components/ng-table/ng-table.css',
+    'bower_components/ng-table/dist/ng-table.css',
     'bower_components/angularjs-toaster/toaster.css',
+    'bower_components/angular-ui-select/dist/select.css',
     'lemur/static/app/styles/lemur.css'
   ];
 
@@ -88,9 +89,9 @@ gulp.task('dev:styles', function () {
     .pipe(gulpif(isBootswatchFile, foreach(function (stream, file) {
       var themeName = path.basename(path.dirname(file.path)),
         content = replaceAll(baseContent, '$theme$', themeName),
-        file = string_src('bootstrap-' +  themeName + '.less', content);
+        file2 = string_src('bootstrap-' +  themeName + '.less', content);
 
-      return file;
+      return file2;
     })))
     .pipe(less())
     .pipe(gulpif(isBootstrapFile, foreach(function (stream, file) {
@@ -100,7 +101,7 @@ gulp.task('dev:styles', function () {
       // http://stackoverflow.com/questions/21719833/gulp-how-to-add-src-files-in-the-middle-of-a-pipe
       // https://github.com/gulpjs/gulp/blob/master/docs/recipes/using-multiple-sources-in-one-task.md
       return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css']))
-        .pipe(concat('style-' + themeName + ".css"));
+        .pipe(concat('style-' + themeName + '.css'));
     })))
     .pipe(plumber())
     .pipe(concat('styles.css'))
@@ -112,7 +113,7 @@ gulp.task('dev:styles', function () {
 
 // http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
 function escapeRegExp(string) {
-  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
 }
 
 function replaceAll(string, find, replace) {
@@ -122,7 +123,7 @@ function replaceAll(string, find, replace) {
 function string_src(filename, string) {
   var src = require('stream').Readable({ objectMode: true });
   src._read = function () {
-    this.push(new gutil.File({ cwd: "", base: "", path: filename, contents: new Buffer(string) }));
+    this.push(new gutil.File({ cwd: '', base: '', path: filename, contents: new Buffer(string) }));
     this.push(null);
   };
   return src;
@@ -143,26 +144,18 @@ gulp.task('build:extras', function () {
 function injectHtml(isDev) {
   return gulp.src('lemur/static/app/index.html')
     .pipe(
-    inject(gulp.src(bowerFiles({ base: 'app' }), {
+    inject(gulp.src(bowerFiles({ base: 'app' })), {
-      read: false
-    }), {
       starttag: '<!-- inject:bower:{{ext}} -->',
       addRootSlash: false,
       ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
     })
   )
-    .pipe(inject(gulp.src(['lemur/static/app/angular/**/*.js'], {
+    .pipe(inject(gulp.src(['lemur/static/app/angular/**/*.js']), {
-      read: false
-    }), {
-      read: false,
       starttag: '<!-- inject:{{ext}} -->',
       addRootSlash: false,
       ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
     }))
-    .pipe(inject(gulp.src(['.tmp/styles/**/*.css'], {
+    .pipe(inject(gulp.src(['.tmp/styles/**/*.css']), {
-      read: false
-    }), {
-      read: false,
       starttag: '<!-- inject:{{ext}} -->',
       addRootSlash: false,
       ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
@@ -170,13 +163,11 @@ function injectHtml(isDev) {
     .pipe(
     gulpif(!isDev,
       inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js'), {
-        read: false,
         starttag: '<!-- inject:ngviews -->',
         addRootSlash: false
       })
     )
-  )
+  ).pipe(gulp.dest('.tmp/'));
-    .pipe(gulp.dest('.tmp/'));
 }
 
 gulp.task('dev:inject', ['dev:styles', 'dev:scripts'], function () {
@@ -199,23 +190,17 @@ gulp.task('build:ngviews', function () {
 });
 
 gulp.task('build:html', ['dev:styles', 'dev:scripts', 'build:ngviews', 'build:inject'], function () {
-  var jsFilter = filter('**/*.js');
+  var jsFilter = filter(['**/*.js'], {'restore': true});
-  var cssFilter = filter('**/*.css');
+  var cssFilter = filter(['**/*.css'], {'restore': true});
-
-  var assets = useref.assets();
 
   return gulp.src('.tmp/index.html')
-    .pipe(assets)
-    .pipe(rev())
     .pipe(jsFilter)
     .pipe(ngAnnotate())
-    .pipe(jsFilter.restore())
+    .pipe(jsFilter.restore)
     .pipe(cssFilter)
     .pipe(csso())
-    .pipe(cssFilter.restore())
+    .pipe(cssFilter.restore)
-    .pipe(assets.restore())
     .pipe(useref())
-    .pipe(revReplace())
     .pipe(gulp.dest('lemur/static/dist'))
     .pipe(size());
 });
@@ -241,10 +226,40 @@ gulp.task('package:strip', function () {
     .pipe(replace('http:\/\/localhost:3000', ''))
     .pipe(replace('http:\/\/localhost:8000', ''))
     .pipe(useref())
-    .pipe(revReplace())
     .pipe(gulp.dest('lemur/static/dist/scripts'))
     .pipe(size());
 });
 
+gulp.task('addUrlContextPath',['addUrlContextPath:revreplace'], function(){
+  var urlContextPathExists = argv.urlContextPath ? true : false;
+  ['lemur/static/dist/scripts/main*.js',
+  'lemur/static/dist/angular/**/*.html']
+  .forEach(function(file){
+    return gulp.src(file)
+      .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
+      .pipe(gulpif(urlContextPathExists, replace('angular/', argv.urlContextPath + '/angular/')))
+      .pipe(gulp.dest(function(file){
+        return file.base;
+      }))
+  })
+});
+
+gulp.task('addUrlContextPath:revision', function(){
+  return gulp.src(['lemur/static/dist/**/*.css','lemur/static/dist/**/*.js'])
+    .pipe(rev())
+    .pipe(gulp.dest('lemur/static/dist'))
+    .pipe(rev.manifest())
+    .pipe(gulp.dest('lemur/static/dist'))
+})
+
+gulp.task('addUrlContextPath:revreplace', ['addUrlContextPath:revision'], function(){
+  var manifest = gulp.src("lemur/static/dist/rev-manifest.json");
+  var urlContextPathExists = argv.urlContextPath ? true : false;
+  return gulp.src( "lemur/static/dist/index.html")
+    .pipe(gulpif(urlContextPathExists, revReplace({prefix: argv.urlContextPath + '/', manifest: manifest}, revReplace({manifest: manifest}))))
+    .pipe(gulp.dest('lemur/static/dist'));
+})
+
+
 gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']);
-gulp.task('package', ['package:strip']);
+gulp.task('package', ['addUrlContextPath', 'package:strip']);

hooks/pre-commit

@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-
-import glob
-import os
-import sys
-
-os.environ['PYFLAKES_NODOCTEST'] = '1'
-
-# pep8.py uses sys.argv to find setup.cfg
-sys.argv = [os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)]
-
-# git usurbs your bin path for hooks and will always run system python
-if 'VIRTUAL_ENV' in os.environ:
-    site_packages = glob.glob(
-        '%s/lib/*/site-packages' % os.environ['VIRTUAL_ENV'])[0]
-    sys.path.insert(0, site_packages)
-
-
-def py_lint(files_modified):
-    from flake8.main import DEFAULT_CONFIG
-    from flake8.engine import get_style_guide
-
-    # remove non-py files and files which no longer exist
-    files_modified = filter(lambda x: x.endswith('.py'), files_modified)
-
-    flake8_style = get_style_guide(parse_argv=True, config_file=DEFAULT_CONFIG)
-    report = flake8_style.check_files(files_modified)
-
-    return report.total_errors != 0
-
-
-def main():
-    from flake8.hooks import run
-
-    gitcmd = "git diff-index --cached --name-only HEAD"
-
-    _, files_modified, _ = run(gitcmd)
-
-    files_modified = filter(lambda x: os.path.exists(x), files_modified)
-
-    if py_lint(files_modified):
-        return 1
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

lemur/__about__.py

@@ -9,10 +9,10 @@ __title__ = "lemur"
 __summary__ = ("Certificate management and orchestration service")
 __uri__ = "https://github.com/Netflix/lemur"
 
-__version__ = "0.2.1"
+__version__ = "0.7.0"
 
 __author__ = "The Lemur developers"
 __email__ = "security@netflix.com"
 
 __license__ = "Apache License, Version 2.0"
-__copyright__ = "Copyright 2015 {0}".format(__author__)
+__copyright__ = "Copyright 2018 {0}".format(__author__)

lemur/__init__.py

@@ -1,16 +1,18 @@
 """
 .. module: lemur
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 
 """
-from __future__ import absolute_import, division, print_function
+import time
+from flask import g, request
 
 from lemur import factory
+from lemur.extensions import metrics
 
 from lemur.users.views import mod as users_bp
 from lemur.roles.views import mod as roles_bp
@@ -23,6 +25,11 @@ from lemur.defaults.views import mod as defaults_bp
 from lemur.plugins.views import mod as plugins_bp
 from lemur.notifications.views import mod as notifications_bp
 from lemur.sources.views import mod as sources_bp
+from lemur.endpoints.views import mod as endpoints_bp
+from lemur.logs.views import mod as logs_bp
+from lemur.api_keys.views import mod as api_key_bp
+from lemur.pending_certificates.views import mod as pending_certificates_bp
+from lemur.dns_providers.views import mod as dns_providers_bp
 
 from lemur.__about__ import (
     __author__, __copyright__, __email__, __license__, __summary__, __title__,
@@ -46,7 +53,12 @@ LEMUR_BLUEPRINTS = (
     defaults_bp,
     plugins_bp,
     notifications_bp,
-    sources_bp
+    sources_bp,
+    endpoints_bp,
+    logs_bp,
+    api_key_bp,
+    pending_certificates_bp,
+    dns_providers_bp,
 )
 
 
@@ -62,16 +74,40 @@ def configure_hook(app):
     :param app:
     :return:
     """
-    from flask.ext.principal import PermissionDenied
+    from flask import jsonify
-    from lemur.decorators import crossdomain
+    from werkzeug.exceptions import HTTPException
-    if app.config.get('CORS'):
+
-        @app.after_request
+    @app.errorhandler(Exception)
-        @crossdomain(origin=u"http://localhost:3000", methods=['PUT', 'HEAD', 'GET', 'POST', 'OPTIONS', 'DELETE'])
+    def handle_error(e):
-        def after(response):
+        code = 500
+        if isinstance(e, HTTPException):
+            code = e.code
+
+        app.logger.exception(e)
+        return jsonify(error=str(e)), code
+
+    @app.before_request
+    def before_request():
+        g.request_start_time = time.time()
+
+    @app.after_request
+    def after_request(response):
+        # Return early if we don't have the start time
+        if not hasattr(g, 'request_start_time'):
             return response
 
-    @app.errorhandler(PermissionDenied)
+        # Get elapsed time in milliseconds
-    def handle_invalid_usage(error):
+        elapsed = time.time() - g.request_start_time
-        response = {'message': 'You are not allow to access this resource'}
+        elapsed = int(round(1000 * elapsed))
-        response.status_code = 403
+
+        # Collect request/response tags
+        tags = {
+            'endpoint': request.endpoint,
+            'request_method': request.method.lower(),
+            'status_code': response.status_code
+        }
+
+        # Record our response time metric
+        metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags)
+        metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
         return response

lemur/analyze/service.py

@@ -1,64 +0,0 @@
-"""
-.. module: lemur.analyze.service
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-"""
-# def analyze(endpoints, truststores):
-#    results = {"headings": ["Endpoint"],
-#               "results": [],
-#               "time": datetime.now().strftime("#Y%m%d %H:%M:%S")}
-#
-#    for store in truststores:
-#        results['headings'].append(os.path.basename(store))
-#
-#    for endpoint in endpoints:
-#        result_row = [endpoint]
-#        for store in truststores:
-#            result = {'details': []}
-#
-#            tests = []
-#            for region, ip in REGIONS.items():
-#                try:
-#                    domain = dns.name.from_text(endpoint)
-#                    if not domain.is_absolute():
-#                        domain = domain.concatenate(dns.name.root)
-#
-#                    my_resolver = dns.resolver.Resolver()
-#                    my_resolver.nameservers = [ip]
-#                    answer = my_resolver.query(domain)
-#
-#                    #force the testing of regional enpoints by changing the dns server
-#                    response = requests.get('https://' + str(answer[0]), verify=store)
-#                    tests.append('pass')
-#                    result['details'].append("{}: SSL testing completed without errors".format(region))
-#
-#                except SSLError as e:
-#                    log.debug(e)
-#                    if 'hostname' in str(e):
-#                        tests.append('pass')
-#                        result['details'].append(
-#                              "{}: This test passed ssl negotiation but failed hostname verification because \
-#                                   the hostname is not included in the certificate".format(region))
-#                    elif 'certificate verify failed' in str(e):
-#                        tests.append('fail')
-#                        result['details'].append("{}: This test failed to verify the SSL certificate".format(region))
-#                    else:
-#                        tests.append('fail')
-#                        result['details'].append("{}: {}".format(region, str(e)))
-#
-#                except Exception as e:
-#                    log.debug(e)
-#                    tests.append('fail')
-#                    result['details'].append("{}: {}".format(region, str(e)))
-#
-#            #any failing tests fails the whole endpoint
-#            if 'fail' in tests:
-#                result['test'] = 'fail'
-#            else:
-#                result['test'] = 'pass'
-#
-#            result_row.append(result)
-#        results['results'].append(result_row)
-#    return results
-#

lemur/api_keys/cli.py

@@ -0,0 +1,41 @@
+"""
+.. module: lemur.api_keys.cli
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from flask_script import Manager
+from lemur.api_keys import service as api_key_service
+from lemur.auth.service import create_token
+
+from datetime import datetime
+
+manager = Manager(usage="Handles all api key related tasks.")
+
+
+@manager.option('-u', '--user-id', dest='uid', help='The User ID this access key belongs too.')
+@manager.option('-n', '--name', dest='name', help='The name of this API Key.')
+@manager.option('-t', '--ttl', dest='ttl', help='The TTL of this API Key. -1 for forever.')
+def create(uid, name, ttl):
+    """
+    Create a new api key for a user.
+    :return:
+    """
+    print("[+] Creating a new api key.")
+    key = api_key_service.create(user_id=uid, name=name,
+        ttl=ttl, issued_at=int(datetime.utcnow().timestamp()), revoked=False)
+    print("[+] Successfully created a new api key. Generating a JWT...")
+    jwt = create_token(uid, key.id, key.ttl)
+    print("[+] Your JWT is: {jwt}".format(jwt=jwt))
+
+
+@manager.option('-a', '--api-key-id', dest='aid', help='The API Key ID to revoke.')
+def revoke(aid):
+    """
+    Revokes an api key for a user.
+    :return:
+    """
+    print("[-] Revoking the API Key api key.")
+    api_key_service.revoke(aid=aid)
+    print("[+] Successfully revoked the api key")

lemur/api_keys/models.py

@@ -0,0 +1,25 @@
+"""
+.. module: lemur.api_keys.models
+    :platform: Unix
+    :synopsis: This module contains all of the models need to create an api key within Lemur.
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from sqlalchemy import BigInteger, Boolean, Column, ForeignKey, Integer, String
+
+from lemur.database import db
+
+
+class ApiKey(db.Model):
+    __tablename__ = 'api_keys'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    user_id = Column(Integer, ForeignKey('users.id'))
+    ttl = Column(BigInteger)
+    issued_at = Column(BigInteger)
+    revoked = Column(Boolean)
+
+    def __repr__(self):
+        return "ApiKey(name={name}, user_id={user_id}, ttl={ttl}, issued_at={iat}, revoked={revoked})".format(
+            user_id=self.user_id, name=self.name, ttl=self.ttl, iat=self.issued_at, revoked=self.revoked)

lemur/api_keys/schemas.py

@@ -0,0 +1,57 @@
+"""
+.. module: lemur.api_keys.schemas
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from flask import g
+from marshmallow import fields
+
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.users.schemas import UserNestedOutputSchema, UserInputSchema
+
+
+def current_user_id():
+    return {'id': g.current_user.id, 'email': g.current_user.email, 'username': g.current_user.username}
+
+
+class ApiKeyInputSchema(LemurInputSchema):
+    name = fields.String(required=False)
+    user = fields.Nested(UserInputSchema, missing=current_user_id, default=current_user_id)
+    ttl = fields.Integer()
+
+
+class ApiKeyRevokeSchema(LemurInputSchema):
+    id = fields.Integer(required=True)
+    name = fields.String()
+    user = fields.Nested(UserInputSchema, required=True)
+    revoked = fields.Boolean()
+    ttl = fields.Integer()
+    issued_at = fields.Integer(required=False)
+
+
+class UserApiKeyInputSchema(LemurInputSchema):
+    name = fields.String(required=False)
+    ttl = fields.Integer()
+
+
+class ApiKeyOutputSchema(LemurOutputSchema):
+    jwt = fields.String()
+
+
+class ApiKeyDescribedOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    name = fields.String()
+    user = fields.Nested(UserNestedOutputSchema)
+    ttl = fields.Integer()
+    issued_at = fields.Integer()
+    revoked = fields.Boolean()
+
+
+api_key_input_schema = ApiKeyInputSchema()
+api_key_revoke_schema = ApiKeyRevokeSchema()
+api_key_output_schema = ApiKeyOutputSchema()
+api_keys_output_schema = ApiKeyDescribedOutputSchema(many=True)
+api_key_described_output_schema = ApiKeyDescribedOutputSchema()
+user_api_key_input_schema = UserApiKeyInputSchema()

lemur/api_keys/service.py

@@ -0,0 +1,97 @@
+"""
+.. module: lemur.api_keys.service
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+"""
+from lemur import database
+from lemur.api_keys.models import ApiKey
+
+
+def get(aid):
+    """
+    Retrieves an api key by its ID.
+    :param aid: The access key id to get.
+    :return:
+    """
+    return database.get(ApiKey, aid)
+
+
+def delete(access_key):
+    """
+    Delete an access key. This is one way to remove a key, though you probably should just set revoked.
+    :param access_key:
+    :return:
+    """
+    database.delete(access_key)
+
+
+def revoke(aid):
+    """
+    Revokes an api key.
+    :param aid:
+    :return:
+    """
+    api_key = get(aid)
+    setattr(api_key, 'revoked', False)
+
+    return database.update(api_key)
+
+
+def get_all_api_keys():
+    """
+    Retrieves all Api Keys.
+    :return:
+    """
+    return ApiKey.query.all()
+
+
+def create(**kwargs):
+    """
+    Creates a new API Key.
+
+    :param kwargs:
+    :return:
+    """
+    api_key = ApiKey(**kwargs)
+    database.create(api_key)
+    return api_key
+
+
+def update(api_key, **kwargs):
+    """
+    Updates an api key.
+    :param api_key:
+    :param kwargs:
+    :return:
+    """
+    for key, value in kwargs.items():
+        setattr(api_key, key, value)
+
+    return database.update(api_key)
+
+
+def render(args):
+    """
+    Helper to parse REST Api requests
+
+    :param args:
+    :return:
+    """
+    query = database.session_query(ApiKey)
+    user_id = args.pop('user_id', None)
+    aid = args.pop('id', None)
+    has_permission = args.pop('has_permission', False)
+    requesting_user_id = args.pop('requesting_user_id')
+
+    if user_id:
+        query = query.filter(ApiKey.user_id == user_id)
+
+    if aid:
+        query = query.filter(ApiKey.id == aid)
+
+    if not has_permission:
+        query = query.filter(ApiKey.user_id == requesting_user_id)
+
+    return database.sort_and_page(query, ApiKey, args)

lemur/api_keys/views.py

@@ -0,0 +1,579 @@
+"""
+.. module: lemur.api_keys.views
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Eric Coan <kungfury@instructure.com>
+
+"""
+from datetime import datetime
+
+from flask import Blueprint, g
+from flask_restful import reqparse, Api
+
+from lemur.api_keys import service
+from lemur.auth.service import AuthenticatedResource, create_token
+from lemur.auth.permissions import ApiKeyCreatorPermission
+
+from lemur.common.schema import validate_schema
+from lemur.common.utils import paginated_parser
+
+from lemur.api_keys.schemas import api_key_input_schema, api_key_revoke_schema, api_key_output_schema, \
+    api_keys_output_schema, api_key_described_output_schema, user_api_key_input_schema
+
+mod = Blueprint('api_keys', __name__)
+api = Api(mod)
+
+
+class ApiKeyList(AuthenticatedResource):
+    """ Defines the 'api_keys' endpoint """
+
+    def __init__(self):
+        super(ApiKeyList, self).__init__()
+
+    @validate_schema(None, api_keys_output_schema)
+    def get(self):
+        """
+        .. http:get:: /keys
+
+           The current list of api keys, that you can see.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [
+                    {
+                      "id": 1,
+                      "name": "custom name",
+                      "user_id": 1,
+                      "ttl": -1,
+                      "issued_at": 12,
+                      "revoked": false
+                    }
+                ],
+                "total": 1
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int default is 1
+           :query count: count number. default is 10
+           :query user_id: a user to filter by.
+           :query id: an access key to filter by.
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        args['has_permission'] = ApiKeyCreatorPermission().can()
+        args['requesting_user_id'] = g.current_user.id
+        return service.render(args)
+
+    @validate_schema(api_key_input_schema, api_key_output_schema)
+    def post(self, data=None):
+        """
+        .. http:post:: /keys
+
+           Creates an API Key.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              POST /keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                "name": "my custom name",
+                "user_id": 1,
+                "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+
+                "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if not ApiKeyCreatorPermission().can():
+            if data['user']['id'] != g.current_user.id:
+                return dict(message="You are not authorized to create tokens for: {0}".format(data['user']['username'])), 403
+
+        access_token = service.create(name=data['name'], user_id=data['user']['id'], ttl=data['ttl'],
+                                      revoked=False, issued_at=int(datetime.utcnow().timestamp()))
+        return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
+
+
+class ApiKeyUserList(AuthenticatedResource):
+    """ Defines the 'keys' endpoint on the 'users' endpoint. """
+
+    def __init__(self):
+        super(ApiKeyUserList, self).__init__()
+
+    @validate_schema(None, api_keys_output_schema)
+    def get(self, user_id):
+        """
+        .. http:get:: /users/:user_id/keys
+
+           The current list of api keys for a user, that you can see.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /users/1/keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [
+                    {
+                      "id": 1,
+                      "name": "custom name",
+                      "user_id": 1,
+                      "ttl": -1,
+                      "issued_at": 12,
+                      "revoked": false
+                    }
+                ],
+                "total": 1
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int default is 1
+           :query count: count number. default is 10
+           :query id: an access key to filter by.
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        args['has_permission'] = ApiKeyCreatorPermission().can()
+        args['requesting_user_id'] = g.current_user.id
+        args['user_id'] = user_id
+        return service.render(args)
+
+    @validate_schema(user_api_key_input_schema, api_key_output_schema)
+    def post(self, user_id, data=None):
+        """
+        .. http:post:: /users/:user_id/keys
+
+           Creates an API Key for a user.
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              POST /users/1/keys HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                "name": "my custom name"
+                "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+
+                "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if not ApiKeyCreatorPermission().can():
+            if user_id != g.current_user.id:
+                return dict(message="You are not authorized to create tokens for: {0}".format(user_id)), 403
+
+        access_token = service.create(name=data['name'], user_id=user_id, ttl=data['ttl'],
+                                      revoked=False, issued_at=int(datetime.utcnow().timestamp()))
+        return dict(jwt=create_token(access_token.user_id, access_token.id, access_token.ttl))
+
+
+class ApiKeys(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(ApiKeys, self).__init__()
+
+    @validate_schema(None, api_key_output_schema)
+    def get(self, aid):
+        """
+        .. http:get:: /keys/1
+
+           Fetch one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    @validate_schema(api_key_revoke_schema, api_key_output_schema)
+    def put(self, aid, data=None):
+        """
+        .. http:put:: /keys/1
+
+           update one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              PUT /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                  "name": "new_name",
+                  "revoked": false,
+                  "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to update this token!"), 403
+
+        service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    def delete(self, aid):
+        """
+        .. http:delete:: /keys/1
+
+           deletes one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              DELETE /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "result": true
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to delete this token!"), 403
+
+        service.delete(access_key)
+        return {'result': True}
+
+
+class UserApiKeys(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(UserApiKeys, self).__init__()
+
+    @validate_schema(None, api_key_output_schema)
+    def get(self, uid, aid):
+        """
+        .. http:get:: /users/1/keys/1
+
+           Fetch one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /users/1/api_keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if uid != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        access_key = service.get(aid)
+
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != uid:
+            return dict(message="You are not authorized to view this token!"), 403
+
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    @validate_schema(api_key_revoke_schema, api_key_output_schema)
+    def put(self, uid, aid, data=None):
+        """
+        .. http:put:: /users/1/keys/1
+
+           update one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              PUT /users/1/keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                  "name": "new_name",
+                  "revoked": false,
+                  "ttl": -1
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "jwt": ""
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if uid != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != uid:
+            return dict(message="You are not authorized to update this token!"), 403
+
+        service.update(access_key, name=data['name'], revoked=data['revoked'], ttl=data['ttl'])
+        return dict(jwt=create_token(access_key.user_id, access_key.id, access_key.ttl))
+
+    def delete(self, uid, aid):
+        """
+        .. http:delete:: /users/1/keys/1
+
+           deletes one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              DELETE /users/1/keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "result": true
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if uid != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != uid:
+            return dict(message="You are not authorized to delete this token!"), 403
+
+        service.delete(access_key)
+        return {'result': True}
+
+
+class ApiKeysDescribed(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(ApiKeysDescribed, self).__init__()
+
+    @validate_schema(None, api_key_described_output_schema)
+    def get(self, aid):
+        """
+        .. http:get:: /keys/1/described
+
+           Fetch one api key
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /keys/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "id": 2,
+                  "name": "hoi",
+                  "user_id": 2,
+                  "ttl": -1,
+                  "issued_at": 1222222,
+                  "revoked": false
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        access_key = service.get(aid)
+        if access_key is None:
+            return dict(message="This token does not exist!"), 404
+
+        if access_key.user_id != g.current_user.id:
+            if not ApiKeyCreatorPermission().can():
+                return dict(message="You are not authorized to view this token!"), 403
+
+        return access_key
+
+
+api.add_resource(ApiKeyList, '/keys', endpoint='api_keys')
+api.add_resource(ApiKeys, '/keys/<int:aid>', endpoint='api_key')
+api.add_resource(ApiKeysDescribed, '/keys/<int:aid>/described', endpoint='api_key_described')
+api.add_resource(ApiKeyUserList, '/users/<int:user_id>/keys', endpoint='user_api_keys')
+api.add_resource(UserApiKeys, '/users/<int:uid>/keys/<int:aid>', endpoint='user_api_key')

lemur/auth/ldap.py

@@ -0,0 +1,187 @@
+"""
+.. module: lemur.auth.ldap
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Ian Stahnke <ian.stahnke@myob.com>
+"""
+import ldap
+
+from flask import current_app
+
+from lemur.users import service as user_service
+from lemur.roles import service as role_service
+from lemur.common.utils import validate_conf, get_psuedo_random_string
+
+
+class LdapPrincipal():
+    """
+    Provides methods for authenticating against an LDAP server.
+    """
+    def __init__(self, args):
+        self._ldap_validate_conf()
+        # setup ldap config
+        if not args['username']:
+            raise Exception("missing ldap username")
+        if not args['password']:
+            self.error_message = "missing ldap password"
+            raise Exception("missing ldap password")
+        self.ldap_principal = args['username']
+        self.ldap_email_domain = current_app.config.get("LDAP_EMAIL_DOMAIN", None)
+        if '@' not in self.ldap_principal:
+            self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
+        self.ldap_username = args['username']
+        if '@' in self.ldap_username:
+            self.ldap_username = args['username'].split("@")[0]
+        self.ldap_password = args['password']
+        self.ldap_server = current_app.config.get('LDAP_BIND_URI', None)
+        self.ldap_base_dn = current_app.config.get("LDAP_BASE_DN", None)
+        self.ldap_use_tls = current_app.config.get("LDAP_USE_TLS", False)
+        self.ldap_cacert_file = current_app.config.get("LDAP_CACERT_FILE", None)
+        self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None)
+        self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None)
+        self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None)
+        self.ldap_attrs = ['memberOf']
+        self.ldap_client = None
+        self.ldap_groups = None
+
+    def _update_user(self, roles):
+        """
+        create or update a local user instance.
+        """
+        # try to get user from local database
+        user = user_service.get_by_email(self.ldap_principal)
+
+        # create them a local account
+        if not user:
+            user = user_service.create(
+                self.ldap_username,
+                get_psuedo_random_string(),
+                self.ldap_principal,
+                True,
+                '',     # thumbnailPhotoUrl
+                list(roles)
+            )
+        else:
+            # we add 'lemur' specific roles, so they do not get marked as removed
+            for ur in user.roles:
+                if not ur.third_party:
+                    roles.add(ur)
+
+            # update any changes to the user
+            user_service.update(
+                user.id,
+                self.ldap_username,
+                self.ldap_principal,
+                user.active,
+                user.profile_picture,
+                list(roles)
+            )
+        return user
+
+    def _authorize(self):
+        """
+        check groups and roles to confirm access.
+        return a list of roles if ok.
+        raise an exception on error.
+        """
+        if not self.ldap_principal:
+            return None
+
+        if self.ldap_required_group:
+            # ensure the user has the required group in their group list
+            if self.ldap_required_group not in self.ldap_groups:
+                return None
+
+        roles = set()
+        if self.ldap_default_role:
+            role = role_service.get_by_name(self.ldap_default_role)
+            if role:
+                if not role.third_party:
+                    role = role.set_third_party(role.id, third_party_status=True)
+                roles.add(role)
+
+        # update their 'roles'
+        role = role_service.get_by_name(self.ldap_principal)
+        if not role:
+            description = "auto generated role based on owner: {0}".format(self.ldap_principal)
+            role = role_service.create(self.ldap_principal, description=description,
+                third_party=True)
+        if not role.third_party:
+            role = role_service.set_third_party(role.id, third_party_status=True)
+        roles.add(role)
+        if not self.ldap_groups_to_roles:
+            return roles
+
+        for ldap_group_name, role_name in self.ldap_groups_to_roles.items():
+            role = role_service.get_by_name(role_name)
+            if role:
+                if ldap_group_name in self.ldap_groups:
+                    current_app.logger.debug("assigning role {0} to ldap user {1}".format(self.ldap_principal, role))
+                    if not role.third_party:
+                        role = role_service.set_third_party(role.id, third_party_status=True)
+                    roles.add(role)
+        return roles
+
+    def authenticate(self):
+        """
+        orchestrate the ldap login.
+        raise an exception on error.
+        """
+        self._bind()
+        roles = self._authorize()
+        if not roles:
+            raise Exception('ldap authorization failed')
+        return self._update_user(roles)
+
+    def _bind(self):
+        """
+        authenticate an ldap user.
+        list groups for a user.
+        raise an exception on error.
+        """
+        if '@' not in self.ldap_principal:
+            self.ldap_principal = '%s@%s' % (self.ldap_principal, self.ldap_email_domain)
+        ldap_filter = 'userPrincipalName=%s' % self.ldap_principal
+
+        # query ldap for auth
+        try:
+            # build a client
+            if not self.ldap_client:
+                self.ldap_client = ldap.initialize(self.ldap_server)
+            # perform a synchronous bind
+            self.ldap_client.set_option(ldap.OPT_REFERRALS, 0)
+            if self.ldap_use_tls:
+                ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
+                self.ldap_client.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
+                self.ldap_client.set_option(ldap.OPT_X_TLS, ldap.OPT_X_TLS_DEMAND)
+                self.ldap_client.set_option(ldap.OPT_X_TLS_DEMAND, True)
+                self.ldap_client.set_option(ldap.OPT_DEBUG_LEVEL, 255)
+            if self.ldap_cacert_file:
+                self.ldap_client.set_option(ldap.OPT_X_TLS_CACERTFILE, self.ldap_cacert_file)
+            self.ldap_client.simple_bind_s(self.ldap_principal, self.ldap_password)
+        except ldap.INVALID_CREDENTIALS:
+            self.ldap_client.unbind()
+            raise Exception('The supplied ldap credentials are invalid')
+        except ldap.SERVER_DOWN:
+            raise Exception('ldap server unavailable')
+        except ldap.LDAPError as e:
+            raise Exception("ldap error: {0}".format(e))
+
+        lgroups = self.ldap_client.search_s(self.ldap_base_dn,
+                        ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf']
+        # lgroups is a list of utf-8 encoded strings
+        # convert to a single string of groups to allow matching
+        self.ldap_groups = b''.join(lgroups).decode('ascii')
+        self.ldap_client.unbind()
+
+    def _ldap_validate_conf(self):
+        """
+        Confirms required ldap config settings exist.
+        """
+        required_vars = [
+            'LDAP_BIND_URI',
+            'LDAP_BASE_DN',
+            'LDAP_EMAIL_DOMAIN',
+        ]
+        validate_conf(current_app, required_vars)

lemur/auth/permissions.py

@@ -2,43 +2,50 @@
 .. module: lemur.auth.permissions
     :platform: Unix
     :synopsis: This module defines all the permission used within Lemur
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from functools import partial
 from collections import namedtuple
 
-from flask.ext.principal import Permission, RoleNeed
+from flask_principal import Permission, RoleNeed
 
 # Permissions
 operator_permission = Permission(RoleNeed('operator'))
 admin_permission = Permission(RoleNeed('admin'))
 
-CertificateCreator = namedtuple('certificate', ['method', 'value'])
+CertificateOwner = namedtuple('certificate', ['method', 'value'])
-CertificateCreatorNeed = partial(CertificateCreator, 'key')
+CertificateOwnerNeed = partial(CertificateOwner, 'role')
 
 
-class ViewKeyPermission(Permission):
+class SensitiveDomainPermission(Permission):
-    def __init__(self, certificate_id, owner):
+    def __init__(self):
-        c_need = CertificateCreatorNeed(certificate_id)
+        super(SensitiveDomainPermission, self).__init__(RoleNeed('admin'))
-        super(ViewKeyPermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
 
 
-class UpdateCertificatePermission(Permission):
+class CertificatePermission(Permission):
-    def __init__(self, certificate_id, owner):
+    def __init__(self, owner, roles):
-        c_need = CertificateCreatorNeed(certificate_id)
+        needs = [RoleNeed('admin'), RoleNeed(owner), RoleNeed('creator')]
-        super(UpdateCertificatePermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
+        for r in roles:
+            needs.append(CertificateOwnerNeed(str(r)))
+
+        super(CertificatePermission, self).__init__(*needs)
 
 
-RoleUser = namedtuple('role', ['method', 'value'])
+class ApiKeyCreatorPermission(Permission):
-ViewRoleCredentialsNeed = partial(RoleUser, 'roleView')
+    def __init__(self):
+        super(ApiKeyCreatorPermission, self).__init__(RoleNeed('admin'))
 
 
-class ViewRoleCredentialsPermission(Permission):
+RoleMember = namedtuple('role', ['method', 'value'])
+RoleMemberNeed = partial(RoleMember, 'member')
+
+
+class RoleMemberPermission(Permission):
     def __init__(self, role_id):
-        need = ViewRoleCredentialsNeed(role_id)
+        needs = [RoleNeed('admin'), RoleMemberNeed(role_id)]
-        super(ViewRoleCredentialsPermission, self).__init__(need, RoleNeed('admin'))
+        super(RoleMemberPermission, self).__init__(*needs)
 
 
 AuthorityCreator = namedtuple('authority', ['method', 'value'])

lemur/auth/service.py

@@ -3,16 +3,13 @@
     :platform: Unix
     :synopsis: This module contains all of the authentication duties for
     lemur
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from __future__ import unicode_literals
-from builtins import bytes
 import jwt
 import json
-import base64
 import binascii
 
 from functools import wraps
@@ -20,31 +17,18 @@ from datetime import datetime, timedelta
 
 from flask import g, current_app, jsonify, request
 
-from flask.ext.restful import Resource
+from flask_restful import Resource
-from flask.ext.principal import identity_loaded, RoleNeed, UserNeed
+from flask_principal import identity_loaded, RoleNeed, UserNeed
 
-from flask.ext.principal import Identity, identity_changed
+from flask_principal import Identity, identity_changed
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicNumbers
 
 from lemur.users import service as user_service
-from lemur.auth.permissions import CertificateCreatorNeed, \
+from lemur.api_keys import service as api_key_service
-    AuthorityCreatorNeed, ViewRoleCredentialsNeed
+from lemur.auth.permissions import AuthorityCreatorNeed, RoleMemberNeed
-
-
-def base64url_decode(data):
-    rem = len(data) % 4
-
-    if rem > 0:
-        data += '=' * (4 - rem)
-
-    return base64.urlsafe_b64decode(bytes(data.encode('latin-1')))
-
-
-def base64url_encode(data):
-    return base64.urlsafe_b64encode(data).replace('=', '')
 
 
 def get_rsa_public_key(n, e):
@@ -55,8 +39,9 @@ def get_rsa_public_key(n, e):
     :param e:
     :return: a RSA Public Key in PEM format
     """
-    n = int(binascii.hexlify(base64url_decode(n)), 16)
+    n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, 'utf-8'))), 16)
-    e = int(binascii.hexlify(base64url_decode(e)), 16)
+    e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, 'utf-8'))), 16)
+
     pub = RSAPublicNumbers(e, n).public_key(default_backend())
     return pub.public_bytes(
         encoding=serialization.Encoding.PEM,
@@ -64,9 +49,9 @@ def get_rsa_public_key(n, e):
     )
 
 
-def create_token(user):
+def create_token(user, aid=None, ttl=None):
     """
-    Create a valid JWT for a given user, this token is then used to authenticate
+    Create a valid JWT for a given user/api key, this token is then used to authenticate
     sessions until the token expires.
 
     :param user:
@@ -74,17 +59,31 @@ def create_token(user):
     """
     expiration_delta = timedelta(days=int(current_app.config.get('LEMUR_TOKEN_EXPIRATION', 1)))
     payload = {
-        'sub': user.id,
+        'iat': datetime.utcnow(),
-        'iat': datetime.now(),
+        'exp': datetime.utcnow() + expiration_delta
-        'exp': datetime.now() + expiration_delta
     }
+
+    # Handle Just a User ID & User Object.
+    if isinstance(user, int):
+        payload['sub'] = user
+    else:
+        payload['sub'] = user.id
+    if aid is not None:
+        payload['aid'] = aid
+    # Custom TTLs are only supported on Access Keys.
+    if ttl is not None and aid is not None:
+        # Tokens that are forever until revoked.
+        if ttl == -1:
+            del payload['exp']
+        else:
+            payload['exp'] = ttl
     token = jwt.encode(payload, current_app.config['LEMUR_TOKEN_SECRET'])
     return token.decode('unicode_escape')
 
 
 def login_required(f):
     """
-    Validates the JWT and ensures that is has not expired.
+    Validates the JWT and ensures that is has not expired and the user is still active.
 
     :param f:
     :return:
@@ -110,7 +109,22 @@ def login_required(f):
         except jwt.InvalidTokenError:
             return dict(message='Token is invalid'), 403
 
-        g.current_user = user_service.get(payload['sub'])
+        if 'aid' in payload:
+            access_key = api_key_service.get(payload['aid'])
+            if access_key.revoked:
+                return dict(message='Token has been revoked'), 403
+            if access_key.ttl != -1:
+                current_time = datetime.utcnow()
+                expired_time = datetime.fromtimestamp(access_key.issued_at + access_key.ttl)
+                if current_time >= expired_time:
+                    return dict(message='Token has expired'), 403
+
+        user = user_service.get(payload['sub'])
+
+        if not user.active:
+            return dict(message='User is not currently active'), 403
+
+        g.current_user = user
 
         if not g.current_user:
             return dict(message='You are not logged in'), 403
@@ -138,13 +152,10 @@ def fetch_token_header(token):
         raise jwt.DecodeError('Not enough segments')
 
     try:
-        return json.loads(base64url_decode(header_segment))
+        return json.loads(jwt.utils.base64url_decode(header_segment).decode('utf-8'))
     except TypeError as e:
         current_app.logger.exception(e)
         raise jwt.DecodeError('Invalid header padding')
-    except binascii.Error as e:
-        current_app.logger.exception(e)
-        raise jwt.DecodeError('Invalid header padding')
 
 
 @identity_loaded.connect
@@ -165,19 +176,14 @@ def on_identity_loaded(sender, identity):
     # identity with the roles that the user provides
     if hasattr(user, 'roles'):
         for role in user.roles:
-            identity.provides.add(ViewRoleCredentialsNeed(role.id))
             identity.provides.add(RoleNeed(role.name))
+            identity.provides.add(RoleMemberNeed(role.id))
 
     # apply ownership for authorities
     if hasattr(user, 'authorities'):
         for authority in user.authorities:
             identity.provides.add(AuthorityCreatorNeed(authority.id))
 
-    # apply ownership of certificates
-    if hasattr(user, 'certificates'):
-        for certificate in user.certificates:
-            identity.provides.add(CertificateCreatorNeed(certificate.id))
-
     g.user = user
 
 

lemur/auth/views.py

@@ -1,7 +1,7 @@
 """
 .. module: lemur.auth.views
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
@@ -9,22 +9,191 @@ import jwt
 import base64
 import requests
 
-from flask import g, Blueprint, current_app
+from flask import Blueprint, current_app
 
-from flask.ext.restful import reqparse, Resource, Api
+from flask_restful import reqparse, Resource, Api
-from flask.ext.principal import Identity, identity_changed
+from flask_principal import Identity, identity_changed
 
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
+from lemur.extensions import metrics
 from lemur.common.utils import get_psuedo_random_string
 
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
 from lemur.auth.service import create_token, fetch_token_header, get_rsa_public_key
+import lemur.auth.ldap as ldap
 
 
 mod = Blueprint('auth', __name__)
 api = Api(mod)
 
 
+def exchange_for_access_token(code, redirect_uri, client_id, secret, access_token_url=None, verify_cert=True):
+    """
+    Exchanges authorization code for access token.
+
+    :param code:
+    :param redirect_uri:
+    :param client_id:
+    :param secret:
+    :param access_token_url:
+    :param verify_cert:
+    :return:
+    :return:
+    """
+    # take the information we have received from the provider to create a new request
+    params = {
+        'grant_type': 'authorization_code',
+        'scope': 'openid email profile address',
+        'code': code,
+        'redirect_uri': redirect_uri,
+        'client_id': client_id
+    }
+
+    # the secret and cliendId will be given to you when you signup for the provider
+    token = '{0}:{1}'.format(client_id, secret)
+
+    basic = base64.b64encode(bytes(token, 'utf-8'))
+    headers = {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'authorization': 'basic {0}'.format(basic.decode('utf-8'))
+    }
+
+    # exchange authorization code for access token.
+    r = requests.post(access_token_url, headers=headers, params=params, verify=verify_cert)
+    if r.status_code == 400:
+        r = requests.post(access_token_url, headers=headers, data=params, verify=verify_cert)
+    id_token = r.json()['id_token']
+    access_token = r.json()['access_token']
+
+    return id_token, access_token
+
+
+def validate_id_token(id_token, client_id, jwks_url):
+    """
+    Ensures that the token we receive is valid.
+
+    :param id_token:
+    :param client_id:
+    :param jwks_url:
+    :return:
+    """
+    # fetch token public key
+    header_data = fetch_token_header(id_token)
+
+    # retrieve the key material as specified by the token header
+    r = requests.get(jwks_url)
+    for key in r.json()['keys']:
+        if key['kid'] == header_data['kid']:
+            secret = get_rsa_public_key(key['n'], key['e'])
+            algo = header_data['alg']
+            break
+    else:
+        return dict(message='Key not found'), 401
+
+    # validate your token based on the key it was signed with
+    try:
+        jwt.decode(id_token, secret.decode('utf-8'), algorithms=[algo], audience=client_id)
+    except jwt.DecodeError:
+        return dict(message='Token is invalid'), 401
+    except jwt.ExpiredSignatureError:
+        return dict(message='Token has expired'), 401
+    except jwt.InvalidTokenError:
+        return dict(message='Token is invalid'), 401
+
+
+def retrieve_user(user_api_url, access_token):
+    """
+    Fetch user information from provided user api_url.
+
+    :param user_api_url:
+    :param access_token:
+    :return:
+    """
+    user_params = dict(access_token=access_token, schema='profile')
+
+    # retrieve information about the current user.
+    r = requests.get(user_api_url, params=user_params)
+    profile = r.json()
+
+    user = user_service.get_by_email(profile['email'])
+    return user, profile
+
+
+def create_user_roles(profile):
+    """Creates new roles based on profile information.
+
+    :param profile:
+    :return:
+    """
+    roles = []
+
+    # update their google 'roles'
+    for group in profile['googleGroups']:
+        role = role_service.get_by_name(group)
+        if not role:
+            role = role_service.create(group, description='This is a google group based role created by Lemur', third_party=True)
+        if not role.third_party:
+            role = role_service.set_third_party(role.id, third_party_status=True)
+        roles.append(role)
+
+    role = role_service.get_by_name(profile['email'])
+
+    if not role:
+        role = role_service.create(profile['email'], description='This is a user specific role', third_party=True)
+    if not role.third_party:
+        role = role_service.set_third_party(role.id, third_party_status=True)
+
+    roles.append(role)
+
+    # every user is an operator (tied to a default role)
+    if current_app.config.get('LEMUR_DEFAULT_ROLE'):
+        default = role_service.get_by_name(current_app.config['LEMUR_DEFAULT_ROLE'])
+        if not default:
+            default = role_service.create(current_app.config['LEMUR_DEFAULT_ROLE'], description='This is the default Lemur role.')
+        if not default.third_party:
+            role_service.set_third_party(default.id, third_party_status=True)
+        roles.append(default)
+
+    return roles
+
+
+def update_user(user, profile, roles):
+    """Updates user with current profile information and associated roles.
+
+    :param user:
+    :param profile:
+    :param roles:
+    """
+
+    # if we get an sso user create them an account
+    if not user:
+        user = user_service.create(
+            profile['email'],
+            get_psuedo_random_string(),
+            profile['email'],
+            True,
+            profile.get('thumbnailPhotoUrl'),
+            roles
+        )
+
+    else:
+        # we add 'lemur' specific roles, so they do not get marked as removed
+        for ur in user.roles:
+            if not ur.third_party:
+                roles.append(ur)
+
+        # update any changes to the user
+        user_service.update(
+            user.id,
+            profile['email'],
+            profile['email'],
+            True,
+            profile.get('thumbnailPhotoUrl'),  # profile isn't google+ enabled
+            roles
+        )
+
+
 class Login(Resource):
     """
     Provides an endpoint for Lemur's basic authentication. It takes a username and password
@@ -92,32 +261,54 @@ class Login(Resource):
         else:
             user = user_service.get_by_username(args['username'])
 
-        if user and user.check_password(args['password']):
+        # default to local authentication
+        if user and user.check_password(args['password']) and user.active:
             # Tell Flask-Principal the identity changed
             identity_changed.send(current_app._get_current_object(),
                                   identity=Identity(user.id))
+
+            metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
             return dict(token=create_token(user))
 
-        return dict(message='The supplied credentials are invalid'), 401
+        # try ldap login
+        if current_app.config.get("LDAP_AUTH"):
+            try:
+                ldap_principal = ldap.LdapPrincipal(args)
+                user = ldap_principal.authenticate()
+                if user and user.active:
+                    # Tell Flask-Principal the identity changed
+                    identity_changed.send(current_app._get_current_object(),
+                                  identity=Identity(user.id))
+                    metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+                    return dict(token=create_token(user))
+            except Exception as e:
+                    current_app.logger.error("ldap error: {0}".format(e))
+                    ldap_message = 'ldap error: %s' % e
+                    metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+                    return dict(message=ldap_message), 403
 
-    def get(self):
+        # if not valid user - no certificates for you
-        return {'username': g.current_user.username, 'roles': [r.name for r in g.current_user.roles]}
+        metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+        return dict(message='The supplied credentials are invalid'), 403
 
 
 class Ping(Resource):
     """
     This class serves as an example of how one might implement an SSO provider for use with Lemur. In
-    this example we use a OpenIDConnect authentication flow, that is essentially OAuth2 underneath. If you have an
+    this example we use an OpenIDConnect authentication flow, that is essentially OAuth2 underneath. If you have an
     OAuth2 provider you want to use Lemur there would be two steps:
 
-    1. Define your own class that inherits from :class:`flask.ext.restful.Resource` and create the HTTP methods the \
+    1. Define your own class that inherits from :class:`flask_restful.Resource` and create the HTTP methods the \
-    provider uses for it's callbacks.
+    provider uses for its callbacks.
     2. Add or change the Lemur AngularJS Configuration to point to your new provider
     """
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Ping, self).__init__()
 
+    def get(self):
+        return 'Redirecting...'
+
     def post(self):
         self.reqparse.add_argument('clientId', type=str, required=True, location='json')
         self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
@@ -125,108 +316,85 @@ class Ping(Resource):
 
         args = self.reqparse.parse_args()
 
-        # take the information we have received from the provider to create a new request
-        params = {
-            'client_id': args['clientId'],
-            'grant_type': 'authorization_code',
-            'scope': 'openid email profile address',
-            'redirect_uri': args['redirectUri'],
-            'code': args['code']
-        }
-
         # you can either discover these dynamically or simply configure them
         access_token_url = current_app.config.get('PING_ACCESS_TOKEN_URL')
         user_api_url = current_app.config.get('PING_USER_API_URL')
 
-        # the secret and cliendId will be given to you when you signup for the provider
+        secret = current_app.config.get('PING_SECRET')
-        basic = base64.b64encode('{0}:{1}'.format(args['clientId'], current_app.config.get("PING_SECRET")))
-        headers = {'Authorization': 'Basic {0}'.format(basic)}
 
-        # exchange authorization code for access token.
+        id_token, access_token = exchange_for_access_token(
+            args['code'],
+            args['redirectUri'],
+            args['clientId'],
+            secret,
+            access_token_url=access_token_url
+        )
 
-        r = requests.post(access_token_url, headers=headers, params=params)
-        id_token = r.json()['id_token']
-        access_token = r.json()['access_token']
-
-        # fetch token public key
-        header_data = fetch_token_header(id_token)
         jwks_url = current_app.config.get('PING_JWKS_URL')
+        validate_id_token(id_token, args['clientId'], jwks_url)
 
-        # retrieve the key material as specified by the token header
+        user, profile = retrieve_user(user_api_url, access_token)
-        r = requests.get(jwks_url)
+        roles = create_user_roles(profile)
-        for key in r.json()['keys']:
+        update_user(user, profile, roles)
-            if key['kid'] == header_data['kid']:
-                secret = get_rsa_public_key(key['n'], key['e'])
-                algo = header_data['alg']
-                break
-        else:
-            return dict(message='Key not found'), 403
 
-        # validate your token based on the key it was signed with
+        if not user or not user.active:
-        try:
+            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
-            jwt.decode(id_token, secret, algorithms=[algo], audience=args['clientId'])
+            return dict(message='The supplied credentials are invalid'), 403
-        except jwt.DecodeError:
-            return dict(message='Token is invalid'), 403
-        except jwt.ExpiredSignatureError:
-            return dict(message='Token has expired'), 403
-        except jwt.InvalidTokenError:
-            return dict(message='Token is invalid'), 403
-
-        user_params = dict(access_token=access_token, schema='profile')
-
-        # retrieve information about the current user.
-        r = requests.get(user_api_url, params=user_params)
-        profile = r.json()
-
-        user = user_service.get_by_email(profile['email'])
-
-        # update their google 'roles'
-        roles = []
-
-        for group in profile['googleGroups']:
-            role = role_service.get_by_name(group)
-            if not role:
-                role = role_service.create(group, description='This is a google group based role created by Lemur')
-            roles.append(role)
-
-        # if we get an sso user create them an account
-        # we still pick a random password in case sso is down
-        if not user:
-
-            # every user is an operator (tied to a default role)
-            if current_app.config.get('LEMUR_DEFAULT_ROLE'):
-                v = role_service.get_by_name(current_app.config.get('LEMUR_DEFAULT_ROLE'))
-                if v:
-                    roles.append(v)
-
-            user = user_service.create(
-                profile['email'],
-                get_psuedo_random_string(),
-                profile['email'],
-                True,
-                profile.get('thumbnailPhotoUrl'),
-                roles
-            )
-
-        else:
-            # we add 'lemur' specific roles, so they do not get marked as removed
-            for ur in user.roles:
-                if ur.authority_id:
-                    roles.append(ur)
-
-            # update any changes to the user
-            user_service.update(
-                user.id,
-                profile['email'],
-                profile['email'],
-                True,
-                profile.get('thumbnailPhotoUrl'),  # incase profile isn't google+ enabled
-                roles
-            )
 
         # Tell Flask-Principal the identity changed
         identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
 
+        metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+        return dict(token=create_token(user))
+
+
+class OAuth2(Resource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(OAuth2, self).__init__()
+
+    def get(self):
+        return 'Redirecting...'
+
+    def post(self):
+        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
+        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
+        self.reqparse.add_argument('code', type=str, required=True, location='json')
+
+        args = self.reqparse.parse_args()
+
+        # you can either discover these dynamically or simply configure them
+        access_token_url = current_app.config.get('OAUTH2_ACCESS_TOKEN_URL')
+        user_api_url = current_app.config.get('OAUTH2_USER_API_URL')
+        verify_cert = current_app.config.get('OAUTH2_VERIFY_CERT')
+
+        secret = current_app.config.get('OAUTH2_SECRET')
+
+        id_token, access_token = exchange_for_access_token(
+            args['code'],
+            args['redirectUri'],
+            args['clientId'],
+            secret,
+            access_token_url=access_token_url,
+            verify_cert=verify_cert
+        )
+
+        jwks_url = current_app.config.get('PING_JWKS_URL')
+        validate_id_token(id_token, args['clientId'], jwks_url)
+
+        user, profile = retrieve_user(user_api_url, access_token)
+        roles = create_user_roles(profile)
+        update_user(user, profile, roles)
+
+        if not user.active:
+            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+            return dict(message='The supplied credentials are invalid'), 403
+
+        # Tell Flask-Principal the identity changed
+        identity_changed.send(current_app._get_current_object(), identity=Identity(user.id))
+
+        metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+
         return dict(token=create_token(user))
 
 
@@ -265,15 +433,22 @@ class Google(Resource):
 
         user = user_service.get_by_email(profile['email'])
 
+        if not (user and user.active):
+            metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+            return dict(message='The supplied credentials are invalid.'), 403
+
         if user:
+            metrics.send('login', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
             return dict(token=create_token(user))
 
+        metrics.send('login', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+
 
 class Providers(Resource):
     def get(self):
         active_providers = []
 
-        for provider in current_app.config.get("ACTIVE_PROVIDERS"):
+        for provider in current_app.config.get("ACTIVE_PROVIDERS", []):
             provider = provider.lower()
 
             if provider == "google":
@@ -291,16 +466,33 @@ class Providers(Resource):
                     'clientId': current_app.config.get("PING_CLIENT_ID"),
                     'responseType': 'code',
                     'scope': ['openid', 'email', 'profile', 'address'],
-                    'scopeDelimeter': ' ',
+                    'scopeDelimiter': ' ',
                     'authorizationEndpoint': current_app.config.get("PING_AUTH_ENDPOINT"),
                     'requiredUrlParams': ['scope'],
                     'type': '2.0'
                 })
 
+            elif provider == "oauth2":
+                active_providers.append({
+                    'name': current_app.config.get("OAUTH2_NAME"),
+                    'url': current_app.config.get('OAUTH2_REDIRECT_URI'),
+                    'redirectUri': current_app.config.get("OAUTH2_REDIRECT_URI"),
+                    'clientId': current_app.config.get("OAUTH2_CLIENT_ID"),
+                    'responseType': 'code',
+                    'scope': ['openid', 'email', 'profile', 'groups'],
+                    'scopeDelimiter': ' ',
+                    'authorizationEndpoint': current_app.config.get("OAUTH2_AUTH_ENDPOINT"),
+                    'requiredUrlParams': ['scope', 'state', 'nonce'],
+                    'state': 'STATE',
+                    'nonce': get_psuedo_random_string(),
+                    'type': '2.0'
+                })
+
         return active_providers
 
 
 api.add_resource(Login, '/auth/login', endpoint='login')
 api.add_resource(Ping, '/auth/ping', endpoint='ping')
 api.add_resource(Google, '/auth/google', endpoint='google')
+api.add_resource(OAuth2, '/auth/oauth2', endpoint='oauth2')
 api.add_resource(Providers, '/auth/providers', endpoint='providers')

lemur/authorities/models.py

@@ -1,58 +1,52 @@
 """
 .. module: lemur.authorities.models
     :platform: unix
-    :synopsis: This module contains all of the models need to create a authority within Lemur.
+    :synopsis: This module contains all of the models need to create an authority within Lemur.
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
-
 from sqlalchemy.orm import relationship
 from sqlalchemy import Column, Integer, String, Text, func, ForeignKey, DateTime, PassiveDefault, Boolean
 from sqlalchemy.dialects.postgresql import JSON
 
 from lemur.database import db
-from lemur.certificates.models import get_cn, get_not_after, get_not_before
+from lemur.plugins.base import plugins
+from lemur.models import roles_authorities
 
 
 class Authority(db.Model):
     __tablename__ = 'authorities'
     id = Column(Integer, primary_key=True)
-    owner = Column(String(128))
+    owner = Column(String(128), nullable=False)
     name = Column(String(128), unique=True)
     body = Column(Text())
     chain = Column(Text())
-    bits = Column(Integer())
-    cn = Column(String(128))
-    not_before = Column(DateTime)
-    not_after = Column(DateTime)
     active = Column(Boolean, default=True)
-    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
     plugin_name = Column(String(64))
     description = Column(Text)
     options = Column(JSON)
-    roles = relationship('Role', backref=db.backref('authority'), lazy='dynamic')
+    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
+    roles = relationship('Role', secondary=roles_authorities, passive_deletes=True, backref=db.backref('authority'), lazy='dynamic')
     user_id = Column(Integer, ForeignKey('users.id'))
-    certificates = relationship("Certificate", backref='authority')
+    authority_certificate = relationship("Certificate", backref='root_authority', uselist=False, foreign_keys='Certificate.root_authority_id')
+    certificates = relationship("Certificate", backref='authority', foreign_keys='Certificate.authority_id')
 
-    def __init__(self, name, owner, plugin_name, body, roles=None, chain=None, description=None):
+    authority_pending_certificate = relationship("PendingCertificate", backref='root_authority', uselist=False, foreign_keys='PendingCertificate.root_authority_id')
-        self.name = name
+    pending_certificates = relationship('PendingCertificate', backref='authority', foreign_keys='PendingCertificate.authority_id')
-        self.body = body
-        self.chain = chain
-        self.owner = owner
-        self.plugin_name = plugin_name
-        cert = x509.load_pem_x509_certificate(str(body), default_backend())
-        self.cn = get_cn(cert)
-        self.not_before = get_not_before(cert)
-        self.not_after = get_not_after(cert)
-        self.roles = roles
-        self.description = description
 
-    def as_dict(self):
+    def __init__(self, **kwargs):
-        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+        self.owner = kwargs['owner']
+        self.roles = kwargs.get('roles', [])
+        self.name = kwargs.get('name')
+        self.description = kwargs.get('description')
+        self.authority_certificate = kwargs['authority_certificate']
+        self.plugin_name = kwargs['plugin']['slug']
+        self.options = kwargs.get('options')
 
-    def serialize(self):
+    @property
-        blob = self.as_dict()
+    def plugin(self):
-        return blob
+        return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Authority(name={name})".format(name=self.name)

lemur/authorities/schemas.py

@@ -0,0 +1,119 @@
+"""
+.. module: lemur.authorities.schemas
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import current_app
+
+from marshmallow import fields, validates_schema, pre_load
+from marshmallow import validate
+from marshmallow.exceptions import ValidationError
+
+from lemur.schemas import PluginInputSchema, PluginOutputSchema, ExtensionSchema, AssociatedAuthoritySchema, AssociatedRoleSchema
+from lemur.users.schemas import UserNestedOutputSchema
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.common import validators, missing
+
+from lemur.common.fields import ArrowDateTime
+
+
+class AuthorityInputSchema(LemurInputSchema):
+    name = fields.String(required=True)
+    owner = fields.Email(required=True)
+    description = fields.String()
+    common_name = fields.String(required=True, validate=validators.common_name)
+
+    validity_start = ArrowDateTime()
+    validity_end = ArrowDateTime()
+    validity_years = fields.Integer()
+
+    # certificate body fields
+    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
+    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
+    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
+    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
+    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+
+    plugin = fields.Nested(PluginInputSchema)
+
+    # signing related options
+    type = fields.String(validate=validate.OneOf(['root', 'subca']), missing='root')
+    parent = fields.Nested(AssociatedAuthoritySchema)
+    signing_algorithm = fields.String(validate=validate.OneOf(['sha256WithRSA', 'sha1WithRSA']), missing='sha256WithRSA')
+    key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')
+    key_name = fields.String()
+    sensitivity = fields.String(validate=validate.OneOf(['medium', 'high']), missing='medium')
+    serial_number = fields.Integer()
+    first_serial = fields.Integer(missing=1)
+
+    extensions = fields.Nested(ExtensionSchema)
+
+    roles = fields.Nested(AssociatedRoleSchema(many=True))
+
+    @validates_schema
+    def validate_dates(self, data):
+        validators.dates(data)
+
+    @validates_schema
+    def validate_subca(self, data):
+        if data['type'] == 'subca':
+            if not data.get('parent'):
+                raise ValidationError("If generating a subca, parent 'authority' must be specified.")
+
+    @pre_load
+    def ensure_dates(self, data):
+        return missing.convert_validity_years(data)
+
+
+class AuthorityUpdateSchema(LemurInputSchema):
+    owner = fields.Email(required=True)
+    description = fields.String()
+    active = fields.Boolean(missing=True)
+    roles = fields.Nested(AssociatedRoleSchema(many=True))
+
+
+class RootAuthorityCertificateOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    active = fields.Boolean()
+    bits = fields.Integer()
+    body = fields.String()
+    chain = fields.String()
+    description = fields.String()
+    name = fields.String()
+    cn = fields.String()
+    not_after = fields.DateTime()
+    not_before = fields.DateTime()
+    owner = fields.Email()
+    status = fields.Boolean()
+    user = fields.Nested(UserNestedOutputSchema)
+
+
+class AuthorityOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    plugin = fields.Nested(PluginOutputSchema)
+    active = fields.Boolean()
+    options = fields.Dict()
+    roles = fields.List(fields.Nested(AssociatedRoleSchema))
+    authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema)
+
+
+class AuthorityNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    plugin = fields.Nested(PluginOutputSchema)
+    active = fields.Boolean()
+
+
+authority_update_schema = AuthorityUpdateSchema()
+authority_input_schema = AuthorityInputSchema()
+authority_output_schema = AuthorityOutputSchema()
+authorities_output_schema = AuthorityOutputSchema(many=True)

lemur/authorities/service.py

@@ -3,108 +3,121 @@
     :platform: Unix
     :synopsis: This module contains all of the services level functions used to
     administer authorities in Lemur
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from flask import g
+
-from flask import current_app
+import json
 
 from lemur import database
+from lemur.common.utils import truthiness
+from lemur.extensions import metrics
 from lemur.authorities.models import Authority
 from lemur.roles import service as role_service
-from lemur.notifications import service as notification_service
 
-from lemur.roles.models import Role
+from lemur.certificates.service import upload
-from lemur.certificates.models import Certificate
-
-from lemur.plugins.base import plugins
 
 
-def update(authority_id, description=None, owner=None, active=None, roles=None):
+def update(authority_id, description, owner, active, roles):
     """
-    Update a an authority with new values.
+    Update an authority with new values.
 
     :param authority_id:
     :param roles: roles that are allowed to use this authority
-    :rtype : Authority
     :return:
     """
     authority = get(authority_id)
-    if roles:
-        authority = database.update_list(authority, 'roles', Role, roles)
-
-    if active:
-        authority.active = active
 
+    authority.roles = roles
+    authority.active = active
     authority.description = description
     authority.owner = owner
+
     return database.update(authority)
 
 
-def create(kwargs):
+def mint(**kwargs):
     """
-    Create a new authority.
+    Creates the authority based on the plugin provided.
+    """
+    issuer = kwargs['plugin']['plugin_object']
+    values = issuer.create_authority(kwargs)
 
-    :rtype : Authority
+    # support older plugins
+    if len(values) == 3:
+        body, chain, roles = values
+        private_key = None
+    elif len(values) == 4:
+        body, private_key, chain, roles = values
+
+    roles = create_authority_roles(roles, kwargs['owner'], kwargs['plugin']['plugin_object'].title, kwargs['creator'])
+    return body, private_key, chain, roles
+
+
+def create_authority_roles(roles, owner, plugin_title, creator):
+    """
+    Creates all of the necessary authority roles.
+    :param creator:
+    :param roles:
     :return:
     """
-
-    issuer = plugins.get(kwargs.get('pluginName'))
-
-    kwargs['creator'] = g.current_user.email
-    cert_body, intermediate, issuer_roles = issuer.create_authority(kwargs)
-
-    cert = Certificate(cert_body, chain=intermediate)
-    cert.owner = kwargs['ownerEmail']
-
-    if kwargs['caType'] == 'subca':
-        cert.description = "This is the ROOT certificate for the {0} sub certificate authority the parent \
-                                authority is {1}.".format(kwargs.get('caName'), kwargs.get('caParent'))
-    else:
-        cert.description = "This is the ROOT certificate for the {0} certificate authority.".format(
-            kwargs.get('caName')
-        )
-
-    cert.user = g.current_user
-
-    cert.notifications = notification_service.create_default_expiration_notifications(
-        'DEFAULT_SECURITY',
-        current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
-    )
-
-    # we create and attach any roles that the issuer gives us
     role_objs = []
-    for r in issuer_roles:
+    for r in roles:
-
+        role = role_service.get_by_name(r['name'])
-        role = role_service.create(
+        if not role:
-            r['name'],
+            role = role_service.create(
-            password=r['password'],
+                r['name'],
-            description="{0} auto generated role".format(kwargs.get('pluginName')),
+                password=r['password'],
-            username=r['username'])
+                description="Auto generated role for {0}".format(plugin_title),
+                username=r['username'])
 
         # the user creating the authority should be able to administer it
         if role.username == 'admin':
-            g.current_user.roles.append(role)
+            creator.roles.append(role)
 
         role_objs.append(role)
 
-    authority = Authority(
+    # create an role for the owner and assign it
-        kwargs.get('caName'),
+    owner_role = role_service.get_by_name(owner)
-        kwargs['ownerEmail'],
+    if not owner_role:
-        kwargs['pluginName'],
+        owner_role = role_service.create(
-        cert_body,
+            owner,
-        description=kwargs['caDescription'],
+            description="Auto generated role based on owner: {0}".format(owner)
-        chain=intermediate,
+        )
-        roles=role_objs
-    )
 
-    database.update(cert)
+    role_objs.append(owner_role)
+    return role_objs
+
+
+def create(**kwargs):
+    """
+    Creates a new authority.
+    """
+    body, private_key, chain, roles = mint(**kwargs)
+
+    kwargs['creator'].roles = list(set(list(kwargs['creator'].roles) + roles))
+
+    kwargs['body'] = body
+    kwargs['private_key'] = private_key
+    kwargs['chain'] = chain
+
+    if kwargs.get('roles'):
+        kwargs['roles'] += roles
+    else:
+        kwargs['roles'] = roles
+
+    cert = upload(**kwargs)
+    kwargs['authority_certificate'] = cert
+    if kwargs.get('plugin', {}).get('plugin_options', []):
+        kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options'])
+
+    authority = Authority(**kwargs)
     authority = database.create(authority)
+    kwargs['creator'].authorities.append(authority)
 
-    g.current_user.authorities.append(authority)
+    metrics.send('authority_created', 'counter', 1, metric_tags=dict(owner=authority.owner))
-
     return authority
 
 
@@ -123,7 +136,6 @@ def get(authority_id):
     """
     Retrieves an authority given it's ID
 
-    :rtype : Authority
     :param authority_id:
     :return:
     """
@@ -135,28 +147,22 @@ def get_by_name(authority_name):
     Retrieves an authority given it's name.
 
     :param authority_name:
-    :rtype : Authority
     :return:
     """
     return database.get(Authority, authority_name, field='name')
 
 
-def get_authority_role(ca_name):
+def get_authority_role(ca_name, creator=None):
     """
     Attempts to get the authority role for a given ca uses current_user
     as a basis for accomplishing that.
 
     :param ca_name:
     """
-    if g.current_user.is_admin:
+    if creator:
-        authority = get_by_name(ca_name)
+        if creator.is_admin:
-        # TODO we should pick admin ca roles for admin
+            return role_service.get_by_name("{0}_admin".format(ca_name))
-        return authority.roles[0]
+    return role_service.get_by_name("{0}_operator".format(ca_name))
-    else:
-        for role in g.current_user.roles:
-            if role.authority:
-                if role.authority.name == ca_name:
-                    return role
 
 
 def render(args):
@@ -166,30 +172,24 @@ def render(args):
     :return:
     """
     query = database.session_query(Authority)
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
 
     if filt:
         terms = filt.split(';')
-        if 'active' in filt:  # this is really weird but strcmp seems to not work here??
+        if 'active' in filt:
-            query = query.filter(Authority.active == terms[1])
+            query = query.filter(Authority.active == truthiness(terms[1]))
         else:
             query = database.filter(query, Authority, terms)
 
-    # we make sure that a user can only use an authority they either own are are a member of - admins can see all
+    # we make sure that a user can only use an authority they either own are a member of - admins can see all
-    if not g.current_user.is_admin:
+    if not args['user'].is_admin:
         authority_ids = []
-        for role in g.current_user.roles:
+        for authority in args['user'].authorities:
-            if role.authority:
+            authority_ids.append(authority.id)
-                authority_ids.append(role.authority.id)
+
+        for role in args['user'].roles:
+            for authority in role.authorities:
+                authority_ids.append(authority.id)
         query = query.filter(Authority.id.in_(authority_ids))
 
-    query = database.find_all(query, Authority, args)
+    return database.sort_and_page(query, Authority, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Authority, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)

lemur/authorities/views.py

@@ -1,36 +1,23 @@
 """
 .. module: lemur.authorities.views
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import Blueprint, g
-from flask.ext.restful import reqparse, fields, Api
+from flask_restful import reqparse, Api
 
-from lemur.authorities import service
+from lemur.common.utils import paginated_parser
-from lemur.roles import service as role_service
+from lemur.common.schema import validate_schema
-from lemur.certificates import service as certificate_service
 from lemur.auth.service import AuthenticatedResource
-
 from lemur.auth.permissions import AuthorityPermission
 
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.certificates import service as certificate_service
 
+from lemur.authorities import service
+from lemur.authorities.schemas import authority_input_schema, authority_output_schema, authorities_output_schema, authority_update_schema
 
-FIELDS = {
-    'name': fields.String,
-    'owner': fields.String,
-    'description': fields.String,
-    'options': fields.Raw,
-    'pluginName': fields.String,
-    'body': fields.String,
-    'chain': fields.String,
-    'active': fields.Boolean,
-    'notBefore': fields.DateTime(dt_format='iso8601', attribute='not_before'),
-    'notAfter': fields.DateTime(dt_format='iso8601', attribute='not_after'),
-    'id': fields.Integer,
-}
 
 mod = Blueprint('authorities', __name__)
 api = Api(mod)
@@ -42,7 +29,7 @@ class AuthoritiesList(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(AuthoritiesList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, authorities_output_schema)
     def get(self):
         """
         .. http:get:: /authorities
@@ -66,28 +53,52 @@ class AuthoritiesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
+                "items": [{
-                    {
+                    "name": "TestAuthority",
-                      "id": 1,
+                    "roles": [{
-                      "name": "authority1",
+                        "id": 123,
-                      "description": "this is authority1",
+                        "name": "secure@example.com"
-                      "pluginName": null,
+                    }, {
-                      "chain": "-----Begin ...",
+                        "id": 564,
-                      "body": "-----Begin ...",
+                        "name": "TestAuthority_admin"
-                      "active": true,
+                    }, {
-                      "notBefore": "2015-06-05T17:09:39",
+                        "id": 565,
-                      "notAfter": "2015-06-10T17:09:39"
+                        "name": "TestAuthority_operator"
-                      "options": null
+                    }],
-                    }
+                    "options": null,
-                  ]
+                    "active": true,
+                    "authorityCertificate": {
+                        "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
+                        "status": true,
+                        "cn": "AcommonName",
+                        "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                        "chain": "",
+                        "notBefore": "2016-06-02T00:00:15+00:00",
+                        "notAfter": "2023-06-02T23:59:15+00:00",
+                        "owner": "secure@example.com",
+                        "user": {
+                            "username": "joe@example.com",
+                            "active": true,
+                            "email": "joe@example.com",
+                            "id": 3
+                        },
+                        "active": true,
+                        "bits": 2048,
+                        "id": 2235,
+                        "name": "TestAuthority"
+                    },
+                    "owner": "secure@example.com",
+                    "id": 43,
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority."
+                }
                 "total": 1
               }
 
            :query sortBy: field to sort on
-           :query sortDir: acs or desc
+           :query sortDir: asc or desc
-           :query page: int. default is 1
+           :query page: int default is 1
-           :query filter: key value pair. format is k=v;
+           :query filter: key value pair. format is k;v
-           :query limit: limit number. default is 10
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
@@ -96,10 +107,11 @@ class AuthoritiesList(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
+        args['user'] = g.current_user
         return service.render(args)
 
-    @marshal_items(FIELDS)
+    @validate_schema(authority_input_schema, authority_output_schema)
-    def post(self):
+    def post(self, data=None):
         """
         .. http:post:: /authorities
 
@@ -113,31 +125,30 @@ class AuthoritiesList(AuthenticatedResource):
               Host: example.com
               Accept: application/json, text/javascript
 
-              {
+             {
-                "caDN": {
+                "country": "US",
-                  "country": "US",
+                "state": "California",
-                  "state": "CA",
+                "location": "Los Gatos",
-                  "location": "A Location",
+                "organization": "Netflix",
-                  "organization": "ExampleInc",
+                "organizationalUnit": "Operations",
-                  "organizationalUnit": "Operations",
+                "type": "root",
-                  "commonName": "a common name"
+                "signingAlgorithm": "sha256WithRSA",
-                },
+                "sensitivity": "medium",
-                "caType": "root",
-                "caSigningAlgo": "sha256WithRSA",
-                "caSensitivity": "medium",
                 "keyType": "RSA2048",
-                "pluginName": "cloudca",
+                "plugin": {
-                "validityStart": "2015-06-11T07:00:00.000Z",
+                    "slug": "cloudca-issuer",
-                "validityEnd": "2015-06-13T07:00:00.000Z",
-                "caName": "DoctestCA",
-                "ownerEmail": "jimbob@example.com",
-                "caDescription": "Example CA",
-                "extensions": {
-                  "subAltNames": {
-                    "names": []
-                  }
                 },
-              }
+                "name": "TimeTestAuthority5",
+                "owner": "secure@example.com",
+                "description": "test",
+                "commonName": "AcommonName",
+                "validityYears": "20",
+                "extensions": {
+                    "subAltNames": {
+                        "names": []
+                    },
+                    "custom": []
+             }
 
            **Example response**:
 
@@ -148,57 +159,68 @@ class AuthoritiesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
+                "name": "TestAuthority",
-                "name": "authority1",
+                "roles": [{
-                "description": "this is authority1",
+                    "id": 123,
-                "pluginName": null,
+                    "name": "secure@example.com"
-                "chain": "-----Begin ...",
+                }, {
-                "body": "-----Begin ...",
+                    "id": 564,
+                    "name": "TestAuthority_admin"
+                }, {
+                    "id": 565,
+                    "name": "TestAuthority_operator"
+                }],
+                "options": null,
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
+                "authorityCertificate": {
-                "notAfter": "2015-06-10T17:09:39"
+                    "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
-                "options": null
+                    "status": true,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-02T00:00:15+00:00",
+                    "notAfter": "2023-06-02T23:59:15+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2235,
+                    "name": "TestAuthority"
+                },
+                "owner": "secure@example.com",
+                "id": 43,
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
-           :arg caName: authority's name
+
-           :arg caDescription: a sensible description about what the CA with be used for
+           :arg name: authority's name
-           :arg ownerEmail: the team or person who 'owns' this authority
+           :arg description: a sensible description about what the CA with be used for
+           :arg owner: the team or person who 'owns' this authority
            :arg validityStart: when this authority should start issuing certificates
            :arg validityEnd: when this authority should stop issuing certificates
+           :arg validityYears: starting from `now` how many years into the future the authority should be valid
            :arg extensions: certificate extensions
-           :arg pluginName: name of the plugin to create the authority
+           :arg plugin: name of the plugin to create the authority
-           :arg caType: the type of authority (root/subca)
+           :arg type: the type of authority (root/subca)
-           :arg caParent: the parent authority if this is to be a subca
+           :arg parent: the parent authority if this is to be a subca
-           :arg caSigningAlgo: algorithm used to sign the authority
+           :arg signingAlgorithm: algorithm used to sign the authority
            :arg keyType: key type
-           :arg caSensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored
+           :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored
            in an HSM
-           :arg caKeyName: name of the key to store in the HSM (CloudCA)
+           :arg keyName: name of the key to store in the HSM (CloudCA)
-           :arg caSerialNumber: serial number of the authority
+           :arg serialNumber: serial number of the authority
-           :arg caFirstSerial: specifies the starting serial number for certificates issued off of this authority
+           :arg firstSerial: specifies the starting serial number for certificates issued off of this authority
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 403: unauthenticated
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('caName', type=str, location='json', required=True)
+        data['creator'] = g.current_user
-        self.reqparse.add_argument('caDescription', type=str, location='json', required=False)
+        return service.create(**data)
-        self.reqparse.add_argument('ownerEmail', type=str, location='json', required=True)
-        self.reqparse.add_argument('caDN', type=dict, location='json', required=False)
-        self.reqparse.add_argument('validityStart', type=str, location='json', required=False)  # TODO validate
-        self.reqparse.add_argument('validityEnd', type=str, location='json', required=False)  # TODO validate
-        self.reqparse.add_argument('extensions', type=dict, location='json', required=False)
-        self.reqparse.add_argument('pluginName', type=str, location='json', required=True)
-        self.reqparse.add_argument('caType', type=str, location='json', required=False)
-        self.reqparse.add_argument('caParent', type=str, location='json', required=False)
-        self.reqparse.add_argument('caSigningAlgo', type=str, location='json', required=False)
-        self.reqparse.add_argument('keyType', type=str, location='json', required=False)
-        self.reqparse.add_argument('caSensitivity', type=str, location='json', required=False)
-        self.reqparse.add_argument('caKeyName', type=str, location='json', required=False)
-        self.reqparse.add_argument('caSerialNumber', type=int, location='json', required=False)
-        self.reqparse.add_argument('caFirstSerial', type=int, location='json', required=False)
-
-        args = self.reqparse.parse_args()
-        return service.create(args)
 
 
 class Authorities(AuthenticatedResource):
@@ -206,7 +228,7 @@ class Authorities(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Authorities, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, authority_output_schema)
     def get(self, authority_id):
         """
         .. http:get:: /authorities/1
@@ -230,30 +252,40 @@ class Authorities(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
+                "roles": [{
-                "name": "authority1",
+                    "id": 123,
-                "description": "this is authority1",
+                    "name": "secure@example.com"
-                "pluginName": null,
+                }, {
-                "chain": "-----Begin ...",
+                    "id": 564,
-                "body": "-----Begin ...",
+                    "name": "TestAuthority_admin"
+                }, {
+                    "id": 565,
+                    "name": "TestAuthority_operator"
+                }],
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
+                "owner": "secure@example.com",
-                "notAfter": "2015-06-10T17:09:39"
+                "id": 43,
-                "options": null
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
+           :arg description: a sensible description about what the CA with be used for
+           :arg owner: the team or person who 'owns' this authority
+           :arg active: set whether this authoritity is currently in use
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 403: unauthenticated
+           :statuscode 200: no error
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
         return service.get(authority_id)
 
-    @marshal_items(FIELDS)
+    @validate_schema(authority_update_schema, authority_output_schema)
-    def put(self, authority_id):
+    def put(self, authority_id, data=None):
         """
         .. http:put:: /authorities/1
 
-           Update a authority
+           Update an authority
 
            **Example request**:
 
@@ -264,11 +296,42 @@ class Authorities(AuthenticatedResource):
               Accept: application/json, text/javascript
 
               {
-                 "roles": [],
+                "name": "TestAuthority5",
-                 "active": false,
+                "roles": [{
-                 "owner": "bob@example.com",
+                    "id": 566,
-                 "description": "this is authority1"
+                    "name": "TestAuthority5_admin"
-              }
+                }, {
+                    "id": 567,
+                    "name": "TestAuthority5_operator"
+                }, {
+                    "id": 123,
+                    "name": "secure@example.com"
+                }],
+                "active": true,
+                "authorityCertificate": {
+                    "body": "-----BEGIN CERTIFICATE-----",
+                    "status": null,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority5 certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-03T00:00:51+00:00",
+                    "notAfter": "2036-06-03T23:59:51+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2280,
+                    "name": "TestAuthority5"
+                },
+                "owner": "secure@example.com",
+                "id": 44,
+                "description": "This is the ROOT certificate for the TestAuthority5 certificate authority."
+               }
 
            **Example response**:
 
@@ -279,64 +342,74 @@ class Authorities(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
+                "name": "TestAuthority",
-                "name": "authority1",
+                "roles": [{
-                "description": "this is authority1",
+                    "id": 123,
-                "pluginName": null,
+                    "name": "secure@example.com"
-                "chain": "-----begin ...",
+                }, {
-                "body": "-----begin ...",
+                    "id": 564,
-                "active": false,
+                    "name": "TestAuthority_admin"
-                "notBefore": "2015-06-05t17:09:39",
+                }, {
-                "notAfter": "2015-06-10t17:09:39"
+                    "id": 565,
-                "options": null
+                    "name": "TestAuthority_operator"
+                }],
+                "options": null,
+                "active": true,
+                "authorityCertificate": {
+                    "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
+                    "status": true,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-02T00:00:15+00:00",
+                    "notAfter": "2023-06-02T23:59:15+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2235,
+                    "name": "TestAuthority"
+                },
+                "owner": "secure@example.com",
+                "id": 43,
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
-        self.reqparse.add_argument('roles', type=list, default=[], location='json')
-        self.reqparse.add_argument('active', type=str, location='json', required=True)
-        self.reqparse.add_argument('owner', type=str, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json', required=True)
-        args = self.reqparse.parse_args()
-
         authority = service.get(authority_id)
-        role = role_service.get_by_name(authority.owner)
+
+        if not authority:
+            return dict(message='Not Found'), 404
 
         # all the authority role members should be allowed
         roles = [x.name for x in authority.roles]
-
-        # allow "owner" roles by team DL
-        roles.append(role)
         permission = AuthorityPermission(authority_id, roles)
 
-        # we want to make sure that we cannot add roles that we are not members of
-        if not g.current_user.is_admin:
-            role_ids = set([r['id'] for r in args['roles']])
-            user_role_ids = set([r.id for r in g.current_user.roles])
-
-            if not role_ids.issubset(user_role_ids):
-                return dict(message="You are not allowed to associate a role which you are not a member of"), 400
-
         if permission.can():
             return service.update(
                 authority_id,
-                owner=args['owner'],
+                owner=data['owner'],
-                description=args['description'],
+                description=data['description'],
-                active=args['active'],
+                active=data['active'],
-                roles=args['roles']
+                roles=data['roles']
             )
 
-        return dict(message="You are not authorized to update this authority"), 403
+        return dict(message="You are not authorized to update this authority."), 403
 
 
 class CertificateAuthority(AuthenticatedResource):
     def __init__(self):
-        self.reqparse = reqparse.RequestParser()
         super(CertificateAuthority, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, authority_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/authority
@@ -360,16 +433,42 @@ class CertificateAuthority(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
+                "name": "TestAuthority",
-                "name": "authority1",
+                "roles": [{
-                "description": "this is authority1",
+                    "id": 123,
-                "pluginName": null,
+                    "name": "secure@example.com"
-                "chain": "-----Begin ...",
+                }, {
-                "body": "-----Begin ...",
+                    "id": 564,
+                    "name": "TestAuthority_admin"
+                }, {
+                    "id": 565,
+                    "name": "TestAuthority_operator"
+                }],
+                "options": null,
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
+                "authorityCertificate": {
-                "notAfter": "2015-06-10T17:09:39"
+                    "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
-                "options": null
+                    "status": true,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-02T00:00:15+00:00",
+                    "notAfter": "2023-06-02T23:59:15+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2235,
+                    "name": "TestAuthority"
+                },
+                "owner": "secure@example.com",
+                "id": 43,
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
            :reqheader Authorization: OAuth token to authenticate
@@ -378,10 +477,36 @@ class CertificateAuthority(AuthenticatedResource):
         """
         cert = certificate_service.get(certificate_id)
         if not cert:
-            return dict(message="Certificate not found"), 404
+            return dict(message="Certificate not found."), 404
 
         return cert.authority
 
+
+class AuthorityVisualizations(AuthenticatedResource):
+    def get(self, authority_id):
+        """
+        {"name": "flare",
+        "children": [
+            {
+                "name": "analytics",
+                "children": [
+                    {
+                        "name": "cluster",
+                        "children": [
+                            {"name": "AgglomerativeCluster", "size": 3938},
+                            {"name": "CommunityStructure", "size": 3812},
+                            {"name": "HierarchicalCluster", "size": 6714},
+                            {"name": "MergeEdge", "size": 743}
+                        ]
+                    }
+            }
+        ]}
+        """
+        authority = service.get(authority_id)
+        return dict(name=authority.name, children=[{"name": c.name} for c in authority.certificates])
+
+
 api.add_resource(AuthoritiesList, '/authorities', endpoint='authorities')
 api.add_resource(Authorities, '/authorities/<int:authority_id>', endpoint='authority')
+api.add_resource(AuthorityVisualizations, '/authorities/<int:authority_id>/visualize', endpoint='authority_visualizations')
 api.add_resource(CertificateAuthority, '/certificates/<int:certificate_id>/authority', endpoint='certificateAuthority')

lemur/authorizations/models.py

@@ -0,0 +1,34 @@
+"""
+.. module: lemur.authorizations.models
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Netflix Secops <secops@netflix.com>
+"""
+from sqlalchemy import Column, Integer, String
+from sqlalchemy_utils import JSONType
+from lemur.database import db
+
+from lemur.plugins.base import plugins
+
+
+class Authorization(db.Model):
+    __tablename__ = 'pending_dns_authorizations'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    account_number = Column(String(128))
+    domains = Column(JSONType)
+    dns_provider_type = Column(String(128))
+    options = Column(JSONType)
+
+    @property
+    def plugin(self):
+        return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Authorization(id={id})".format(label=self.id)
+
+    def __init__(self, account_number, domains, dns_provider_type, options=None):
+        self.account_number = account_number
+        self.domains = domains
+        self.dns_provider_type = dns_provider_type
+        self.options = options

lemur/authorizations/service.py

@@ -0,0 +1,24 @@
+"""
+.. module: lemur.pending_certificates.service
+    Copyright (c) 2018 and onwards Netflix, Inc.  All rights reserved.
+.. moduleauthor:: Secops <secops@netflix.com>
+"""
+from lemur import database
+
+from lemur.authorizations.models import Authorization
+
+
+def get(authorization_id):
+    """
+    Retrieve dns authorization by ID
+    """
+    return database.get(Authorization, authorization_id)
+
+
+def create(account_number, domains, dns_provider_type, options=None):
+    """
+    Creates a new dns authorization.
+    """
+
+    authorization = Authorization(account_number, domains, dns_provider_type, options)
+    return database.create(authorization)

lemur/certificates/cli.py

@@ -0,0 +1,373 @@
+"""
+.. module: lemur.certificate.cli
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+import sys
+import multiprocessing
+from tabulate import tabulate
+from sqlalchemy import or_
+
+from flask import current_app
+
+from flask_script import Manager
+from flask_principal import Identity, identity_changed
+
+
+from lemur import database
+from lemur.extensions import sentry
+from lemur.extensions import metrics
+from lemur.plugins.base import plugins
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
+from lemur.deployment import service as deployment_service
+from lemur.endpoints import service as endpoint_service
+from lemur.notifications.messaging import send_rotation_notification
+from lemur.domains.models import Domain
+from lemur.authorities.models import Authority
+from lemur.certificates.schemas import CertificateOutputSchema
+from lemur.certificates.models import Certificate
+from lemur.certificates.service import (
+    reissue_certificate,
+    get_certificate_primitives,
+    get_all_pending_reissue,
+    get_by_name,
+    get_all_certs,
+    get
+)
+
+from lemur.certificates.verify import verify_string
+
+manager = Manager(usage="Handles all certificate related tasks.")
+
+
+def print_certificate_details(details):
+    """
+    Print the certificate details with formatting.
+    :param details:
+    :return:
+    """
+    details, errors = CertificateOutputSchema().dump(details)
+    print("[+] Re-issuing certificate with the following details: ")
+    print(
+        "\t[+] Common Name: {common_name}\n"
+        "\t[+] Subject Alternate Names: {sans}\n"
+        "\t[+] Authority: {authority_name}\n"
+        "\t[+] Validity Start: {validity_start}\n"
+        "\t[+] Validity End: {validity_end}\n".format(
+            common_name=details['commonName'],
+            sans=",".join(x['value'] for x in details['extensions']['subAltNames']['names']) or None,
+            authority_name=details['authority']['name'],
+            validity_start=details['validityStart'],
+            validity_end=details['validityEnd']
+        )
+    )
+
+
+def validate_certificate(certificate_name):
+    """
+    Ensuring that the specified certificate exists.
+    :param certificate_name:
+    :return:
+    """
+    if certificate_name:
+        cert = get_by_name(certificate_name)
+
+        if not cert:
+            print("[-] No certificate found with name: {0}".format(certificate_name))
+            sys.exit(1)
+
+        return cert
+
+
+def validate_endpoint(endpoint_name):
+    """
+    Ensuring that the specified endpoint exists.
+    :param endpoint_name:
+    :return:
+    """
+    if endpoint_name:
+        endpoint = endpoint_service.get_by_name(endpoint_name)
+
+        if not endpoint:
+            print("[-] No endpoint found with name: {0}".format(endpoint_name))
+            sys.exit(1)
+
+        return endpoint
+
+
+def request_rotation(endpoint, certificate, message, commit):
+    """
+    Rotates a certificate and handles any exceptions during
+    execution.
+    :param endpoint:
+    :param certificate:
+    :param message:
+    :param commit:
+    :return:
+    """
+    status = FAILURE_METRIC_STATUS
+    if commit:
+        try:
+            deployment_service.rotate_certificate(endpoint, certificate)
+
+            if message:
+                send_rotation_notification(certificate)
+
+            status = SUCCESS_METRIC_STATUS
+
+        except Exception as e:
+            print(
+                "[!] Failed to rotate endpoint {0} to certificate {1} reason: {2}".format(
+                    endpoint.name,
+                    certificate.name,
+                    e
+                )
+            )
+
+    metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': status})
+
+
+def request_reissue(certificate, commit):
+    """
+    Reissuing certificate and handles any exceptions.
+    :param certificate:
+    :param commit:
+    :return:
+    """
+    status = FAILURE_METRIC_STATUS
+    try:
+        print("[+] {0} is eligible for re-issuance".format(certificate.name))
+
+        # set the lemur identity for all cli commands
+        identity_changed.send(current_app._get_current_object(), identity=Identity(1))
+
+        details = get_certificate_primitives(certificate)
+        print_certificate_details(details)
+
+        if commit:
+            new_cert = reissue_certificate(certificate, replace=True)
+            print("[+] New certificate named: {0}".format(new_cert.name))
+
+        status = SUCCESS_METRIC_STATUS
+
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.exception("Error reissuing certificate.", exc_info=True)
+        print(
+            "[!] Failed to reissue certificates. Reason: {}".format(
+                e
+            )
+        )
+
+    metrics.send('certificate_reissue', 'counter', 1, metric_tags={'status': status})
+
+
+@manager.option('-e', '--endpoint', dest='endpoint_name', help='Name of the endpoint you wish to rotate.')
+@manager.option('-n', '--new-certificate', dest='new_certificate_name', help='Name of the certificate you wish to rotate to.')
+@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to rotate.')
+@manager.option('-a', '--notify', dest='message', action='store_true', help='Send a rotation notification to the certificates owner.')
+@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, commit):
+    """
+    Rotates an endpoint and reissues it if it has not already been replaced. If it has
+    been replaced, will use the replacement certificate for the rotation.
+    """
+    if commit:
+        print("[!] Running in COMMIT mode.")
+
+    print("[+] Starting endpoint rotation.")
+
+    status = FAILURE_METRIC_STATUS
+
+    try:
+        old_cert = validate_certificate(old_certificate_name)
+        new_cert = validate_certificate(new_certificate_name)
+        endpoint = validate_endpoint(endpoint_name)
+
+        if endpoint and new_cert:
+            print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name))
+            request_rotation(endpoint, new_cert, message, commit)
+
+        elif old_cert and new_cert:
+            print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name))
+
+            for endpoint in old_cert.endpoints:
+                print("[+] Rotating {0}".format(endpoint.name))
+                request_rotation(endpoint, new_cert, message, commit)
+
+        else:
+            print("[+] Rotating all endpoints that have new certificates available")
+            for endpoint in endpoint_service.get_all_pending_rotation():
+                if len(endpoint.certificate.replaced) == 1:
+                    print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name))
+                    request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
+                else:
+                    metrics.send('endpoint_rotation', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+                    print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format(
+                        endpoint.name
+                    ))
+
+        status = SUCCESS_METRIC_STATUS
+        print("[+] Done!")
+
+    except Exception as e:
+        sentry.captureException()
+
+    metrics.send('endpoint_rotation_job', 'counter', 1, metric_tags={'status': status})
+
+
+@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to reissue.')
+@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+def reissue(old_certificate_name, commit):
+    """
+    Reissues certificate with the same parameters as it was originally issued with.
+    If not time period is provided, reissues certificate as valid from today to
+    today + length of original.
+    """
+    if commit:
+        print("[!] Running in COMMIT mode.")
+
+    print("[+] Starting certificate re-issuance.")
+
+    status = FAILURE_METRIC_STATUS
+
+    try:
+        old_cert = validate_certificate(old_certificate_name)
+
+        if not old_cert:
+            for certificate in get_all_pending_reissue():
+                request_reissue(certificate, commit)
+        else:
+            request_reissue(old_cert, commit)
+
+        status = SUCCESS_METRIC_STATUS
+        print("[+] Done!")
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.exception("Error reissuing certificate.", exc_info=True)
+        print(
+            "[!] Failed to reissue certificates. Reason: {}".format(
+                e
+            )
+        )
+
+    metrics.send('certificate_reissue_job', 'counter', 1, metric_tags={'status': status})
+
+
+@manager.option('-f', '--fqdns', dest='fqdns', help='FQDNs to query. Multiple fqdns specified via comma.')
+@manager.option('-i', '--issuer', dest='issuer', help='Issuer to query for.')
+@manager.option('-o', '--owner', dest='owner', help='Owner to query for.')
+@manager.option('-e', '--expired', dest='expired', type=bool, default=False, help='Include expired certificates.')
+def query(fqdns, issuer, owner, expired):
+    """Prints certificates that match the query params."""
+    table = []
+
+    q = database.session_query(Certificate)
+
+    sub_query = database.session_query(Authority.id) \
+        .filter(Authority.name.ilike('%{0}%'.format(issuer))) \
+        .subquery()
+
+    q = q.filter(
+        or_(
+            Certificate.issuer.ilike('%{0}%'.format(issuer)),
+            Certificate.authority_id.in_(sub_query)
+        )
+    )
+
+    q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner)))
+
+    if not expired:
+        q = q.filter(Certificate.expired == False)  # noqa
+
+    for f in fqdns.split(','):
+        q = q.filter(
+            or_(
+                Certificate.cn.ilike('%{0}%'.format(f)),
+                Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f)))
+            )
+        )
+
+    for c in q.all():
+        table.append([c.id, c.name, c.owner, c.issuer])
+
+    print(tabulate(table, headers=['Id', 'Name', 'Owner', 'Issuer'], tablefmt='csv'))
+
+
+def worker(data, commit, reason):
+    parts = [x for x in data.split(' ') if x]
+    try:
+        cert = get(int(parts[0].strip()))
+        plugin = plugins.get(cert.authority.plugin_name)
+
+        print('[+] Revoking certificate. Id: {0} Name: {1}'.format(cert.id, cert.name))
+        if commit:
+            plugin.revoke_certificate(cert, reason)
+
+        metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': SUCCESS_METRIC_STATUS})
+
+    except Exception as e:
+        sentry.captureException()
+        metrics.send('certificate_revoke', 'counter', 1, metric_tags={'status': FAILURE_METRIC_STATUS})
+        print(
+            "[!] Failed to revoke certificates. Reason: {}".format(
+                e
+            )
+        )
+
+
+@manager.command
+def clear_pending():
+    """
+    Function clears all pending certificates.
+    :return:
+    """
+    v = plugins.get('verisign-issuer')
+    v.clear_pending_certificates()
+
+
+@manager.option('-p', '--path', dest='path', help='Absolute file path to a Lemur query csv.')
+@manager.option('-r', '--reason', dest='reason', help='Reason to revoke certificate.')
+@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+def revoke(path, reason, commit):
+    """
+    Revokes given certificate.
+    """
+    if commit:
+        print("[!] Running in COMMIT mode.")
+
+    print("[+] Starting certificate revocation.")
+
+    with open(path, 'r') as f:
+        args = [[x, commit, reason] for x in f.readlines()[2:]]
+
+    with multiprocessing.Pool(processes=3) as pool:
+        pool.starmap(worker, args)
+
+
+@manager.command
+def check_revoked():
+    """
+    Function attempts to update Lemur's internal cache with revoked
+    certificates. This is called periodically by Lemur. It checks both
+    CRLs and OCSP to see if a certificate is revoked. If Lemur is unable
+    encounters an issue with verification it marks the certificate status
+    as `unknown`.
+    """
+    for cert in get_all_certs():
+        try:
+            if cert.chain:
+                status = verify_string(cert.body, cert.chain)
+            else:
+                status = verify_string(cert.body, "")
+
+            cert.status = 'valid' if status else 'revoked'
+
+        except Exception as e:
+            sentry.captureException()
+            current_app.logger.exception(e)
+            cert.status = 'unknown'
+
+        database.update(cert)

lemur/certificates/exceptions.py

@@ -1,88 +0,0 @@
-"""
-.. module: lemur.certificates.exceptions
-    :synopsis: Defines all monterey specific exceptions
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-"""
-from flask import current_app
-from lemur.exceptions import LemurException
-
-
-class UnknownAuthority(LemurException):
-    def __init__(self, authority):
-        self.code = 404
-        self.authority = authority
-        self.data = {"message": "The authority specified '{}' is not a valid authority".format(self.authority)}
-
-        current_app.logger.warning(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class InsufficientDomains(LemurException):
-    def __init__(self):
-        self.code = 400
-        self.data = {"message": "Need at least one domain specified in order create a certificate"}
-
-        current_app.logger.warning(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class InvalidCertificate(LemurException):
-    def __init__(self):
-        self.code = 400
-        self.data = {"message": "Need at least one domain specified in order create a certificate"}
-
-        current_app.logger.warning(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class UnableToCreateCSR(LemurException):
-    def __init__(self):
-        self.code = 500
-        self.data = {"message": "Unable to generate CSR"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class UnableToCreatePrivateKey(LemurException):
-    def __init__(self):
-        self.code = 500
-        self.data = {"message": "Unable to generate Private Key"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class MissingFiles(LemurException):
-    def __init__(self, path):
-        self.code = 500
-        self.path = path
-        self.data = {"path": self.path, "message": "Expecting missing files"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class NoPersistanceFound(LemurException):
-    def __init__(self):
-        self.code = 500
-        self.data = {"code": 500, "message": "No peristence method found, Lemur cannot persist sensitive information"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])

lemur/certificates/hooks.py

@@ -0,0 +1,38 @@
+"""
+Debugging hooks for dumping imported or generated CSR and certificate details to stdout via OpenSSL.
+
+.. module: lemur.certificates.hooks
+    :platform: Unix
+    :copyright: (c) 2018 by Marti Raudsepp, see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Marti Raudsepp <marti@juffo.org>
+"""
+import subprocess
+
+from flask import current_app
+
+from lemur.certificates.service import csr_created, csr_imported, certificate_issued, certificate_imported
+
+
+def csr_dump_handler(sender, csr, **kwargs):
+    try:
+        subprocess.run(['openssl', 'req', '-text', '-noout', '-reqopt', 'no_sigdump,no_pubkey'],
+                       input=csr.encode('utf8'))
+    except Exception as err:
+        current_app.logger.warning("Error inspecting CSR: %s", err)
+
+
+def cert_dump_handler(sender, certificate, **kwargs):
+    try:
+        subprocess.run(['openssl', 'x509', '-text', '-noout', '-certopt', 'no_sigdump,no_pubkey'],
+                       input=certificate.body.encode('utf8'))
+    except Exception as err:
+        current_app.logger.warning("Error inspecting certificate: %s", err)
+
+
+def activate_debug_dump():
+    csr_created.connect(csr_dump_handler)
+    csr_imported.connect(csr_dump_handler)
+    certificate_issued.connect(cert_dump_handler)
+    certificate_imported.connect(cert_dump_handler)

lemur/certificates/models.py

@@ -1,301 +1,364 @@
 """
 .. module: lemur.certificates.models
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import datetime
+import arrow
+from datetime import timedelta
+
 from flask import current_app
 
 from cryptography import x509
-from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+from idna.core import InvalidCodepoint
+
 from sqlalchemy.orm import relationship
-from sqlalchemy import event, Integer, ForeignKey, String, DateTime, PassiveDefault, func, Column, Text, Boolean
+from sqlalchemy.sql.expression import case, extract
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
+
+from sqlalchemy_utils.types.arrow import ArrowType
+
+import lemur.common.utils
+
+from lemur.database import db
+from lemur.extensions import sentry
 
 from lemur.utils import Vault
-from lemur.database import db
+from lemur.common import defaults
+
 from lemur.plugins.base import plugins
 
-from lemur.domains.models import Domain
+from lemur.extensions import metrics
-
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
-from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
 
 from lemur.models import certificate_associations, certificate_source_associations, \
     certificate_destination_associations, certificate_notification_associations, \
-    certificate_replacement_associations
+    certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations
+
+from lemur.domains.models import Domain
+from lemur.policies.models import RotationPolicy
 
 
-def create_name(issuer, not_before, not_after, subject, san):
+def get_sequence(name):
-    """
+    if '-' not in name:
-    Create a name for our certificate. A naming standard
+        return name, None
-    is based on a series of templates. The name includes
-    useful information such as Common Name, Validation dates,
-    and Issuer.
 
-    :param san:
+    parts = name.split('-')
-    :param subject:
-    :param not_after:
-    :param issuer:
-    :param not_before:
-    :rtype : str
-    :return:
-    """
-    if san:
-        t = SAN_NAMING_TEMPLATE
-    else:
-        t = DEFAULT_NAMING_TEMPLATE
 
-    temp = t.format(
+    # see if we have an int at the end of our name
-        subject=subject,
-        issuer=issuer,
-        not_before=not_before.strftime('%Y%m%d'),
-        not_after=not_after.strftime('%Y%m%d')
-    )
-
-    # NOTE we may want to give more control over naming
-    # aws doesn't allow special chars except '-'
-    disallowed_chars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
-    disallowed_chars = disallowed_chars.replace("-", "")
-    disallowed_chars = disallowed_chars.replace(".", "")
-    temp = temp.replace('*', "WILDCARD")
-
-    for c in disallowed_chars:
-        temp = temp.replace(c, "")
-
-    # white space is silly too
-    return temp.replace(" ", "-")
-
-
-def get_signing_algorithm(cert):
-    return cert.signature_hash_algorithm.name
-
-
-def get_cn(cert):
-    """
-    Attempts to get a sane common name from a given certificate.
-
-    :param cert:
-    :return: Common name or None
-    """
-    return cert.subject.get_attributes_for_oid(
-        x509.OID_COMMON_NAME
-    )[0].value.strip()
-
-
-def get_domains(cert):
-    """
-    Attempts to get an domains listed in a certificate.
-    If 'subjectAltName' extension is not available we simply
-    return the common name.
-
-    :param cert:
-    :return: List of domains
-    """
-    domains = []
     try:
-        ext = cert.extensions.get_extension_for_oid(x509.OID_SUBJECT_ALTERNATIVE_NAME)
+        seq = int(parts[-1])
-        entries = ext.value.get_values_for_type(x509.DNSName)
+    except ValueError:
-        for entry in entries:
+        return name, None
-            domains.append(entry)
-    except Exception as e:
-        current_app.logger.warning("Failed to get SubjectAltName: {0}".format(e))
 
-    return domains
+    # we might have a date at the end of our name
+    if len(parts[-1]) == 8:
+        return name, None
+
+    root = '-'.join(parts[:-1])
+    return root, seq
 
 
-def get_serial(cert):
+def get_or_increase_name(name, serial):
-    """
+    certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).all()
-    Fetch the serial number from the certificate.
 
-    :param cert:
+    if not certificates:
-    :return: serial number
+        return name
-    """
-    return cert.serial
 
+    serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper())
+    certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(serial_name))).all()
 
-def is_san(cert):
+    if not certificates:
-    """
+        return serial_name
-    Determines if a given certificate is a SAN certificate.
-    SAN certificates are simply certificates that cover multiple domains.
 
-    :param cert:
+    ends = [0]
-    :return: Bool
+    root, end = get_sequence(serial_name)
-    """
+    for cert in certificates:
-    if len(get_domains(cert)) > 1:
+        root, end = get_sequence(cert.name)
-        return True
+        if end:
+            ends.append(end)
 
-
+    return '{0}-{1}'.format(root, max(ends) + 1)
-def is_wildcard(cert):
-    """
-    Determines if certificate is a wildcard certificate.
-
-    :param cert:
-    :return: Bool
-    """
-    domains = get_domains(cert)
-    if len(domains) == 1 and domains[0][0:1] == "*":
-        return True
-
-    if cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[0].value[0:1] == "*":
-        return True
-
-
-def get_bitstrength(cert):
-    """
-    Calculates a certificates public key bit length.
-
-    :param cert:
-    :return: Integer
-    """
-    return cert.public_key().key_size
-
-
-def get_issuer(cert):
-    """
-    Gets a sane issuer from a given certificate.
-
-    :param cert:
-    :return: Issuer
-    """
-    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
-    try:
-        issuer = str(cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[0].value)
-        for c in delchars:
-            issuer = issuer.replace(c, "")
-        return issuer
-    except Exception as e:
-        current_app.logger.error("Unable to get issuer! {0}".format(e))
-
-
-def get_not_before(cert):
-    """
-    Gets the naive datetime of the certificates 'not_before' field.
-    This field denotes the first date in time which the given certificate
-    is valid.
-
-    :param cert:
-    :return: Datetime
-    """
-    return cert.not_valid_before
-
-
-def get_not_after(cert):
-    """
-    Gets the naive datetime of the certificates 'not_after' field.
-    This field denotes the last date in time which the given certificate
-    is valid.
-
-    :param cert:
-    :return: Datetime
-    """
-    return cert.not_valid_after
-
-
-def get_name_from_arn(arn):
-    """
-    Extract the certificate name from an arn.
-
-    :param arn: IAM SSL arn
-    :return: name of the certificate as uploaded to AWS
-    """
-    return arn.split("/", 1)[1]
-
-
-def get_account_number(arn):
-    """
-    Extract the account number from an arn.
-
-    :param arn: IAM SSL arn
-    :return: account number associated with ARN
-    """
-    return arn.split(":")[4]
 
 
 class Certificate(db.Model):
     __tablename__ = 'certificates'
     id = Column(Integer, primary_key=True)
-    owner = Column(String(128))
+    external_id = Column(String(128))
-    body = Column(Text())
+    owner = Column(String(128), nullable=False)
-    private_key = Column(Vault)
+    name = Column(String(256), unique=True)
-    status = Column(String(128))
+    description = Column(String(1024))
-    deleted = Column(Boolean, index=True)
+    notify = Column(Boolean, default=True)
-    name = Column(String(128))
+
+    body = Column(Text(), nullable=False)
     chain = Column(Text())
-    bits = Column(Integer())
+    private_key = Column(Vault)
+
     issuer = Column(String(128))
     serial = Column(String(128))
     cn = Column(String(128))
-    description = Column(String(1024))
+    deleted = Column(Boolean, index=True)
-    active = Column(Boolean, default=True)
+    dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='cascade'), nullable=True)
-    san = Column(String(1024))
+
-    not_before = Column(DateTime)
+    not_before = Column(ArrowType)
-    not_after = Column(DateTime)
+    not_after = Column(ArrowType)
-    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
+    date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
+
     signing_algorithm = Column(String(128))
+    status = Column(String(128))
+    bits = Column(Integer())
+    san = Column(String(1024))  # TODO this should be migrated to boolean
+
+    rotation = Column(Boolean, default=False)
     user_id = Column(Integer, ForeignKey('users.id'))
-    authority_id = Column(Integer, ForeignKey('authorities.id'))
+    authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
-    notifications = relationship("Notification", secondary=certificate_notification_associations, backref='certificate')
+    root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
-    destinations = relationship("Destination", secondary=certificate_destination_associations, backref='certificate')
+    rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
-    replaces = relationship("Certificate",
+
+    notifications = relationship('Notification', secondary=certificate_notification_associations, backref='certificate')
+    destinations = relationship('Destination', secondary=certificate_destination_associations, backref='certificate')
+    sources = relationship('Source', secondary=certificate_source_associations, backref='certificate')
+    domains = relationship('Domain', secondary=certificate_associations, backref='certificate')
+    roles = relationship('Role', secondary=roles_certificates, backref='certificate')
+    replaces = relationship('Certificate',
                             secondary=certificate_replacement_associations,
                             primaryjoin=id == certificate_replacement_associations.c.certificate_id,  # noqa
                             secondaryjoin=id == certificate_replacement_associations.c.replaced_certificate_id,  # noqa
                             backref='replaced')
-    sources = relationship("Source", secondary=certificate_source_associations, backref='certificate')
-    domains = relationship("Domain", secondary=certificate_associations, backref="certificate")
 
-    def __init__(self, body, private_key=None, chain=None):
+    replaced_by_pending = relationship('PendingCertificate',
-        self.body = body
+                                       secondary=pending_cert_replacement_associations,
-        # We encrypt the private_key on creation
+                                       backref='pending_replace',
-        self.private_key = private_key
+                                       viewonly=True)
-        self.chain = chain
-        cert = x509.load_pem_x509_certificate(str(self.body), default_backend())
-        self.signing_algorithm = get_signing_algorithm(cert)
-        self.bits = get_bitstrength(cert)
-        self.issuer = get_issuer(cert)
-        self.serial = get_serial(cert)
-        self.cn = get_cn(cert)
-        self.san = is_san(cert)
-        self.not_before = get_not_before(cert)
-        self.not_after = get_not_after(cert)
-        self.name = create_name(self.issuer, self.not_before, self.not_after, self.cn, self.san)
 
-        for domain in get_domains(cert):
+    logs = relationship('Log', backref='certificate')
+    endpoints = relationship('Endpoint', backref='certificate')
+    rotation_policy = relationship("RotationPolicy")
+
+    sensitive_fields = ('private_key',)
+
+    def __init__(self, **kwargs):
+        cert = lemur.common.utils.parse_certificate(kwargs['body'])
+
+        self.issuer = defaults.issuer(cert)
+        self.cn = defaults.common_name(cert)
+        self.san = defaults.san(cert)
+        self.not_before = defaults.not_before(cert)
+        self.not_after = defaults.not_after(cert)
+        self.serial = defaults.serial(cert)
+
+        # when destinations are appended they require a valid name.
+        if kwargs.get('name'):
+            self.name = get_or_increase_name(defaults.text_to_slug(kwargs['name']), self.serial)
+        else:
+            self.name = get_or_increase_name(
+                defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san), self.serial)
+
+        self.owner = kwargs['owner']
+        self.body = kwargs['body'].strip()
+
+        if kwargs.get('private_key'):
+            self.private_key = kwargs['private_key'].strip()
+
+        if kwargs.get('chain'):
+            self.chain = kwargs['chain'].strip()
+
+        self.notify = kwargs.get('notify', True)
+        self.destinations = kwargs.get('destinations', [])
+        self.notifications = kwargs.get('notifications', [])
+        self.description = kwargs.get('description')
+        self.roles = list(set(kwargs.get('roles', [])))
+        self.replaces = kwargs.get('replaces', [])
+        self.rotation = kwargs.get('rotation')
+        self.rotation_policy = kwargs.get('rotation_policy')
+        self.signing_algorithm = defaults.signing_algorithm(cert)
+        self.bits = defaults.bitstrength(cert)
+        self.external_id = kwargs.get('external_id')
+        self.authority_id = kwargs.get('authority_id')
+        self.dns_provider_id = kwargs.get('dns_provider_id')
+
+        for domain in defaults.domains(cert):
             self.domains.append(Domain(name=domain))
 
     @property
-    def is_expired(self):
+    def active(self):
-        if self.not_after < datetime.datetime.now():
+        return self.notify
-            return True
 
     @property
-    def is_unused(self):
+    def organization(self):
-        if self.elb_listeners.count() == 0:
+        cert = lemur.common.utils.parse_certificate(self.body)
-            return True
+        return defaults.organization(cert)
 
     @property
-    def is_revoked(self):
+    def organizational_unit(self):
-        # we might not yet know the condition of the cert
+        cert = lemur.common.utils.parse_certificate(self.body)
-        if self.status:
+        return defaults.organizational_unit(cert)
-            if 'revoked' in self.status:
-                return True
 
-    def get_arn(self, account_number):
+    @property
+    def country(self):
+        cert = lemur.common.utils.parse_certificate(self.body)
+        return defaults.country(cert)
+
+    @property
+    def state(self):
+        cert = lemur.common.utils.parse_certificate(self.body)
+        return defaults.state(cert)
+
+    @property
+    def location(self):
+        cert = lemur.common.utils.parse_certificate(self.body)
+        return defaults.location(cert)
+
+    @property
+    def key_type(self):
+        cert = lemur.common.utils.parse_certificate(self.body)
+        if isinstance(cert.public_key(), rsa.RSAPublicKey):
+            return 'RSA{key_size}'.format(key_size=cert.public_key().key_size)
+
+    @property
+    def validity_remaining(self):
+        return abs(self.not_after - arrow.utcnow())
+
+    @property
+    def validity_range(self):
+        return self.not_after - self.not_before
+
+    @property
+    def subject(self):
+        cert = lemur.common.utils.parse_certificate(self.body)
+        return cert.subject
+
+    @property
+    def public_key(self):
+        cert = lemur.common.utils.parse_certificate(self.body)
+        return cert.public_key()
+
+    @hybrid_property
+    def expired(self):
+        if self.not_after <= arrow.utcnow():
+            return True
+
+    @expired.expression
+    def expired(cls):
+        return case(
+            [
+                (cls.not_after <= arrow.utcnow(), True)
+            ],
+            else_=False
+        )
+
+    @hybrid_property
+    def revoked(self):
+        if 'revoked' == self.status:
+            return True
+
+    @revoked.expression
+    def revoked(cls):
+        return case(
+            [
+                (cls.status == 'revoked', True)
+            ],
+            else_=False
+        )
+
+    @hybrid_property
+    def in_rotation_window(self):
         """
-        Generate a valid AWS IAM arn
+        Determines if a certificate is available for rotation based
-
+        on the rotation policy associated.
-        :rtype : str
-        :param account_number:
         :return:
         """
-        return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)
+        now = arrow.utcnow()
+        end = now + timedelta(days=self.rotation_policy.days)
+
+        if self.not_after <= end:
+            return True
+
+    @in_rotation_window.expression
+    def in_rotation_window(cls):
+        """
+        Determines if a certificate is available for rotation based
+        on the rotation policy associated.
+        :return:
+        """
+        return case(
+            [
+                (extract('day', cls.not_after - func.now()) <= RotationPolicy.days, True)
+            ],
+            else_=False
+        )
+
+    @property
+    def extensions(self):
+        # setup default values
+        return_extensions = {
+            'sub_alt_names': {'names': []}
+        }
+
+        try:
+            cert = lemur.common.utils.parse_certificate(self.body)
+            for extension in cert.extensions:
+                value = extension.value
+                if isinstance(value, x509.BasicConstraints):
+                    return_extensions['basic_constraints'] = value
+
+                elif isinstance(value, x509.SubjectAlternativeName):
+                    return_extensions['sub_alt_names']['names'] = value
+
+                elif isinstance(value, x509.ExtendedKeyUsage):
+                    return_extensions['extended_key_usage'] = value
+
+                elif isinstance(value, x509.KeyUsage):
+                    return_extensions['key_usage'] = value
+
+                elif isinstance(value, x509.SubjectKeyIdentifier):
+                    return_extensions['subject_key_identifier'] = {'include_ski': True}
+
+                elif isinstance(value, x509.AuthorityInformationAccess):
+                    return_extensions['certificate_info_access'] = {'include_aia': True}
+
+                elif isinstance(value, x509.AuthorityKeyIdentifier):
+                    aki = {
+                        'use_key_identifier': False,
+                        'use_authority_cert': False
+                    }
+
+                    if value.key_identifier:
+                        aki['use_key_identifier'] = True
+
+                    if value.authority_cert_issuer:
+                        aki['use_authority_cert'] = True
+
+                    return_extensions['authority_key_identifier'] = aki
+
+                elif isinstance(value, x509.CRLDistributionPoints):
+                    return_extensions['crl_distribution_points'] = {'include_crl_dp': value}
+
+                # TODO: Not supporting custom OIDs yet. https://github.com/Netflix/lemur/issues/665
+                else:
+                    current_app.logger.warning('Custom OIDs not yet supported for clone operation.')
+        except InvalidCodepoint as e:
+            sentry.captureException()
+            current_app.logger.warning('Unable to parse extensions due to underscore in dns name')
+        except ValueError as e:
+            sentry.captureException()
+            current_app.logger.warning('Unable to parse')
+            current_app.logger.exception(e)
+
+        return return_extensions
+
+    def __repr__(self):
+        return "Certificate(name={name})".format(name=self.name)
 
 
 @event.listens_for(Certificate.destinations, 'append')
 def update_destinations(target, value, initiator):
     """
-    Attempt to upload the new certificate to the new destination
+    Attempt to upload certificate to the new destination
 
     :param target:
     :param value:
@@ -303,32 +366,26 @@ def update_destinations(target, value, initiator):
     :return:
     """
     destination_plugin = plugins.get(value.plugin_name)
-    destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
+    status = FAILURE_METRIC_STATUS
+    try:
+        if target.private_key:
+            destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
+            status = SUCCESS_METRIC_STATUS
+    except Exception as e:
+        sentry.captureException()
+
+    metrics.send('destination_upload', 'counter', 1,
+                 metric_tags={'status': status, 'certificate': target.name, 'destination': value.label})
 
 
 @event.listens_for(Certificate.replaces, 'append')
 def update_replacement(target, value, initiator):
     """
-    When a certificate is marked as 'replaced' it is then marked as in-active
+    When a certificate is marked as 'replaced' we should not notify.
 
     :param target:
     :param value:
     :param initiator:
     :return:
     """
-    value.active = False
+    value.notify = False
-
-
-@event.listens_for(Certificate, 'before_update')
-def protect_active(mapper, connection, target):
-    """
-    When a certificate has a replacement do not allow it to be marked as 'active'
-
-    :param connection:
-    :param mapper:
-    :param target:
-    :return:
-    """
-    if target.active:
-        if target.replaced:
-            raise Exception("Cannot mark certificate as active, certificate has been marked as replaced.")

lemur/certificates/schemas.py

@@ -0,0 +1,275 @@
+"""
+.. module: lemur.certificates.schemas
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import current_app
+from marshmallow import fields, validate, validates_schema, post_load, pre_load
+from marshmallow.exceptions import ValidationError
+
+from lemur.authorities.schemas import AuthorityNestedOutputSchema
+from lemur.common import validators, missing
+from lemur.common.fields import ArrowDateTime, Hex
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.constants import CERTIFICATE_KEY_TYPES
+from lemur.destinations.schemas import DestinationNestedOutputSchema
+from lemur.domains.schemas import DomainNestedOutputSchema
+from lemur.notifications import service as notification_service
+from lemur.notifications.schemas import NotificationNestedOutputSchema
+from lemur.policies.schemas import RotationPolicyNestedOutputSchema
+from lemur.roles.schemas import RoleNestedOutputSchema
+from lemur.schemas import (
+    AssociatedAuthoritySchema,
+    AssociatedDestinationSchema,
+    AssociatedCertificateSchema,
+    AssociatedNotificationSchema,
+    AssociatedDnsProviderSchema,
+    PluginInputSchema,
+    ExtensionSchema,
+    AssociatedRoleSchema,
+    EndpointNestedOutputSchema,
+    AssociatedRotationPolicySchema,
+)
+from lemur.users.schemas import UserNestedOutputSchema
+
+
+class CertificateSchema(LemurInputSchema):
+    owner = fields.Email(required=True)
+    description = fields.String(missing='', allow_none=True)
+
+
+class CertificateCreationSchema(CertificateSchema):
+    @post_load
+    def default_notification(self, data):
+        if not data['notifications']:
+            data['notifications'] += notification_service.create_default_expiration_notifications(
+                "DEFAULT_{0}".format(data['owner'].split('@')[0].upper()),
+                [data['owner']],
+            )
+            data['notifications'] += notification_service.create_default_expiration_notifications(
+                'DEFAULT_SECURITY',
+                current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
+            )
+        return data
+
+
+class CertificateInputSchema(CertificateCreationSchema):
+    name = fields.String()
+    common_name = fields.String(required=True, validate=validators.common_name)
+    authority = fields.Nested(AssociatedAuthoritySchema, required=True)
+
+    validity_start = ArrowDateTime()
+    validity_end = ArrowDateTime()
+    validity_years = fields.Integer()
+
+    destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
+    notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
+    replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
+    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
+    roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
+    dns_provider = fields.Nested(AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False)
+
+    csr = fields.String(validate=validators.csr)
+
+    key_type = fields.String(
+        validate=validate.OneOf(CERTIFICATE_KEY_TYPES),
+        missing='RSA2048')
+
+    notify = fields.Boolean(default=True)
+    rotation = fields.Boolean()
+    rotation_policy = fields.Nested(AssociatedRotationPolicySchema, missing={'name': 'default'}, default={'name': 'default'})
+
+    # certificate body fields
+    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
+    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
+    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
+    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
+    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+
+    extensions = fields.Nested(ExtensionSchema)
+
+    @validates_schema
+    def validate_authority(self, data):
+        if not data['authority'].active:
+            raise ValidationError("The authority is inactive.", ['authority'])
+
+    @validates_schema
+    def validate_dates(self, data):
+        validators.dates(data)
+
+    @pre_load
+    def load_data(self, data):
+        if data.get('replacements'):
+            data['replaces'] = data['replacements']  # TODO remove when field is deprecated
+        return missing.convert_validity_years(data)
+
+
+class CertificateEditInputSchema(CertificateSchema):
+    owner = fields.String()
+
+    notify = fields.Boolean()
+    rotation = fields.Boolean()
+
+    destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
+    notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
+    replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
+    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
+    roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
+
+    @pre_load
+    def load_data(self, data):
+        if data.get('replacements'):
+            data['replaces'] = data['replacements']  # TODO remove when field is deprecated
+        return data
+
+    @post_load
+    def enforce_notifications(self, data):
+        """
+        Ensures that when an owner changes, default notifications are added for the new owner.
+        Old owner notifications are retained unless explicitly removed.
+        :param data:
+        :return:
+        """
+        if data['owner']:
+            notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
+            data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']])
+        return data
+
+
+class CertificateNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    name = fields.String()
+    owner = fields.Email()
+    creator = fields.Nested(UserNestedOutputSchema)
+    description = fields.String()
+
+    status = fields.String()
+
+    bits = fields.Integer()
+    body = fields.String()
+    chain = fields.String()
+    active = fields.Boolean()
+
+    rotation = fields.Boolean()
+    notify = fields.Boolean()
+    rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
+
+    # Note aliasing is the first step in deprecating these fields.
+    cn = fields.String()  # deprecated
+    common_name = fields.String(attribute='cn')
+
+    not_after = fields.DateTime()  # deprecated
+    validity_end = ArrowDateTime(attribute='not_after')
+
+    not_before = fields.DateTime()  # deprecated
+    validity_start = ArrowDateTime(attribute='not_before')
+
+    issuer = fields.Nested(AuthorityNestedOutputSchema)
+
+
+class CertificateCloneSchema(LemurOutputSchema):
+    __envelope__ = False
+    description = fields.String()
+    common_name = fields.String()
+
+
+class CertificateOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    external_id = fields.String()
+    bits = fields.Integer()
+    body = fields.String()
+    chain = fields.String()
+    deleted = fields.Boolean(default=False)
+    description = fields.String()
+    issuer = fields.String()
+    name = fields.String()
+    dns_provider_id = fields.Integer(required=False, allow_none=True)
+
+    rotation = fields.Boolean()
+
+    # Note aliasing is the first step in deprecating these fields.
+    notify = fields.Boolean()
+    active = fields.Boolean(attribute='notify')
+
+    cn = fields.String()
+    common_name = fields.String(attribute='cn')
+
+    not_after = fields.DateTime()
+    validity_end = ArrowDateTime(attribute='not_after')
+
+    not_before = fields.DateTime()
+    validity_start = ArrowDateTime(attribute='not_before')
+
+    owner = fields.Email()
+    san = fields.Boolean()
+    serial = fields.String()
+    serial_hex = Hex(attribute='serial')
+    signing_algorithm = fields.String()
+
+    status = fields.String()
+    user = fields.Nested(UserNestedOutputSchema)
+
+    extensions = fields.Nested(ExtensionSchema)
+
+    # associated objects
+    domains = fields.Nested(DomainNestedOutputSchema, many=True)
+    destinations = fields.Nested(DestinationNestedOutputSchema, many=True)
+    notifications = fields.Nested(NotificationNestedOutputSchema, many=True)
+    replaces = fields.Nested(CertificateNestedOutputSchema, many=True)
+    authority = fields.Nested(AuthorityNestedOutputSchema)
+    roles = fields.Nested(RoleNestedOutputSchema, many=True)
+    endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
+    replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
+    rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
+
+
+class CertificateUploadInputSchema(CertificateCreationSchema):
+    name = fields.String()
+    notify = fields.Boolean(missing=True)
+
+    private_key = fields.String(validate=validators.private_key)
+    body = fields.String(required=True, validate=validators.public_certificate)
+    chain = fields.String(validate=validators.public_certificate, missing=None, allow_none=True)  # TODO this could be multiple certificates
+
+    destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
+    notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
+    replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
+    roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
+
+    @validates_schema
+    def keys(self, data):
+        if data.get('destinations'):
+            if not data.get('private_key'):
+                raise ValidationError('Destinations require private key.')
+
+
+class CertificateExportInputSchema(LemurInputSchema):
+    plugin = fields.Nested(PluginInputSchema)
+
+
+class CertificateNotificationOutputSchema(LemurOutputSchema):
+    description = fields.String()
+    issuer = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    user = fields.Nested(UserNestedOutputSchema)
+    validity_end = ArrowDateTime(attribute='not_after')
+    replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
+    endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
+
+
+class CertificateRevokeSchema(LemurInputSchema):
+    comments = fields.String()
+
+
+certificate_input_schema = CertificateInputSchema()
+certificate_output_schema = CertificateOutputSchema()
+certificates_output_schema = CertificateOutputSchema(many=True)
+certificate_upload_input_schema = CertificateUploadInputSchema()
+certificate_export_input_schema = CertificateExportInputSchema()
+certificate_edit_input_schema = CertificateEditInputSchema()
+certificate_notification_output_schema = CertificateNotificationOutputSchema()
+certificate_revoke_schema = CertificateRevokeSchema()

lemur/certificates/service.py

@@ -1,35 +1,46 @@
 """
-.. module: service
+.. module: lemur.certificate.service
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
 
-from sqlalchemy import func, or_
+from flask import current_app
-from flask import g, current_app
+from sqlalchemy import func, or_, not_, cast, Integer
-
-from lemur import database
-from lemur.plugins.base import plugins
-from lemur.certificates.models import Certificate
-
-from lemur.destinations.models import Destination
-from lemur.notifications.models import Notification
-from lemur.authorities.models import Authority
-from lemur.domains.models import Domain
-
-from lemur.roles.models import Role
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
-from cryptography.hazmat.primitives.asymmetric import rsa
+
+from lemur import database
+from lemur.extensions import metrics, signals
+from lemur.plugins.base import plugins
+from lemur.common.utils import generate_private_key, truthiness
+
+from lemur.roles.models import Role
+from lemur.domains.models import Domain
+from lemur.authorities.models import Authority
+from lemur.destinations.models import Destination
+from lemur.certificates.models import Certificate
+from lemur.notifications.models import Notification
+from lemur.pending_certificates.models import PendingCertificate
+
+from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
+
+from lemur.roles import service as role_service
+
+
+csr_created = signals.signal('csr_created', "CSR generated")
+csr_imported = signals.signal('csr_imported', "CSR imported from external source")
+certificate_issued = signals.signal('certificate_issued', "Authority issued a certificate")
+certificate_imported = signals.signal('certificate_imported', "Certificate imported from external source")
 
 
 def get(cert_id):
     """
-    Retrieves certificate by it's ID.
+    Retrieves certificate by its ID.
 
     :param cert_id:
     :return:
@@ -39,7 +50,7 @@ def get(cert_id):
 
 def get_by_name(name):
     """
-    Retrieves certificate by it's Name.
+    Retrieves certificate by its Name.
 
     :param name:
     :return:
@@ -47,6 +58,18 @@ def get_by_name(name):
     return database.get(Certificate, name, field='name')
 
 
+def get_by_serial(serial):
+    """
+    Retrieves certificate by it's Serial.
+    :param serial:
+    :return:
+    """
+    if isinstance(serial, int):
+        # although serial is a number, the DB column is String(128)
+        serial = str(serial)
+    return Certificate.query.filter(Certificate.serial == serial).all()
+
+
 def delete(cert_id):
     """
     Delete's a certificate.
@@ -65,16 +88,45 @@ def get_all_certs():
     return Certificate.query.all()
 
 
-def find_duplicates(cert_body):
+def get_all_pending_cleaning(source):
+    """
+    Retrieves all certificates that are available for cleaning.
+
+    :param source:
+    :return:
+    """
+    return Certificate.query.filter(Certificate.sources.any(id=source.id))\
+                            .filter(not_(Certificate.endpoints.any())).all()
+
+
+def get_all_pending_reissue():
+    """
+    Retrieves all certificates that need to be rotated.
+
+    Must be X days from expiration, uses the certificates rotation
+    policy to determine how many days from expiration the certificate must be
+    for rotation to be pending.
+
+    :return:
+    """
+    return Certificate.query.filter(Certificate.rotation == True)\
+        .filter(not_(Certificate.replaced.any()))\
+        .filter(Certificate.in_rotation_window == True).all()  # noqa
+
+
+def find_duplicates(cert):
     """
     Finds certificates that already exist within Lemur. We do this by looking for
     certificate bodies that are the same. This is the most reliable way to determine
     if a certificate is already being tracked by Lemur.
 
-    :param cert_body:
+    :param cert:
     :return:
     """
-    return Certificate.query.filter_by(body=cert_body).all()
+    if cert['chain']:
+        return Certificate.query.filter_by(body=cert['body'].strip(), chain=cert['chain'].strip()).all()
+    else:
+        return Certificate.query.filter_by(body=cert['body'].strip(), chain=None).all()
 
 
 def export(cert, export_plugin):
@@ -87,75 +139,62 @@ def export(cert, export_plugin):
     :return:
     """
     plugin = plugins.get(export_plugin['slug'])
-
     return plugin.export(cert.body, cert.chain, cert.private_key, export_plugin['pluginOptions'])
 
 
-def update(cert_id, owner, description, active, destinations, notifications, replaces):
+def update(cert_id, **kwargs):
     """
     Updates a certificate
     :param cert_id:
-    :param owner:
-    :param description:
-    :param active:
-    :param destinations:
-    :param notifications:
-    :param replaces:
     :return:
     """
-    from lemur.notifications import service as notification_service
     cert = get(cert_id)
-    cert.active = active
-    cert.description = description
 
-    # we might have to create new notifications if the owner changes
+    for key, value in kwargs.items():
-    new_notifications = []
+        setattr(cert, key, value)
-    # get existing names to remove
-    notification_name = "DEFAULT_{0}".format(cert.owner.split('@')[0].upper())
-    for n in notifications:
-        if notification_name not in n.label:
-            new_notifications.append(n)
-
-    notification_name = "DEFAULT_{0}".format(owner.split('@')[0].upper())
-    new_notifications += notification_service.create_default_expiration_notifications(notification_name, owner)
-
-    cert.notifications = new_notifications
-
-    database.update_list(cert, 'destinations', Destination, destinations)
-    database.update_list(cert, 'replaces', Certificate, replaces)
-
-    cert.owner = owner
 
     return database.update(cert)
 
 
-def mint(issuer_options):
+def create_certificate_roles(**kwargs):
+    # create an role for the owner and assign it
+    owner_role = role_service.get_by_name(kwargs['owner'])
+
+    if not owner_role:
+        owner_role = role_service.create(
+            kwargs['owner'],
+            description="Auto generated role based on owner: {0}".format(kwargs['owner'])
+        )
+
+    # ensure that the authority's owner is also associated with the certificate
+    if kwargs.get('authority'):
+        authority_owner_role = role_service.get_by_name(kwargs['authority'].owner)
+        return [owner_role, authority_owner_role]
+
+    return [owner_role]
+
+
+def mint(**kwargs):
     """
     Minting is slightly different for each authority.
     Support for multiple authorities is handled by individual plugins.
 
-    :param issuer_options:
     """
-    authority = issuer_options['authority']
+    authority = kwargs['authority']
 
     issuer = plugins.get(authority.plugin_name)
 
     # allow the CSR to be specified by the user
-    if not issuer_options.get('csr'):
+    if not kwargs.get('csr'):
-        csr, private_key = create_csr(issuer_options)
+        csr, private_key = create_csr(**kwargs)
+        csr_created.send(authority=authority, csr=csr)
     else:
-        csr = issuer_options.get('csr')
+        csr = str(kwargs.get('csr'))
         private_key = None
+        csr_imported.send(authority=authority, csr=csr)
 
-    issuer_options['creator'] = g.user.email
+    cert_body, cert_chain, external_id = issuer.create_certificate(csr, kwargs)
-    cert_body, cert_chain = issuer.create_certificate(csr, issuer_options)
+    return cert_body, private_key, cert_chain, external_id, csr
-
-    cert = Certificate(cert_body, private_key, cert_chain)
-
-    cert.user = g.user
-    cert.authority = authority
-    database.update(cert)
-    return cert, private_key, cert_chain,
 
 
 def import_certificate(**kwargs):
@@ -171,68 +210,36 @@ def import_certificate(**kwargs):
 
     :param kwargs:
     """
-    from lemur.users import service as user_service
+    if not kwargs.get('owner'):
-    from lemur.notifications import service as notification_service
+        kwargs['owner'] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0]
-    cert = Certificate(kwargs['public_certificate'], chain=kwargs['intermediate_certificate'])
 
-    # TODO future source plugins might have a better understanding of who the 'owner' is we should support this
+    return upload(**kwargs)
-    cert.owner = kwargs.get('owner', current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0])
-    cert.creator = kwargs.get('creator', user_service.get_by_email('lemur@nobody'))
-
-    # NOTE existing certs may not follow our naming standard we will
-    # overwrite the generated name with the actual cert name
-    if kwargs.get('name'):
-        cert.name = kwargs.get('name')
-
-    if kwargs.get('user'):
-        cert.user = kwargs.get('user')
-
-    notification_name = 'DEFAULT_SECURITY'
-    notifications = notification_service.create_default_expiration_notifications(notification_name, current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
-    database.update_list(cert, 'replaces', Certificate, kwargs['replacements'])
-    cert.notifications = notifications
-
-    cert = database.create(cert)
-    return cert
 
 
 def upload(**kwargs):
     """
     Allows for pre-made certificates to be imported into Lemur.
     """
-    from lemur.notifications import service as notification_service
+    roles = create_certificate_roles(**kwargs)
-    cert = Certificate(
-        kwargs.get('public_cert'),
-        kwargs.get('private_key'),
-        kwargs.get('intermediate_cert'),
-    )
 
-    # we override the generated name if one is provided
+    if kwargs.get('roles'):
-    if kwargs.get('name'):
+        kwargs['roles'] += roles
-        cert.name = kwargs['name']
+    else:
+        kwargs['roles'] = roles
 
-    cert.description = kwargs.get('description')
+    if kwargs.get('private_key'):
+        private_key = kwargs['private_key']
+        if not isinstance(private_key, bytes):
+            kwargs['private_key'] = private_key.encode('utf-8')
+
+    cert = Certificate(**kwargs)
 
-    cert.owner = kwargs['owner']
     cert = database.create(cert)
 
-    g.user.certificates.append(cert)
+    kwargs['creator'].certificates.append(cert)
 
-    database.update_list(cert, 'destinations', Destination, kwargs.get('destinations'))
+    cert = database.update(cert)
-    database.update_list(cert, 'notifications', Notification, kwargs.get('notifications'))
+    certificate_imported.send(certificate=cert, authority=cert.authority)
-    database.update_list(cert, 'replaces', Certificate, kwargs['replacements'])
-
-    # create default notifications for this certificate if none are provided
-    notifications = []
-    if not kwargs.get('notifications'):
-        notification_name = "DEFAULT_{0}".format(cert.owner.split('@')[0].upper())
-        notifications += notification_service.create_default_expiration_notifications(notification_name, [cert.owner])
-
-    notification_name = 'DEFAULT_SECURITY'
-    notifications += notification_service.create_default_expiration_notifications(notification_name, current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
-    cert.notifications = notifications
-
-    database.update(cert)
     return cert
 
 
@@ -240,34 +247,34 @@ def create(**kwargs):
     """
     Creates a new certificate.
     """
-    from lemur.notifications import service as notification_service
+    cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs)
-    cert, private_key, cert_chain = mint(kwargs)
+    kwargs['body'] = cert_body
+    kwargs['private_key'] = private_key
+    kwargs['chain'] = cert_chain
+    kwargs['external_id'] = external_id
+    kwargs['csr'] = csr
 
-    cert.owner = kwargs['owner']
+    roles = create_certificate_roles(**kwargs)
 
-    database.create(cert)
+    if kwargs.get('roles'):
-    cert.description = kwargs['description']
+        kwargs['roles'] += roles
-    g.user.certificates.append(cert)
+    else:
-    database.update(g.user)
+        kwargs['roles'] = roles
 
-    # do this after the certificate has already been created because if it fails to upload to the third party
+    if cert_body:
-    # we do not want to lose the certificate information.
+        cert = Certificate(**kwargs)
-    database.update_list(cert, 'destinations', Destination, kwargs.get('destinations'))
+        kwargs['creator'].certificates.append(cert)
-    database.update_list(cert, 'replaces', Certificate, kwargs['replacements'])
+    else:
-    database.update_list(cert, 'notifications', Notification, kwargs.get('notifications'))
+        cert = PendingCertificate(**kwargs)
+        kwargs['creator'].pending_certificates.append(cert)
 
-    # create default notifications for this certificate if none are provided
+    cert.authority = kwargs['authority']
-    notifications = cert.notifications
-    if not kwargs.get('notifications'):
-        notification_name = "DEFAULT_{0}".format(cert.owner.split('@')[0].upper())
-        notifications += notification_service.create_default_expiration_notifications(notification_name, [cert.owner])
 
-    notification_name = 'DEFAULT_SECURITY'
+    database.commit()
-    notifications += notification_service.create_default_expiration_notifications(notification_name,
-                                                                                  current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
-    cert.notifications = notifications
 
-    database.update(cert)
+    if isinstance(cert, Certificate):
+        certificate_issued.send(certificate=cert, authority=cert.authority)
+        metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
     return cert
 
 
@@ -291,40 +298,55 @@ def render(args):
 
     if filt:
         terms = filt.split(';')
+        term = '%{0}%'.format(terms[1])
+        # Exact matches for quotes. Only applies to name, issuer, and cn
+        if terms[1].startswith('"') and terms[1].endswith('"'):
+            term = terms[1][1:-1]
 
         if 'issuer' in terms:
             # we can't rely on issuer being correct in the cert directly so we combine queries
             sub_query = database.session_query(Authority.id)\
-                .filter(Authority.name.ilike('%{0}%'.format(terms[1])))\
+                .filter(Authority.name.ilike(term))\
                 .subquery()
 
             query = query.filter(
                 or_(
-                    Certificate.issuer.ilike('%{0}%'.format(terms[1])),
+                    Certificate.issuer.ilike(term),
                     Certificate.authority_id.in_(sub_query)
                 )
             )
-            return database.sort_and_page(query, Certificate, args)
 
         elif 'destination' in terms:
             query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
-        elif 'active' in filt:  # this is really weird but strcmp seems to not work here??
+        elif 'notify' in filt:
-            query = query.filter(Certificate.active == terms[1])
+            query = query.filter(Certificate.notify == truthiness(terms[1]))
+        elif 'active' in filt:
+            query = query.filter(Certificate.active == truthiness(terms[1]))
         elif 'cn' in terms:
             query = query.filter(
                 or_(
-                    Certificate.cn.ilike('%{0}%'.format(terms[1])),
+                    Certificate.cn.ilike(term),
-                    Certificate.domains.any(Domain.name.ilike('%{0}%'.format(terms[1])))
+                    Certificate.domains.any(Domain.name.ilike(term))
+                )
+            )
+        elif 'id' in terms:
+            query = query.filter(Certificate.id == cast(terms[1], Integer))
+        elif 'name' in terms:
+            query = query.filter(
+                or_(
+                    Certificate.name.ilike(term),
+                    Certificate.domains.any(Domain.name.ilike(term)),
+                    Certificate.cn.ilike(term),
                 )
             )
         else:
             query = database.filter(query, Certificate, terms)
 
     if show:
-        sub_query = database.session_query(Role.name).filter(Role.user_id == g.user.id).subquery()
+        sub_query = database.session_query(Role.name).filter(Role.user_id == args['user'].id).subquery()
         query = query.filter(
             or_(
-                Certificate.user_id == g.user.id,
+                Certificate.user_id == args['user'].id,
                 Certificate.owner.in_(sub_query)
             )
         )
@@ -343,106 +365,74 @@ def render(args):
     return database.sort_and_page(query, Certificate, args)
 
 
-def create_csr(csr_config):
+def create_csr(**csr_config):
     """
     Given a list of domains create the appropriate csr
     for those domains
 
     :param csr_config:
     """
-    private_key = rsa.generate_private_key(
+    private_key = generate_private_key(csr_config.get('key_type'))
-        public_exponent=65537,
-        key_size=2048,
-        backend=default_backend()
-    )
 
-    # TODO When we figure out a better way to validate these options they should be parsed as str
     builder = x509.CertificateSigningRequestBuilder()
-    builder = builder.subject_name(x509.Name([
+    name_list = [x509.NameAttribute(x509.OID_COMMON_NAME, csr_config['common_name'])]
-        x509.NameAttribute(x509.OID_COMMON_NAME, csr_config['commonName']),
+    if current_app.config.get('LEMUR_OWNER_EMAIL_IN_SUBJECT', True):
-        x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config['organization']),
+        name_list.append(x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config['owner']))
-        x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config['organizationalUnit']),
+    if 'organization' in csr_config and csr_config['organization'].strip():
-        x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config['country']),
+        name_list.append(x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config['organization']))
-        x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config['state']),
+    if 'organizational_unit' in csr_config and csr_config['organizational_unit'].strip():
-        x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config['location']),
+        name_list.append(x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config['organizational_unit']))
-    ]))
+    if 'country' in csr_config and csr_config['country'].strip():
+        name_list.append(x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config['country']))
+    if 'state' in csr_config and csr_config['state'].strip():
+        name_list.append(x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config['state']))
+    if 'location' in csr_config and csr_config['location'].strip():
+        name_list.append(x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config['location']))
+    builder = builder.subject_name(x509.Name(name_list))
 
-    builder = builder.add_extension(
+    extensions = csr_config.get('extensions', {})
-        x509.BasicConstraints(ca=False, path_length=None), critical=True,
+    critical_extensions = ['basic_constraints', 'sub_alt_names', 'key_usage']
-    )
+    noncritical_extensions = ['extended_key_usage']
+    for k, v in extensions.items():
+        if v:
+            if k in critical_extensions:
+                current_app.logger.debug('Adding Critical Extension: {0} {1}'.format(k, v))
+                if k == 'sub_alt_names':
+                    if v['names']:
+                        builder = builder.add_extension(v['names'], critical=True)
+                else:
+                    builder = builder.add_extension(v, critical=True)
 
-    if csr_config.get('extensions'):
+            if k in noncritical_extensions:
-        for k, v in csr_config.get('extensions', {}).items():
+                current_app.logger.debug('Adding Extension: {0} {1}'.format(k, v))
-            if k == 'subAltNames':
+                builder = builder.add_extension(v, critical=False)
-                # map types to their x509 objects
-                general_names = []
-                for name in v['names']:
-                    if name['nameType'] == 'DNSName':
-                        general_names.append(x509.DNSName(name['value']))
 
-                builder = builder.add_extension(
+    ski = extensions.get('subject_key_identifier', {})
-                    x509.SubjectAlternativeName(general_names), critical=True
+    if ski.get('include_ski', False):
-                )
+        builder = builder.add_extension(
-
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
-    # TODO support more CSR options, none of the authority plugins currently support these options
+            critical=False
-    #    builder.add_extension(
+        )
-    #        x509.KeyUsage(
-    #            digital_signature=digital_signature,
-    #            content_commitment=content_commitment,
-    #            key_encipherment=key_enipherment,
-    #            data_encipherment=data_encipherment,
-    #            key_agreement=key_agreement,
-    #            key_cert_sign=key_cert_sign,
-    #            crl_sign=crl_sign,
-    #            encipher_only=enchipher_only,
-    #            decipher_only=decipher_only
-    #        ), critical=True
-    #    )
-    #
-    #    # we must maintain our own list of OIDs here
-    #    builder.add_extension(
-    #        x509.ExtendedKeyUsage(
-    #            server_authentication=server_authentication,
-    #            email=
-    #        )
-    #    )
-    #
-    #    builder.add_extension(
-    #        x509.AuthorityInformationAccess()
-    #    )
-    #
-    #    builder.add_extension(
-    #        x509.AuthorityKeyIdentifier()
-    #    )
-    #
-    #    builder.add_extension(
-    #        x509.SubjectKeyIdentifier()
-    #    )
-    #
-    #    builder.add_extension(
-    #        x509.CRLDistributionPoints()
-    #    )
-    #
-    #    builder.add_extension(
-    #        x509.ObjectIdentifier(oid)
-    #    )
 
     request = builder.sign(
         private_key, hashes.SHA256(), default_backend()
     )
 
     # serialize our private key and CSR
-    pem = private_key.private_bytes(
+    private_key = private_key.private_bytes(
         encoding=serialization.Encoding.PEM,
         format=serialization.PrivateFormat.TraditionalOpenSSL,  # would like to use PKCS8 but AWS ELBs don't like it
         encryption_algorithm=serialization.NoEncryption()
     )
 
+    if isinstance(private_key, bytes):
+        private_key = private_key.decode('utf-8')
+
     csr = request.public_bytes(
         encoding=serialization.Encoding.PEM
-    )
+    ).decode('utf-8')
 
-    return csr, pem
+    return csr, private_key
 
 
 def stats(**kwargs):
@@ -473,3 +463,88 @@ def stats(**kwargs):
         values.append(count)
 
     return {'labels': keys, 'values': values}
+
+
+def get_account_number(arn):
+    """
+    Extract the account number from an arn.
+
+    :param arn: IAM SSL arn
+    :return: account number associated with ARN
+    """
+    return arn.split(":")[4]
+
+
+def get_name_from_arn(arn):
+    """
+    Extract the certificate name from an arn.
+
+    :param arn: IAM SSL arn
+    :return: name of the certificate as uploaded to AWS
+    """
+    return arn.split("/", 1)[1]
+
+
+def calculate_reissue_range(start, end):
+    """
+    Determine what the new validity_start and validity_end dates should be.
+    :param start:
+    :param end:
+    :return:
+    """
+    span = end - start
+
+    new_start = arrow.utcnow()
+    new_end = new_start + span
+
+    return new_start, arrow.get(new_end)
+
+
+def get_certificate_primitives(certificate):
+    """
+    Retrieve key primitive from a certificate such that the certificate
+    could be recreated with new expiration or be used to build upon.
+    :param certificate:
+    :return: dict of certificate primitives, should be enough to effectively re-issue
+    certificate via `create`.
+    """
+    start, end = calculate_reissue_range(certificate.not_before, certificate.not_after)
+    data = CertificateInputSchema().load(CertificateOutputSchema().dump(certificate).data).data
+
+    # we can't quite tell if we are using a custom name, as this is an automated process (typically)
+    # we will rely on the Lemur generated name
+    data.pop('name', None)
+
+    # TODO this can be removed once we migrate away from cn
+    data['cn'] = data['common_name']
+
+    # needed until we move off not_*
+    data['not_before'] = start
+    data['not_after'] = end
+    data['validity_start'] = start
+    data['validity_end'] = end
+    return data
+
+
+def reissue_certificate(certificate, replace=None, user=None):
+    """
+    Reissue certificate with the same properties of the given certificate.
+    :param certificate:
+    :param replace:
+    :param user:
+    :return:
+    """
+    primitives = get_certificate_primitives(certificate)
+
+    if not user:
+        primitives['creator'] = certificate.user
+
+    else:
+        primitives['creator'] = user
+
+    if replace:
+        primitives['replaces'] = [certificate]
+
+    new_cert = create(**primitives)
+
+    return new_cert

lemur/certificates/verify.py

@@ -1,25 +1,25 @@
 """
 .. module: lemur.certificates.verify
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import requests
 import subprocess
-from OpenSSL import crypto
+from requests.exceptions import ConnectionError, InvalidSchema
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 
-from flask import current_app
 from lemur.utils import mktempfile
+from lemur.common.utils import parse_certificate
 
 
 def ocsp_verify(cert_path, issuer_chain_path):
     """
     Attempts to verify a certificate via OCSP. OCSP is a more modern version
     of CRL in that it will query the OCSP URI in order to determine if the
-    certificate as been revoked
+    certificate has been revoked
 
     :param cert_path:
     :param issuer_chain_path:
@@ -33,13 +33,16 @@ def ocsp_verify(cert_path, issuer_chain_path):
                            '-cert', cert_path, "-url", url.strip()], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     message, err = p2.communicate()
-    if 'error' in message or 'Error' in message:
+
+    p_message = message.decode('utf-8')
+
+    if 'error' in p_message or 'Error' in p_message:
         raise Exception("Got error when parsing OCSP url")
 
-    elif 'revoked' in message:
+    elif 'revoked' in p_message:
         return
 
-    elif 'good' not in message:
+    elif 'good' not in p_message:
         raise Exception("Did not receive a valid response")
 
     return True
@@ -54,17 +57,39 @@ def crl_verify(cert_path):
     :raise Exception: If certificate does not have CRL
     """
     with open(cert_path, 'rt') as c:
-        cert = x509.load_pem_x509_certificate(c.read(), default_backend())
+        cert = parse_certificate(c.read())
 
     distribution_points = cert.extensions.get_extension_for_oid(x509.OID_CRL_DISTRIBUTION_POINTS).value
+
     for p in distribution_points:
         point = p.full_name[0].value
-        response = requests.get(point)
+
-        crl = crypto.load_crl(crypto.FILETYPE_ASN1, response.content)  # TODO this should be switched to cryptography when support exists
+        try:
-        revoked = crl.get_revoked()
+            response = requests.get(point)
-        for r in revoked:
+
-            if cert.serial == r.get_serial():
+            if response.status_code != 200:
+                raise Exception("Unable to retrieve CRL: {0}".format(point))
+        except InvalidSchema:
+            # Unhandled URI scheme (like ldap://); skip this distribution point.
+            continue
+        except ConnectionError:
+            raise Exception("Unable to retrieve CRL: {0}".format(point))
+
+        crl = x509.load_der_x509_crl(response.content, backend=default_backend())
+
+        for r in crl:
+            if cert.serial == r.serial_number:
+                try:
+                    reason = r.extensions.get_extension_for_class(x509.CRLReason).value
+                    # Handle "removeFromCRL" revoke reason as unrevoked; continue with the next distribution point.
+                    # Per RFC 5280 section 6.3.3 (k): https://tools.ietf.org/html/rfc5280#section-6.3.3
+                    if reason == x509.ReasonFlags.remove_from_crl:
+                        break
+                except x509.ExtensionNotFound:
+                    pass
+
                 return
+
     return True
 
 
@@ -81,13 +106,10 @@ def verify(cert_path, issuer_chain_path):
     try:
         return ocsp_verify(cert_path, issuer_chain_path)
     except Exception as e:
-        current_app.logger.debug("Could not use OCSP: {0}".format(e))
         try:
             return crl_verify(cert_path)
         except Exception as e:
-            current_app.logger.debug("Could not use CRL: {0}".format(e))
             raise Exception("Failed to verify")
-        raise Exception("Failed to verify")
 
 
 def verify_string(cert_string, issuer_string):

lemur/certificates/views.py

@@ -1,97 +1,41 @@
 """
 .. module: lemur.certificates.views
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import base64
 from builtins import str
-from flask import Blueprint, make_response, jsonify
+
-from flask.ext.restful import reqparse, Api, fields
+from flask import Blueprint, make_response, jsonify, g
-from cryptography import x509
+from flask_restful import reqparse, Api
-from cryptography.hazmat.backends import default_backend
+
-from cryptography.hazmat.primitives import serialization
+from lemur.common.schema import validate_schema
-from lemur.certificates import service
+from lemur.common.utils import paginated_parser
-from lemur.authorities.models import Authority
+
 from lemur.auth.service import AuthenticatedResource
-from lemur.auth.permissions import ViewKeyPermission, AuthorityPermission, UpdateCertificatePermission
+from lemur.auth.permissions import AuthorityPermission, CertificatePermission
+
+from lemur.certificates import service
+from lemur.certificates.models import Certificate
+from lemur.plugins.base import plugins
+from lemur.certificates.schemas import (
+    certificate_input_schema,
+    certificate_output_schema,
+    certificate_upload_input_schema,
+    certificates_output_schema,
+    certificate_export_input_schema,
+    certificate_edit_input_schema
+)
+
 from lemur.roles import service as role_service
-from lemur.common.utils import marshal_items, paginated_parser
+from lemur.logs import service as log_service
-from lemur.notifications.views import notification_list
+
 
 mod = Blueprint('certificates', __name__)
 api = Api(mod)
 
-FIELDS = {
-    'name': fields.String,
-    'id': fields.Integer,
-    'bits': fields.Integer,
-    'deleted': fields.String,
-    'issuer': fields.String,
-    'serial': fields.String,
-    'owner': fields.String,
-    'chain': fields.String,
-    'san': fields.String,
-    'active': fields.Boolean,
-    'description': fields.String,
-    'notBefore': fields.DateTime(dt_format='iso8601', attribute='not_before'),
-    'notAfter': fields.DateTime(dt_format='iso8601', attribute='not_after'),
-    'cn': fields.String,
-    'signingAlgorithm': fields.String(attribute='signing_algorithm'),
-    'status': fields.String,
-    'body': fields.String
-}
-
-
-def valid_authority(authority_options):
-    """
-    Defends against invalid authorities
-
-    :param authority_options:
-    :return: :raise ValueError:
-    """
-    name = authority_options['name']
-    authority = Authority.query.filter(Authority.name == name).one()
-
-    if not authority:
-        raise ValueError("Unable to find authority specified")
-
-    if not authority.active:
-        raise ValueError("Selected authority [{0}] is not currently active".format(name))
-
-    return authority
-
-
-def pem_str(value, name):
-    """
-    Used to validate that the given string is a PEM formatted string
-
-    :param value:
-    :param name:
-    :return: :raise ValueError:
-    """
-    try:
-        x509.load_pem_x509_certificate(bytes(value), default_backend())
-    except Exception:
-        raise ValueError("The parameter '{0}' needs to be a valid PEM string".format(name))
-    return value
-
-
-def private_key_str(value, name):
-    """
-    User to validate that a given string is a RSA private key
-
-    :param value:
-    :param name:
-    :return: :raise ValueError:
-    """
-    try:
-        serialization.load_pem_private_key(bytes(value), None, backend=default_backend())
-    except Exception:
-        raise ValueError("The parameter '{0}' needs to be a valid RSA private key".format(name))
-    return value
-
 
 class CertificatesList(AuthenticatedResource):
     """ Defines the 'certificates' endpoint """
@@ -100,7 +44,7 @@ class CertificatesList(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(CertificatesList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, certificates_output_schema)
     def get(self):
         """
         .. http:get:: /certificates
@@ -124,37 +68,66 @@ class CertificatesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
+                "items": [{
-                    {
+                    "status": null,
-                      "id": 1,
+                    "cn": "*.test.example.net",
-                      "name": "cert1",
+                    "chain": "",
-                      "description": "this is cert1",
+                    "authority": {
-                      "bits": 2048,
+                        "active": true,
-                      "deleted": false,
+                        "owner": "secure@example.com",
-                      "issuer": "ExampeInc.",
+                        "id": 1,
-                      "serial": "123450",
+                        "description": "verisign test authority",
-                      "chain": "-----Begin ...",
+                        "name": "verisign"
-                      "body": "-----Begin ...",
+                    },
-                      "san": true,
+                    "owner": "joe@example.com",
-                      "owner": 'bob@example.com",
+                    "serial": "82311058732025924142789179368889309156",
-                      "active": true,
+                    "id": 2288,
-                      "notBefore": "2015-06-05T17:09:39",
+                    "issuer": "SymantecCorporation",
-                      "notAfter": "2015-06-10T17:09:39",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
-                      "cn": "example.com",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
-                      "status": "unknown"
+                    "destinations": [],
-                    }
+                    "bits": 2048,
-                  ]
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }],
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "replaced": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
                 "total": 1
               }
 
            :query sortBy: field to sort on
-           :query sortDir: acs or desc
+           :query sortDir: asc or desc
            :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
+           :query filter: key value pair format is k;v
-           :query limit: limit number. default is 10
+           :query count: count number. default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
         parser = paginated_parser.copy()
         parser.add_argument('timeRange', type=int, dest='time_range', location='args')
@@ -166,10 +139,11 @@ class CertificatesList(AuthenticatedResource):
         parser.add_argument('show', type=str, location='args')
 
         args = parser.parse_args()
+        args['user'] = g.user
         return service.render(args)
 
-    @marshal_items(FIELDS)
+    @validate_schema(certificate_input_schema, certificate_output_schema)
-    def post(self):
+    def post(self, data=None):
         """
         .. http:post:: /certificates
 
@@ -184,90 +158,38 @@ class CertificatesList(AuthenticatedResource):
               Accept: application/json, text/javascript
 
               {
-                "country": "US",
+                  "owner": "secure@example.net",
-                "state": "CA",
+                  "commonName": "test.example.net",
-                "location": "A Place",
+                  "country": "US",
-                "organization": "ExampleInc.",
+                  "extensions": {
-                "organizationalUnit": "Operations",
-                "owner": "bob@example.com",
-                "description": "test",
-                "selectedAuthority": "timetest2",
-                "csr",
-                "authority": {
-                    "body": "-----BEGIN...",
-                    "name": "timetest2",
-                    "chain": "",
-                    "notBefore": "2015-06-05T15:20:59",
-                    "active": true,
-                    "id": 50,
-                    "notAfter": "2015-06-17T15:21:08",
-                    "description": "dsfdsf"
-                },
-                "notifications": [
-                    {
-                      "description": "Default 30 day expiration notification",
-                      "notificationOptions": [
-                        {
-                          "name": "interval",
-                          "required": true,
-                          "value": 30,
-                          "helpMessage": "Number of days to be alert before expiration.",
-                          "validation": "^\\d+$",
-                          "type": "int"
-                        },
-                        {
-                          "available": [
-                            "days",
-                            "weeks",
-                            "months"
-                          ],
-                          "name": "unit",
-                          "required": true,
-                          "value": "days",
-                          "helpMessage": "Interval unit",
-                          "validation": "",
-                          "type": "select"
-                        },
-                        {
-                          "name": "recipients",
-                          "required": true,
-                          "value": "bob@example.com",
-                          "helpMessage": "Comma delimited list of email addresses",
-                          "validation": "^([\\w+-.%]+@[\\w-.]+\\.[A-Za-z]{2,4},?)+$",
-                            "type": "str"
-                          }
-                        ],
-                        "label": "DEFAULT_KGLISSON_30_DAY",
-                        "pluginName": "email-notification",
-                        "active": true,
-                        "id": 7
-                    }
-                ],
-                "extensions": {
-                    "basicConstraints": {},
-                    "keyUsage": {
-                        "isCritical": true,
-                        "useKeyEncipherment": true,
-                        "useDigitalSignature": true
-                    },
-                    "extendedKeyUsage": {
-                        "isCritical": true,
-                        "useServerAuthentication": true
-                    },
-                    "subjectKeyIdentifier": {
-                        "includeSKI": true
-                    },
                     "subAltNames": {
-                        "names": []
+                      "names": [
+                        {
+                          "nameType": "DNSName",
+                          "value": "*.test.example.net"
+                        },
+                        {
+                          "nameType": "DNSName",
+                          "value": "www.test.example.net"
+                        }
+                      ]
                     }
-                },
+                  },
-                "commonName": "test",
+                  "replacements": [{
-                "validityStart": "2015-06-05T07:00:00.000Z",
+                    "id": 1
-                "validityEnd": "2015-06-16T07:00:00.000Z",
+                  }],
-                "replacements": [
+                  "notify": true,
-                    {'id': 123}
+                  "validityEnd": "2026-01-01T08:00:00.000Z",
-                ]
+                  "authority": {
-             }
+                    "name": "verisign"
+                  },
+                  "organization": "Netflix, Inc.",
+                  "location": "Los Gatos",
+                  "state": "California",
+                  "validityStart": "2016-11-11T04:19:48.000Z",
+                  "organizationalUnit": "Operations"
+              }
+
 
            **Example response**:
 
@@ -278,72 +200,80 @@ class CertificatesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
+                "status": null,
-                "name": "cert1",
+                "cn": "*.test.example.net",
-                "description": "this is cert1",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
                 "bits": 2048,
-                "deleted": false,
+                "body": "-----BEGIN CERTIFICATE-----...",
-                "issuer": "ExampeInc.",
+                "description": null,
-                "serial": "123450",
+                "deleted": null,
-                "chain": "-----Begin ...",
+                "notifications": [{
-                "body": "-----Begin ...",
+                    "id": 1
-                "san": true,
+                }],
-                "owner": "jimbob@example.com",
+                "signingAlgorithm": "sha256",
-                "active": false,
+                "user": {
-                "notBefore": "2015-06-05T17:09:39",
+                    "username": "jane",
-                "notAfter": "2015-06-10T17:09:39",
+                    "active": true,
-                "cn": "example.com",
+                    "email": "jane@example.com",
-                "status": "unknown"
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [{
+                    "id": 1
+                }],
+                "rotation": true,
+                "rotationPolicy": {"name": "default"},
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "san": null
               }
 
-           :arg extensions: extensions to be used in the certificate
-           :arg description: description for new certificate
-           :arg owner: owner email
-           :arg validityStart: when the certificate should start being valid
-           :arg validityEnd: when the certificate should expire
-           :arg authority: authority that should issue the certificate
-           :arg country: country for the CSR
-           :arg state: state for the CSR
-           :arg location: location for the CSR
-           :arg organization: organization for CSR
-           :arg commonName: certiifcate common name
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
-        self.reqparse.add_argument('extensions', type=dict, location='json')
+        role = role_service.get_by_name(data['authority'].owner)
-        self.reqparse.add_argument('destinations', type=list, default=[], location='json')
-        self.reqparse.add_argument('notifications', type=list, default=[], location='json')
-        self.reqparse.add_argument('replacements', type=list, default=[], location='json')
-        self.reqparse.add_argument('validityStart', type=str, location='json')  # TODO validate
-        self.reqparse.add_argument('validityEnd', type=str, location='json')  # TODO validate
-        self.reqparse.add_argument('authority', type=valid_authority, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-        self.reqparse.add_argument('country', type=str, location='json', required=True)
-        self.reqparse.add_argument('state', type=str, location='json', required=True)
-        self.reqparse.add_argument('location', type=str, location='json', required=True)
-        self.reqparse.add_argument('organization', type=str, location='json', required=True)
-        self.reqparse.add_argument('organizationalUnit', type=str, location='json', required=True)
-        self.reqparse.add_argument('owner', type=str, location='json', required=True)
-        self.reqparse.add_argument('commonName', type=str, location='json', required=True)
-        self.reqparse.add_argument('csr', type=str, location='json')
-
-        args = self.reqparse.parse_args()
-
-        authority = args['authority']
-        role = role_service.get_by_name(authority.owner)
 
         # all the authority role members should be allowed
-        roles = [x.name for x in authority.roles]
+        roles = [x.name for x in data['authority'].roles]
 
         # allow "owner" roles by team DL
         roles.append(role)
-        permission = AuthorityPermission(authority.id, roles)
+        authority_permission = AuthorityPermission(data['authority'].id, roles)
 
-        if permission.can():
+        if authority_permission.can():
-            return service.create(**args)
+            data['creator'] = g.user
+            cert = service.create(**data)
+            if isinstance(cert, Certificate):
+                # only log if created, not pending
+                log_service.create(g.user, 'create_cert', certificate=cert)
+            return cert
 
-        return dict(message="You are not authorized to use {0}".format(args['authority'].name)), 403
+        return dict(message="You are not authorized to use the authority: {0}".format(data['authority'].name)), 403
 
 
 class CertificatesUpload(AuthenticatedResource):
@@ -353,8 +283,8 @@ class CertificatesUpload(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(CertificatesUpload, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(certificate_upload_input_schema, certificate_output_schema)
-    def post(self):
+    def post(self, data=None):
         """
         .. http:post:: /certificates/upload
 
@@ -369,13 +299,15 @@ class CertificatesUpload(AuthenticatedResource):
               Accept: application/json, text/javascript
 
               {
-                 "owner": "joe@exmaple.com",
+                 "owner": "joe@example.com",
-                 "publicCert": "---Begin Public...",
+                 "body": "-----BEGIN CERTIFICATE-----...",
-                 "intermediateCert": "---Begin Public...",
+                 "chain": "-----BEGIN CERTIFICATE-----...",
-                 "privateKey": "---Begin Private..."
+                 "privateKey": "-----BEGIN RSA PRIVATE KEY-----..."
                  "destinations": [],
                  "notifications": [],
                  "replacements": [],
+                 "roles": [],
+                 "notify": true,
                  "name": "cert1"
               }
 
@@ -388,51 +320,67 @@ class CertificatesUpload(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                 "id": 1,
+                "status": null,
-                 "name": "cert1",
+                "cn": "*.test.example.net",
-                 "description": "this is cert1",
+                "chain": "",
-                 "bits": 2048,
+                "authority": {
-                 "deleted": false,
+                    "active": true,
-                 "issuer": "ExampeInc.",
+                    "owner": "secure@example.com",
-                 "serial": "123450",
+                    "id": 1,
-                 "chain": "-----Begin ...",
+                    "description": "verisign test authority",
-                 "body": "-----Begin ...",
+                    "name": "verisign"
-                 "san": true,
+                },
-                 "owner": "joe@example.com",
+                "owner": "joe@example.com",
-                 "active": true,
+                "serial": "82311058732025924142789179368889309156",
-                 "notBefore": "2015-06-05T17:09:39",
+                "id": 2288,
-                 "notAfter": "2015-06-10T17:09:39",
+                "issuer": "SymantecCorporation",
-                 "signingAlgorithm": "sha2"
+                "notBefore": "2016-06-03T00:00:00+00:00",
-                 "cn": "example.com",
+                "notAfter": "2018-01-12T23:59:59+00:00",
-                 "status": "unknown"
+                "destinations": [],
+                "bits": 2048,
+                "body": "-----BEGIN CERTIFICATE-----...",
+                "description": null,
+                "deleted": null,
+                "notifications": [{
+                    "id": 1
+                }],
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [],
+                "rotation": true,
+                "rotationPolicy": {"name": "default"},
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "san": null
               }
 
-           :arg owner: owner email for certificate
-           :arg publicCert: valid PEM public key for certificate
-           :arg intermediateCert valid PEM intermediate key for certificate
-           :arg privateKey: valid PEM private key for certificate
-           :arg destinations: list of aws destinations to upload the certificate to
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 403: unauthenticated
            :statuscode 200: no error
-        """
-        self.reqparse.add_argument('description', type=str, location='json')
-        self.reqparse.add_argument('owner', type=str, required=True, location='json')
-        self.reqparse.add_argument('name', type=str, location='json')
-        self.reqparse.add_argument('publicCert', type=pem_str, required=True, dest='public_cert', location='json')
-        self.reqparse.add_argument('destinations', type=list, default=[], location='json')
-        self.reqparse.add_argument('notifications', type=list, default=[], location='json')
-        self.reqparse.add_argument('replacements', type=list, default=[], location='json')
-        self.reqparse.add_argument('intermediateCert', type=pem_str, dest='intermediate_cert', location='json')
-        self.reqparse.add_argument('privateKey', type=private_key_str, dest='private_key', location='json')
 
-        args = self.reqparse.parse_args()
+        """
-        if args.get('destinations'):
+        data['creator'] = g.user
-            if args.get('private_key'):
+        if data.get('destinations'):
-                return service.upload(**args)
+            if data.get('private_key'):
+                return service.upload(**data)
             else:
                 raise Exception("Private key must be provided in order to upload certificate to AWS")
-        return service.upload(**args)
+        return service.upload(**data)
 
 
 class CertificatesStats(AuthenticatedResource):
@@ -481,7 +429,7 @@ class CertificatePrivateKey(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                 "key": "----Begin ...",
+                 "key": "-----BEGIN ..."
               }
 
            :reqheader Authorization: OAuth token to authenticate
@@ -492,17 +440,19 @@ class CertificatePrivateKey(AuthenticatedResource):
         if not cert:
             return dict(message="Cannot find specified certificate"), 404
 
-        role = role_service.get_by_name(cert.owner)
+        # allow creators
+        if g.current_user != cert.user:
+            owner_role = role_service.get_by_name(cert.owner)
+            permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
 
-        permission = ViewKeyPermission(certificate_id, getattr(role, 'name', None))
+            if not permission.can():
+                return dict(message='You are not authorized to view this key'), 403
 
-        if permission.can():
+        log_service.create(g.current_user, 'key_view', certificate=cert)
-            response = make_response(jsonify(key=cert.private_key), 200)
+        response = make_response(jsonify(key=cert.private_key), 200)
-            response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store'
+        response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store'
-            response.headers['pragma'] = 'no-cache'
+        response.headers['pragma'] = 'no-cache'
-            return response
+        return response
-
-        return dict(message='You are not authorized to view this key'), 403
 
 
 class Certificates(AuthenticatedResource):
@@ -510,7 +460,7 @@ class Certificates(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Certificates, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, certificate_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1
@@ -534,33 +484,65 @@ class Certificates(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
+                "status": null,
-                "name": "cert1",
+                "cn": "*.test.example.net",
-                "description": "this is cert1",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
                 "bits": 2048,
-                "deleted": false,
+                "body": "-----BEGIN CERTIFICATE-----...",
-                "issuer": "ExampeInc.",
+                "description": null,
-                "serial": "123450",
+                "deleted": null,
-                "chain": "-----Begin ...",
+                "notifications": [{
-                "body": "-----Begin ...",
+                    "id": 1
-                "san": true,
+                }],
-                "owner": "bob@example.com",
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
+                "domains": [{
-                "notAfter": "2015-06-10T17:09:39",
+                    "sensitive": false,
-                "signingAlgorithm": "sha2",
+                    "id": 1090,
-                "cn": "example.com",
+                    "name": "*.test.example.net"
-                "status": "unknown"
+                }],
+                "rotation": true,
+                "rotationPolicy": {"name": "default"},
+                "replaces": [],
+                "replaced": [],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "san": null
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
         return service.get(certificate_id)
 
-    @marshal_items(FIELDS)
+    @validate_schema(certificate_edit_input_schema, certificate_output_schema)
-    def put(self, certificate_id):
+    def put(self, certificate_id, data=None):
         """
         .. http:put:: /certificates/1
 
@@ -591,53 +573,85 @@ class Certificates(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
+                "status": null,
-                "name": "cert1",
+                "cn": "*.test.example.net",
-                "description": "this is cert1",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
                 "bits": 2048,
-                "deleted": false,
+                "body": "-----BEGIN CERTIFICATE-----...",
-                "issuer": "ExampeInc.",
+                "description": null,
-                "serial": "123450",
+                "deleted": null,
-                "chain": "-----Begin ...",
+                "notifications": [{
-                "body": "-----Begin ...",
+                    "id": 1
-                "san": true,
+                }]
-                "owner": "jimbob@example.com",
+                "signingAlgorithm": "sha256",
-                "active": false,
+                "user": {
-                "notBefore": "2015-06-05T17:09:39",
+                    "username": "jane",
-                "notAfter": "2015-06-10T17:09:39",
+                    "active": true,
-                "cn": "example.com",
+                    "email": "jane@example.com",
-                "status": "unknown",
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "rotation": true,
+                "rotationPolicy": {"name": "default"},
+                "san": null
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
-        self.reqparse.add_argument('active', type=bool, location='json')
-        self.reqparse.add_argument('owner', type=str, location='json')
-        self.reqparse.add_argument('description', type=str, location='json')
-        self.reqparse.add_argument('destinations', type=list, default=[], location='json')
-        self.reqparse.add_argument('notifications', type=notification_list, default=[], location='json')
-        self.reqparse.add_argument('replacements', type=list, default=[], location='json')
-        args = self.reqparse.parse_args()
-
         cert = service.get(certificate_id)
-        role = role_service.get_by_name(cert.owner)
 
-        permission = UpdateCertificatePermission(certificate_id, getattr(role, 'name', None))
+        if not cert:
+            return dict(message="Cannot find specified certificate"), 404
 
-        if permission.can():
+        # allow creators
-            return service.update(
+        if g.current_user != cert.user:
-                certificate_id,
+            owner_role = role_service.get_by_name(cert.owner)
-                args['owner'],
+            permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
-                args['description'],
-                args['active'],
-                args['destinations'],
-                args['notifications'],
-                args['replacements']
-            )
 
-        return dict(message='You are not authorized to update this certificate'), 403
+            if not permission.can():
+                return dict(message='You are not authorized to update this certificate'), 403
+
+        for destination in data['destinations']:
+            if destination.plugin.requires_key:
+                if not cert.private_key:
+                    return dict(
+                        message='Unable to add destination: {0}. Certificate does not have required private key.'.format(
+                            destination.label
+                        )
+                    ), 400
+
+        cert = service.update(certificate_id, **data)
+        log_service.create(g.current_user, 'update_cert', certificate=cert)
+        return cert
 
 
 class NotificationCertificatesList(AuthenticatedResource):
@@ -647,7 +661,7 @@ class NotificationCertificatesList(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(NotificationCertificatesList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, certificates_output_schema)
     def get(self, notification_id):
         """
         .. http:get:: /notifications/1/certificates
@@ -671,38 +685,68 @@ class NotificationCertificatesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
+                "items": [{
-                    {
+                    "status": null,
-                      "id": 1,
+                    "cn": "*.test.example.net",
-                      "name": "cert1",
+                    "chain": "",
-                      "description": "this is cert1",
+                    "authority": {
-                      "bits": 2048,
+                        "active": true,
-                      "deleted": false,
+                        "owner": "secure@example.com",
-                      "issuer": "ExampeInc.",
+                        "id": 1,
-                      "serial": "123450",
+                        "description": "verisign test authority",
-                      "chain": "-----Begin ...",
+                        "name": "verisign"
-                      "body": "-----Begin ...",
+                    },
-                      "san": true,
+                    "owner": "joe@example.com",
-                      "owner": 'bob@example.com",
+                    "serial": "82311058732025924142789179368889309156",
-                      "active": true,
+                    "id": 2288,
-                      "notBefore": "2015-06-05T17:09:39",
+                    "issuer": "SymantecCorporation",
-                      "notAfter": "2015-06-10T17:09:39",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
-                      "signingAlgorithm": "sha2",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
-                      "cn": "example.com",
+                    "destinations": [],
-                      "status": "unknown"
+                    "bits": 2048,
-                    }
+                    "body": "-----BEGIN CERTIFICATE-----...",
-                  ]
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }],
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "replaced": [],
+                    "rotation": true,
+                    "rotationPolicy": {"name": "default"},
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
                 "total": 1
               }
 
            :query sortBy: field to sort on
-           :query sortDir: acs or desc
+           :query sortDir: asc or desc
-           :query page: int. default is 1
+           :query page: int default is 1
-           :query filter: key value pair. format is k=v;
+           :query filter: key value pair format is k;v
-           :query limit: limit number. default is 10
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
         parser = paginated_parser.copy()
         parser.add_argument('timeRange', type=int, dest='time_range', location='args')
@@ -715,6 +759,7 @@ class NotificationCertificatesList(AuthenticatedResource):
 
         args = parser.parse_args()
         args['notification_id'] = notification_id
+        args['user'] = g.current_user
         return service.render(args)
 
 
@@ -723,7 +768,7 @@ class CertificatesReplacementsList(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(CertificatesReplacementsList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, certificates_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/replacements
@@ -746,29 +791,64 @@ class CertificatesReplacementsList(AuthenticatedResource):
               Vary: Accept
               Content-Type: text/javascript
 
-              [{
+              {
-                "id": 1,
+                "items": [{
-                "name": "cert1",
+                    "status": null,
-                "description": "this is cert1",
+                    "cn": "*.test.example.net",
-                "bits": 2048,
+                    "chain": "",
-                "deleted": false,
+                    "authority": {
-                "issuer": "ExampeInc.",
+                        "active": true,
-                "serial": "123450",
+                        "owner": "secure@example.com",
-                "chain": "-----Begin ...",
+                        "id": 1,
-                "body": "-----Begin ...",
+                        "description": "verisign test authority",
-                "san": true,
+                        "name": "verisign"
-                "owner": "bob@example.com",
+                    },
-                "active": true,
+                    "owner": "joe@example.com",
-                "notBefore": "2015-06-05T17:09:39",
+                    "serial": "82311058732025924142789179368889309156",
-                "notAfter": "2015-06-10T17:09:39",
+                    "id": 2288,
-                "signingAlgorithm": "sha2",
+                    "issuer": "SymantecCorporation",
-                "cn": "example.com",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
-                "status": "unknown"
+                    "notAfter": "2018-01-12T23:59:59+00:00",
-              }]
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }]
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "replaced": [],
+                    "rotation": true,
+                    "rotationPolicy": {"name": "default"},
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
+                "total": 1
+              }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
         return service.get(certificate_id).replaces
 
@@ -778,7 +858,8 @@ class CertificateExport(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(CertificateExport, self).__init__()
 
-    def post(self, certificate_id):
+    @validate_schema(certificate_export_input_schema, None)
+    def post(self, certificate_id, data=None):
         """
         .. http:post:: /certificates/1/export
 
@@ -842,23 +923,102 @@ class CertificateExport(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
-        self.reqparse.add_argument('export', type=dict, required=True, location='json')
-        args = self.reqparse.parse_args()
-
         cert = service.get(certificate_id)
-        role = role_service.get_by_name(cert.owner)
 
-        permission = UpdateCertificatePermission(certificate_id, getattr(role, 'name', None))
+        if not cert:
+            return dict(message="Cannot find specified certificate"), 404
 
-        if permission.can():
+        plugin = data['plugin']['plugin_object']
-            extension, passphrase, data = service.export(cert, args['export']['plugin'])
-            # we take a hit in message size when b64 encoding
-            return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data))
 
-        return dict(message='You are not authorized to export this certificate'), 403
+        if plugin.requires_key:
+            if not cert.private_key:
+                return dict(
+                    message='Unable to export certificate, plugin: {0} requires a private key but no key was found.'.format(
+                        plugin.slug)), 400
+
+            else:
+                # allow creators
+                if g.current_user != cert.user:
+                    owner_role = role_service.get_by_name(cert.owner)
+                    permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+
+                    if not permission.can():
+                        return dict(message='You are not authorized to export this certificate.'), 403
+
+        options = data['plugin']['plugin_options']
+
+        log_service.create(g.current_user, 'key_view', certificate=cert)
+        extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
+
+        # we take a hit in message size when b64 encoding
+        return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data).decode('utf-8'))
 
 
+class CertificateRevoke(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificateRevoke, self).__init__()
+
+    @validate_schema(None, None)
+    def put(self, certificate_id, data=None):
+        """
+        .. http:put:: /certificates/1/revoke
+
+           Revoke a certificate
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              POST /certificates/1/revoke HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                'id': 1
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        cert = service.get(certificate_id)
+
+        if not cert:
+            return dict(message="Cannot find specified certificate"), 404
+
+        # allow creators
+        if g.current_user != cert.user:
+            owner_role = role_service.get_by_name(cert.owner)
+            permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+
+            if not permission.can():
+                return dict(message='You are not authorized to revoke this certificate.'), 403
+
+        if not cert.external_id:
+            return dict(message='Cannot revoke certificate. No external id found.'), 400
+
+        if cert.endpoints:
+            return dict(message='Cannot revoke certificate. Endpoints are deployed with the given certificate.'), 403
+
+        plugin = plugins.get(cert.authority.plugin_name)
+        plugin.revoke_certificate(cert, data)
+        log_service.create(g.current_user, 'revoke_cert', certificate=cert)
+        return dict(id=cert.id)
+
+
+api.add_resource(CertificateRevoke, '/certificates/<int:certificate_id>/revoke', endpoint='revokeCertificate')
 api.add_resource(CertificatesList, '/certificates', endpoint='certificates')
 api.add_resource(Certificates, '/certificates/<int:certificate_id>', endpoint='certificate')
 api.add_resource(CertificatesStats, '/certificates/stats', endpoint='certificateStats')

lemur/common/defaults.py

@@ -0,0 +1,267 @@
+import re
+import unicodedata
+
+from cryptography import x509
+from flask import current_app
+from lemur.extensions import sentry
+from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
+
+
+def text_to_slug(value):
+    """Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters."""
+
+    # Strip all character accents: decompose Unicode characters and then drop combining chars.
+    value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c))
+
+    # Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash.
+    # Except, keep 'xn--' used in IDNA domain names as is.
+    value = re.sub(r'[^A-Za-z0-9.]+(?<!xn--)', '-', value)
+
+    # '-' in the beginning or end of string looks ugly.
+    return value.strip('-')
+
+
+def certificate_name(common_name, issuer, not_before, not_after, san):
+    """
+    Create a name for our certificate. A naming standard
+    is based on a series of templates. The name includes
+    useful information such as Common Name, Validation dates,
+    and Issuer.
+
+    :param san:
+    :param common_name:
+    :param not_after:
+    :param issuer:
+    :param not_before:
+    :rtype: str
+    :return:
+    """
+    if san:
+        t = SAN_NAMING_TEMPLATE
+    else:
+        t = DEFAULT_NAMING_TEMPLATE
+
+    temp = t.format(
+        subject=common_name,
+        issuer=issuer.replace(' ', ''),
+        not_before=not_before.strftime('%Y%m%d'),
+        not_after=not_after.strftime('%Y%m%d')
+    )
+
+    temp = temp.replace('*', "WILDCARD")
+    return text_to_slug(temp)
+
+
+def signing_algorithm(cert):
+    return cert.signature_hash_algorithm.name
+
+
+def common_name(cert):
+    """
+    Attempts to get a sane common name from a given certificate.
+
+    :param cert:
+    :return: Common name or None
+    """
+    try:
+        return cert.subject.get_attributes_for_oid(
+            x509.OID_COMMON_NAME
+        )[0].value.strip()
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.error("Unable to get common name! {0}".format(e))
+
+
+def organization(cert):
+    """
+    Attempt to get the organization name from a given certificate.
+    :param cert:
+    :return:
+    """
+    try:
+        return cert.subject.get_attributes_for_oid(
+            x509.OID_ORGANIZATION_NAME
+        )[0].value.strip()
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.error("Unable to get organization! {0}".format(e))
+
+
+def organizational_unit(cert):
+    """
+    Attempt to get the organization unit from a given certificate.
+    :param cert:
+    :return:
+    """
+    try:
+        return cert.subject.get_attributes_for_oid(
+            x509.OID_ORGANIZATIONAL_UNIT_NAME
+        )[0].value.strip()
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.error("Unable to get organizational unit! {0}".format(e))
+
+
+def country(cert):
+    """
+    Attempt to get the country from a given certificate.
+    :param cert:
+    :return:
+    """
+    try:
+        return cert.subject.get_attributes_for_oid(
+            x509.OID_COUNTRY_NAME
+        )[0].value.strip()
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.error("Unable to get country! {0}".format(e))
+
+
+def state(cert):
+    """
+    Attempt to get the from a given certificate.
+    :param cert:
+    :return:
+    """
+    try:
+        return cert.subject.get_attributes_for_oid(
+            x509.OID_STATE_OR_PROVINCE_NAME
+        )[0].value.strip()
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.error("Unable to get state! {0}".format(e))
+
+
+def location(cert):
+    """
+    Attempt to get the location name from a given certificate.
+    :param cert:
+    :return:
+    """
+    try:
+        return cert.subject.get_attributes_for_oid(
+            x509.OID_LOCALITY_NAME
+        )[0].value.strip()
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.error("Unable to get location! {0}".format(e))
+
+
+def domains(cert):
+    """
+    Attempts to get an domains listed in a certificate.
+    If 'subjectAltName' extension is not available we simply
+    return the common name.
+
+    :param cert:
+    :return: List of domains
+    """
+    domains = []
+    try:
+        ext = cert.extensions.get_extension_for_oid(x509.OID_SUBJECT_ALTERNATIVE_NAME)
+        entries = ext.value.get_values_for_type(x509.DNSName)
+        for entry in entries:
+            domains.append(entry)
+    except x509.ExtensionNotFound:
+        if current_app.config.get("LOG_SSL_SUBJ_ALT_NAME_ERRORS", True):
+            sentry.captureException()
+    except Exception as e:
+        sentry.captureException()
+
+    return domains
+
+
+def serial(cert):
+    """
+    Fetch the serial number from the certificate.
+
+    :param cert:
+    :return: serial number
+    """
+    return cert.serial_number
+
+
+def san(cert):
+    """
+    Determines if a given certificate is a SAN certificate.
+    SAN certificates are simply certificates that cover multiple domains.
+
+    :param cert:
+    :return: Bool
+    """
+    if len(domains(cert)) > 1:
+        return True
+
+
+def is_wildcard(cert):
+    """
+    Determines if certificate is a wildcard certificate.
+
+    :param cert:
+    :return: Bool
+    """
+    d = domains(cert)
+    if len(d) == 1 and d[0][0:1] == "*":
+        return True
+
+    if cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[0].value[0:1] == "*":
+        return True
+
+
+def bitstrength(cert):
+    """
+    Calculates a certificates public key bit length.
+
+    :param cert:
+    :return: Integer
+    """
+    try:
+        return cert.public_key().key_size
+    except AttributeError:
+        sentry.captureException()
+        current_app.logger.debug('Unable to get bitstrength.')
+
+
+def issuer(cert):
+    """
+    Gets a sane issuer name from a given certificate.
+
+    :param cert:
+    :return: Issuer
+    """
+    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
+    try:
+        # Try organization name or fall back to CN
+        issuer = (cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
+                  or cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME))
+        issuer = str(issuer[0].value)
+        for c in delchars:
+            issuer = issuer.replace(c, "")
+        return issuer
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.error("Unable to get issuer! {0}".format(e))
+        return "Unknown"
+
+
+def not_before(cert):
+    """
+    Gets the naive datetime of the certificates 'not_before' field.
+    This field denotes the first date in time which the given certificate
+    is valid.
+
+    :param cert:
+    :return: Datetime
+    """
+    return cert.not_valid_before
+
+
+def not_after(cert):
+    """
+    Gets the naive datetime of the certificates 'not_after' field.
+    This field denotes the last date in time which the given certificate
+    is valid.
+
+    :return: Datetime
+    """
+    return cert.not_valid_after

lemur/common/fields.py

@@ -0,0 +1,414 @@
+"""
+.. module: lemur.common.fields
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+import arrow
+import warnings
+import ipaddress
+
+from flask import current_app
+from datetime import datetime as dt
+
+from cryptography import x509
+
+from marshmallow import utils
+from marshmallow.fields import Field
+from marshmallow.exceptions import ValidationError
+
+from lemur.common import validators
+
+
+class Hex(Field):
+    """
+    A hex formatted string.
+    """
+    def _serialize(self, value, attr, obj):
+        if value:
+            value = hex(int(value))[2:].upper()
+        return value
+
+
+class ArrowDateTime(Field):
+    """A formatted datetime string in UTC.
+
+    Example: ``'2014-12-22T03:12:58.019077+00:00'``
+
+    Timezone-naive `datetime` objects are converted to
+    UTC (+00:00) by :meth:`Schema.dump <marshmallow.Schema.dump>`.
+    :meth:`Schema.load <marshmallow.Schema.load>` returns `datetime`
+    objects that are timezone-aware.
+
+    :param str format: Either ``"rfc"`` (for RFC822), ``"iso"`` (for ISO8601),
+        or a date format string. If `None`, defaults to "iso".
+    :param kwargs: The same keyword arguments that :class:`Field` receives.
+
+    """
+
+    DATEFORMAT_SERIALIZATION_FUNCS = {
+        'iso': utils.isoformat,
+        'iso8601': utils.isoformat,
+        'rfc': utils.rfcformat,
+        'rfc822': utils.rfcformat,
+    }
+
+    DATEFORMAT_DESERIALIZATION_FUNCS = {
+        'iso': utils.from_iso,
+        'iso8601': utils.from_iso,
+        'rfc': utils.from_rfc,
+        'rfc822': utils.from_rfc,
+    }
+
+    DEFAULT_FORMAT = 'iso'
+
+    localtime = False
+    default_error_messages = {
+        'invalid': 'Not a valid datetime.',
+        'format': '"{input}" cannot be formatted as a datetime.',
+    }
+
+    def __init__(self, format=None, **kwargs):
+        super(ArrowDateTime, self).__init__(**kwargs)
+        # Allow this to be None. It may be set later in the ``_serialize``
+        # or ``_desrialize`` methods This allows a Schema to dynamically set the
+        # dateformat, e.g. from a Meta option
+        self.dateformat = format
+
+    def _add_to_schema(self, field_name, schema):
+        super(ArrowDateTime, self)._add_to_schema(field_name, schema)
+        self.dateformat = self.dateformat or schema.opts.dateformat
+
+    def _serialize(self, value, attr, obj):
+        if value is None:
+            return None
+        self.dateformat = self.dateformat or self.DEFAULT_FORMAT
+        format_func = self.DATEFORMAT_SERIALIZATION_FUNCS.get(self.dateformat, None)
+        if format_func:
+            try:
+                return format_func(value, localtime=self.localtime)
+            except (AttributeError, ValueError) as err:
+                self.fail('format', input=value)
+        else:
+            return value.strftime(self.dateformat)
+
+    def _deserialize(self, value, attr, data):
+        if not value:  # Falsy values, e.g. '', None, [] are not valid
+            raise self.fail('invalid')
+        self.dateformat = self.dateformat or self.DEFAULT_FORMAT
+        func = self.DATEFORMAT_DESERIALIZATION_FUNCS.get(self.dateformat)
+        if func:
+            try:
+                return arrow.get(func(value))
+            except (TypeError, AttributeError, ValueError):
+                raise self.fail('invalid')
+        elif self.dateformat:
+            try:
+                return dt.datetime.strptime(value, self.dateformat)
+            except (TypeError, AttributeError, ValueError):
+                raise self.fail('invalid')
+        elif utils.dateutil_available:
+            try:
+                return arrow.get(utils.from_datestring(value))
+            except TypeError:
+                raise self.fail('invalid')
+        else:
+            warnings.warn('It is recommended that you install python-dateutil '
+                          'for improved datetime deserialization.')
+            raise self.fail('invalid')
+
+
+class KeyUsageExtension(Field):
+    """An x509.KeyUsage ExtensionType object
+
+    Dict of KeyUsage names/values are deserialized into an x509.KeyUsage object
+    and back.
+
+    :param kwargs: The same keyword arguments that :class:`Field` receives.
+
+    """
+
+    def _serialize(self, value, attr, obj):
+        return {
+            'useDigitalSignature': value.digital_signature,
+            'useNonRepudiation': value.content_commitment,
+            'useKeyEncipherment': value.key_encipherment,
+            'useDataEncipherment': value.data_encipherment,
+            'useKeyAgreement': value.key_agreement,
+            'useKeyCertSign': value.key_cert_sign,
+            'useCRLSign': value.crl_sign,
+            'useEncipherOnly': value._encipher_only,
+            'useDecipherOnly': value._decipher_only
+        }
+
+    def _deserialize(self, value, attr, data):
+        keyusages = {
+            'digital_signature': False,
+            'content_commitment': False,
+            'key_encipherment': False,
+            'data_encipherment': False,
+            'key_agreement': False,
+            'key_cert_sign': False,
+            'crl_sign': False,
+            'encipher_only': False,
+            'decipher_only': False
+        }
+
+        for k, v in value.items():
+            if k == 'useDigitalSignature':
+                keyusages['digital_signature'] = v
+
+            elif k == 'useNonRepudiation':
+                keyusages['content_commitment'] = v
+
+            elif k == 'useKeyEncipherment':
+                keyusages['key_encipherment'] = v
+
+            elif k == 'useDataEncipherment':
+                keyusages['data_encipherment'] = v
+
+            elif k == 'useKeyCertSign':
+                keyusages['key_cert_sign'] = v
+
+            elif k == 'useCRLSign':
+                keyusages['crl_sign'] = v
+
+            elif k == 'useKeyAgreement':
+                keyusages['key_agreement'] = v
+
+            elif k == 'useEncipherOnly' and v:
+                keyusages['encipher_only'] = True
+                keyusages['key_agreement'] = True
+
+            elif k == 'useDecipherOnly' and v:
+                keyusages['decipher_only'] = True
+                keyusages['key_agreement'] = True
+
+        if keyusages['encipher_only'] and keyusages['decipher_only']:
+            raise ValidationError('A certificate cannot have both Encipher Only and Decipher Only Extended Key Usages.')
+
+        return x509.KeyUsage(
+            digital_signature=keyusages['digital_signature'],
+            content_commitment=keyusages['content_commitment'],
+            key_encipherment=keyusages['key_encipherment'],
+            data_encipherment=keyusages['data_encipherment'],
+            key_agreement=keyusages['key_agreement'],
+            key_cert_sign=keyusages['key_cert_sign'],
+            crl_sign=keyusages['crl_sign'],
+            encipher_only=keyusages['encipher_only'],
+            decipher_only=keyusages['decipher_only']
+        )
+
+
+class ExtendedKeyUsageExtension(Field):
+    """An x509.ExtendedKeyUsage ExtensionType object
+
+    Dict of ExtendedKeyUsage names/values are deserialized into an x509.ExtendedKeyUsage object
+    and back.
+
+    :param kwargs: The same keyword arguments that :class:`Field` receives.
+
+    """
+
+    def _serialize(self, value, attr, obj):
+        usages = value._usages
+        usage_list = {}
+        for usage in usages:
+            if usage == x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH:
+                usage_list['useClientAuthentication'] = True
+
+            elif usage == x509.oid.ExtendedKeyUsageOID.SERVER_AUTH:
+                usage_list['useServerAuthentication'] = True
+
+            elif usage == x509.oid.ExtendedKeyUsageOID.CODE_SIGNING:
+                usage_list['useCodeSigning'] = True
+
+            elif usage == x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION:
+                usage_list['useEmailProtection'] = True
+
+            elif usage == x509.oid.ExtendedKeyUsageOID.TIME_STAMPING:
+                usage_list['useTimestamping'] = True
+
+            elif usage == x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING:
+                usage_list['useOCSPSigning'] = True
+
+            elif usage.dotted_string == '1.3.6.1.5.5.7.3.14':
+                usage_list['useEapOverLAN'] = True
+
+            elif usage.dotted_string == '1.3.6.1.5.5.7.3.13':
+                usage_list['useEapOverPPP'] = True
+
+            elif usage.dotted_string == '1.3.6.1.4.1.311.20.2.2':
+                usage_list['useSmartCardLogon'] = True
+
+            else:
+                current_app.logger.warning('Unable to serialize ExtendedKeyUsage with OID: {usage}'.format(usage=usage.dotted_string))
+
+        return usage_list
+
+    def _deserialize(self, value, attr, data):
+        usage_oids = []
+        for k, v in value.items():
+            if k == 'useClientAuthentication' and v:
+                usage_oids.append(x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH)
+
+            elif k == 'useServerAuthentication' and v:
+                usage_oids.append(x509.oid.ExtendedKeyUsageOID.SERVER_AUTH)
+
+            elif k == 'useCodeSigning' and v:
+                usage_oids.append(x509.oid.ExtendedKeyUsageOID.CODE_SIGNING)
+
+            elif k == 'useEmailProtection' and v:
+                usage_oids.append(x509.oid.ExtendedKeyUsageOID.EMAIL_PROTECTION)
+
+            elif k == 'useTimestamping' and v:
+                usage_oids.append(x509.oid.ExtendedKeyUsageOID.TIME_STAMPING)
+
+            elif k == 'useOCSPSigning' and v:
+                usage_oids.append(x509.oid.ExtendedKeyUsageOID.OCSP_SIGNING)
+
+            elif k == 'useEapOverLAN' and v:
+                usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.14"))
+
+            elif k == 'useEapOverPPP' and v:
+                usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.5.5.7.3.13"))
+
+            elif k == 'useSmartCardLogon' and v:
+                usage_oids.append(x509.oid.ObjectIdentifier("1.3.6.1.4.1.311.20.2.2"))
+
+            else:
+                current_app.logger.warning('Unable to deserialize ExtendedKeyUsage with name: {key}'.format(key=k))
+
+        return x509.ExtendedKeyUsage(usage_oids)
+
+
+class BasicConstraintsExtension(Field):
+    """An x509.BasicConstraints ExtensionType object
+
+    Dict of CA boolean and a path_length integer names/values are deserialized into an x509.BasicConstraints object
+    and back.
+
+    :param kwargs: The same keyword arguments that :class:`Field` receives.
+
+    """
+
+    def _serialize(self, value, attr, obj):
+        return {'ca': value.ca, 'path_length': value.path_length}
+
+    def _deserialize(self, value, attr, data):
+        ca = value.get('ca', False)
+        path_length = value.get('path_length', None)
+
+        if ca:
+            if not isinstance(path_length, (type(None), int)):
+                raise ValidationError('A CA certificate path_length (for BasicConstraints) must be None or an integer.')
+            return x509.BasicConstraints(ca=True, path_length=path_length)
+        else:
+            return x509.BasicConstraints(ca=False, path_length=None)
+
+
+class SubjectAlternativeNameExtension(Field):
+    """An x509.SubjectAlternativeName ExtensionType object
+
+    Dict of CA boolean and a path_length integer names/values are deserialized into an x509.BasicConstraints object
+    and back.
+
+    :param kwargs: The same keyword arguments that :class:`Field` receives.
+
+    """
+    def _serialize(self, value, attr, obj):
+        general_names = []
+        name_type = None
+
+        if value:
+            for name in value._general_names:
+                value = name.value
+
+                if isinstance(name, x509.DNSName):
+                    name_type = 'DNSName'
+
+                elif isinstance(name, x509.IPAddress):
+                    if isinstance(value, ipaddress.IPv4Network):
+                        name_type = 'IPNetwork'
+                    else:
+                        name_type = 'IPAddress'
+
+                    value = str(value)
+
+                elif isinstance(name, x509.UniformResourceIdentifier):
+                    name_type = 'uniformResourceIdentifier'
+
+                elif isinstance(name, x509.DirectoryName):
+                    name_type = 'directoryName'
+
+                elif isinstance(name, x509.RFC822Name):
+                    name_type = 'rfc822Name'
+
+                elif isinstance(name, x509.RegisteredID):
+                    name_type = 'registeredID'
+                    value = value.dotted_string
+                else:
+                    current_app.logger.warning('Unknown SubAltName type: {name}'.format(name=name))
+
+                general_names.append({'nameType': name_type, 'value': value})
+
+        return general_names
+
+    def _deserialize(self, value, attr, data):
+        general_names = []
+        for name in value:
+            if name['nameType'] == 'DNSName':
+                validators.sensitive_domain(name['value'])
+                general_names.append(x509.DNSName(name['value']))
+
+            elif name['nameType'] == 'IPAddress':
+                general_names.append(x509.IPAddress(ipaddress.ip_address(name['value'])))
+
+            elif name['nameType'] == 'IPNetwork':
+                general_names.append(x509.IPAddress(ipaddress.ip_network(name['value'])))
+
+            elif name['nameType'] == 'uniformResourceIdentifier':
+                general_names.append(x509.UniformResourceIdentifier(name['value']))
+
+            elif name['nameType'] == 'directoryName':
+                # TODO: Need to parse a string in name['value'] like:
+                # 'CN=Common Name, O=Org Name, OU=OrgUnit Name, C=US, ST=ST, L=City/emailAddress=person@example.com'
+                # or
+                # 'CN=Common Name/O=Org Name/OU=OrgUnit Name/C=US/ST=NH/L=City/emailAddress=person@example.com'
+                # and turn it into something like:
+                # x509.Name([
+                #     x509.NameAttribute(x509.OID_COMMON_NAME, "Common Name"),
+                #     x509.NameAttribute(x509.OID_ORGANIZATION_NAME, "Org Name"),
+                #     x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, "OrgUnit Name"),
+                #     x509.NameAttribute(x509.OID_COUNTRY_NAME, "US"),
+                #     x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, "NH"),
+                #     x509.NameAttribute(x509.OID_LOCALITY_NAME, "City"),
+                #     x509.NameAttribute(x509.OID_EMAIL_ADDRESS, "person@example.com")
+                # ]
+                # general_names.append(x509.DirectoryName(x509.Name(BLAH))))
+                pass
+
+            elif name['nameType'] == 'rfc822Name':
+                general_names.append(x509.RFC822Name(name['value']))
+
+            elif name['nameType'] == 'registeredID':
+                general_names.append(x509.RegisteredID(x509.ObjectIdentifier(name['value'])))
+
+            elif name['nameType'] == 'otherName':
+                # This has two inputs (type and value), so it doesn't fit the mold of the rest of these GeneralName entities.
+                # general_names.append(x509.OtherName(name['type'], bytes(name['value']), 'utf-8'))
+                pass
+
+            elif name['nameType'] == 'x400Address':
+                # The Python Cryptography library doesn't support x400Address types (yet?)
+                pass
+
+            elif name['nameType'] == 'EDIPartyName':
+                # The Python Cryptography library doesn't support EDIPartyName types (yet?)
+                pass
+
+            else:
+                current_app.logger.warning('Unable to deserialize SubAltName with type: {name_type}'.format(name_type=name['nameType']))
+
+        return x509.SubjectAlternativeName(general_names)

lemur/common/health.py

@@ -1,16 +1,29 @@
 """
 .. module: lemur.common.health
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import Blueprint
+from lemur.database import db
+from lemur.extensions import sentry
 
 mod = Blueprint('healthCheck', __name__)
 
 
 @mod.route('/healthcheck')
 def health():
-    return 'ok'
+    try:
+        if healthcheck(db):
+            return 'ok'
+    except Exception:
+        sentry.captureException()
+        return 'db check failed'
+
+
+def healthcheck(db):
+    with db.engine.connect() as connection:
+        connection.execute('SELECT 1;')
+    return True

lemur/common/managers.py

@@ -1,13 +1,15 @@
 """
 .. module: lemur.common.managers
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import current_app
 
+from lemur.exceptions import InvalidConfiguration
+
 
 # inspired by https://github.com/getsentry/sentry
 class InstanceManager(object):
@@ -58,9 +60,14 @@ class InstanceManager(object):
                     results.append(cls())
                 else:
                     results.append(cls)
-            except Exception:
+
-                current_app.logger.exception('Unable to import %s', cls_path)
+            except InvalidConfiguration as e:
+                current_app.logger.warning("Plugin '{0}' may not work correctly. {1}".format(class_name, e))
+
+            except Exception as e:
+                current_app.logger.exception("Unable to import {0}. Reason: {1}".format(cls_path, e))
                 continue
+
         self.cache = results
 
         return results

lemur/common/missing.py

@@ -0,0 +1,24 @@
+import arrow
+from flask import current_app
+
+from lemur.common.utils import is_weekend
+
+
+def convert_validity_years(data):
+    """
+    Convert validity years to validity_start and validity_end
+
+    :param data:
+    :return:
+    """
+    if data.get('validity_years'):
+        now = arrow.utcnow()
+        data['validity_start'] = now.isoformat()
+
+        end = now.replace(years=+int(data['validity_years']))
+        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+            if is_weekend(end):
+                end = end.replace(days=-2)
+
+        data['validity_end'] = end.isoformat()
+    return data

lemur/common/schema.py

@@ -0,0 +1,174 @@
+"""
+.. module: lemur.common.schema
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+
+"""
+from functools import wraps
+from flask import request, current_app
+
+from sqlalchemy.orm.collections import InstrumentedList
+
+from inflection import camelize, underscore
+from marshmallow import Schema, post_dump, pre_load
+
+from lemur.extensions import sentry
+
+
+class LemurSchema(Schema):
+    """
+    Base schema from which all grouper schema's inherit
+    """
+    __envelope__ = True
+
+    def under(self, data, many=None):
+        items = []
+        if many:
+            for i in data:
+                items.append(
+                    {underscore(key): value for key, value in i.items()}
+                )
+            return items
+        return {
+            underscore(key): value
+            for key, value in data.items()
+        }
+
+    def camel(self, data, many=None):
+        items = []
+        if many:
+            for i in data:
+                items.append(
+                    {camelize(key, uppercase_first_letter=False): value for key, value in i.items()}
+                )
+            return items
+        return {
+            camelize(key, uppercase_first_letter=False): value
+            for key, value in data.items()
+        }
+
+    def wrap_with_envelope(self, data, many):
+        if many:
+            if 'total' in self.context.keys():
+                return dict(total=self.context['total'], items=data)
+        return data
+
+
+class LemurInputSchema(LemurSchema):
+    @pre_load(pass_many=True)
+    def preprocess(self, data, many):
+        return self.under(data, many=many)
+
+
+class LemurOutputSchema(LemurSchema):
+    @pre_load(pass_many=True)
+    def preprocess(self, data, many):
+        if many:
+            data = self.unwrap_envelope(data, many)
+        return self.under(data, many=many)
+
+    def unwrap_envelope(self, data, many):
+        if many:
+            if data['items']:
+                if isinstance(data, InstrumentedList) or isinstance(data, list):
+                    self.context['total'] = len(data)
+                    return data
+                else:
+                    self.context['total'] = data['total']
+            else:
+                self.context['total'] = 0
+                data = {'items': []}
+
+            return data['items']
+
+        return data
+
+    @post_dump(pass_many=True)
+    def post_process(self, data, many):
+        if data:
+            data = self.camel(data, many=many)
+        if self.__envelope__:
+            return self.wrap_with_envelope(data, many=many)
+        else:
+            return data
+
+
+def format_errors(messages):
+    errors = {}
+    for k, v in messages.items():
+        key = camelize(k, uppercase_first_letter=False)
+        if isinstance(v, dict):
+            errors[key] = format_errors(v)
+        elif isinstance(v, list):
+            errors[key] = v[0]
+    return errors
+
+
+def wrap_errors(messages):
+    errors = dict(message='Validation Error.')
+    if messages.get('_schema'):
+        errors['reasons'] = {'Schema': {'rule': messages['_schema']}}
+    else:
+        errors['reasons'] = format_errors(messages)
+    return errors
+
+
+def unwrap_pagination(data, output_schema):
+    if not output_schema:
+        return data
+
+    if isinstance(data, dict):
+        if 'total' in data.keys():
+            if data.get('total') == 0:
+                return data
+
+            marshaled_data = {'total': data['total']}
+            marshaled_data['items'] = output_schema.dump(data['items'], many=True).data
+            return marshaled_data
+
+        return output_schema.dump(data).data
+
+    elif isinstance(data, list):
+        marshaled_data = {'total': len(data)}
+        marshaled_data['items'] = output_schema.dump(data, many=True).data
+        return marshaled_data
+    return output_schema.dump(data).data
+
+
+def validate_schema(input_schema, output_schema):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            if input_schema:
+                if request.get_json():
+                    request_data = request.get_json()
+                else:
+                    request_data = request.args
+
+                data, errors = input_schema.load(request_data)
+
+                if errors:
+                    return wrap_errors(errors), 400
+
+                kwargs['data'] = data
+
+            try:
+                resp = f(*args, **kwargs)
+            except Exception as e:
+                sentry.captureException()
+                current_app.logger.exception(e)
+                return dict(message=str(e)), 500
+
+            if isinstance(resp, tuple):
+                return resp[0], resp[1]
+
+            if not resp:
+                return dict(message="No data found"), 404
+
+            return unwrap_pagination(resp, output_schema), 200
+
+        return decorated_function
+    return decorator

lemur/common/utils.py

@@ -1,20 +1,31 @@
 """
 .. module: lemur.common.utils
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import string
 import random
-from functools import wraps
+import string
 
-from flask import current_app
+import sqlalchemy
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa, ec
+from flask_restful.reqparse import RequestParser
+from sqlalchemy import and_, func
 
-from flask.ext.restful import marshal
+from lemur.constants import CERTIFICATE_KEY_TYPES
-from flask.ext.restful.reqparse import RequestParser
+from lemur.exceptions import InvalidConfiguration
-from flask.ext.sqlalchemy import Pagination
+
+paginated_parser = RequestParser()
+
+paginated_parser.add_argument('count', type=int, default=10, location='args')
+paginated_parser.add_argument('page', type=int, default=1, location='args')
+paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
+paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
+paginated_parser.add_argument('filter', type=str, location='args')
 
 
 def get_psuedo_random_string():
@@ -28,51 +39,175 @@ def get_psuedo_random_string():
     return challenge
 
 
-class marshal_items(object):
+def parse_certificate(body):
-    def __init__(self, fields, envelope=None):
+    """
-        self.fields = fields
+    Helper function that parses a PEM certificate.
-        self.envelop = envelope
 
-    def __call__(self, f):
+    :param body:
-        def _filter_items(items):
+    :return:
-            filtered_items = []
+    """
-            for item in items:
+    if isinstance(body, str):
-                filtered_items.append(marshal(item, self.fields))
+        body = body.encode('utf-8')
-            return filtered_items
 
-        @wraps(f)
+    return x509.load_pem_x509_certificate(body, default_backend())
-        def wrapper(*args, **kwargs):
-            try:
-                resp = f(*args, **kwargs)
-
-                # this is a bit weird way to handle non standard error codes returned from the marshaled function
-                if isinstance(resp, tuple):
-                    return resp[0], resp[1]
-
-                if isinstance(resp, Pagination):
-                    return {'items': _filter_items(resp.items), 'total': resp.total}
-
-                if isinstance(resp, list):
-                    return {'items': _filter_items(resp), 'total': len(resp)}
-
-                return marshal(resp, self.fields)
-            except Exception as e:
-                current_app.logger.exception(e)
-                # this is a little weird hack to respect flask restful parsing errors on marshaled functions
-                if hasattr(e, 'code'):
-                    if hasattr(e, 'data'):
-                        return {'message': e.data['message']}, 400
-                    else:
-                        return {'message': {'exception': 'unknown'}}, 400
-                else:
-                    return {'message': {'exception': str(e)}}, 400
-        return wrapper
 
 
-paginated_parser = RequestParser()
+def parse_csr(csr):
+    """
+    Helper function that parses a CSR.
 
-paginated_parser.add_argument('count', type=int, default=10, location='args')
+    :param csr:
-paginated_parser.add_argument('page', type=int, default=1, location='args')
+    :return:
-paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
+    """
-paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
+    if isinstance(csr, str):
-paginated_parser.add_argument('filter', type=str, location='args')
+        csr = csr.encode('utf-8')
+
+    return x509.load_pem_x509_csr(csr, default_backend())
+
+
+def get_authority_key(body):
+    """Returns the authority key for a given certificate in hex format"""
+    parsed_cert = parse_certificate(body)
+    authority_key = parsed_cert.extensions.get_extension_for_class(
+        x509.AuthorityKeyIdentifier).value.key_identifier
+    return authority_key.hex()
+
+
+def generate_private_key(key_type):
+    """
+    Generates a new private key based on key_type.
+
+    Valid key types: RSA2048, RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1',
+        'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
+        'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1',
+        'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2'
+
+    :param key_type:
+    :return:
+    """
+
+    _CURVE_TYPES = {
+        "ECCPRIME192V1": ec.SECP192R1(),
+        "ECCPRIME256V1": ec.SECP256R1(),
+
+        "ECCSECP192R1": ec.SECP192R1(),
+        "ECCSECP224R1": ec.SECP224R1(),
+        "ECCSECP256R1": ec.SECP256R1(),
+        "ECCSECP384R1": ec.SECP384R1(),
+        "ECCSECP521R1": ec.SECP521R1(),
+        "ECCSECP256K1": ec.SECP256K1(),
+
+        "ECCSECT163K1": ec.SECT163K1(),
+        "ECCSECT233K1": ec.SECT233K1(),
+        "ECCSECT283K1": ec.SECT283K1(),
+        "ECCSECT409K1": ec.SECT409K1(),
+        "ECCSECT571K1": ec.SECT571K1(),
+
+        "ECCSECT163R2": ec.SECT163R2(),
+        "ECCSECT233R1": ec.SECT233R1(),
+        "ECCSECT283R1": ec.SECT283R1(),
+        "ECCSECT409R1": ec.SECT409R1(),
+        "ECCSECT571R2": ec.SECT571R1(),
+    }
+
+    if key_type not in CERTIFICATE_KEY_TYPES:
+        raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format(
+            key_type=key_type,
+            choices=",".join(CERTIFICATE_KEY_TYPES)
+        ))
+
+    if 'RSA' in key_type:
+        key_size = int(key_type[3:])
+        return rsa.generate_private_key(
+            public_exponent=65537,
+            key_size=key_size,
+            backend=default_backend()
+        )
+    elif 'ECC' in key_type:
+        return ec.generate_private_key(
+            curve=_CURVE_TYPES[key_type],
+            backend=default_backend()
+        )
+
+
+def is_weekend(date):
+    """
+    Determines if a given date is on a weekend.
+
+    :param date:
+    :return:
+    """
+    if date.weekday() > 5:
+        return True
+
+
+def validate_conf(app, required_vars):
+    """
+    Ensures that the given fields are set in the applications conf.
+
+    :param app:
+    :param required_vars: list
+    """
+    for var in required_vars:
+        if not app.config.get(var):
+            raise InvalidConfiguration("Required variable '{var}' is not set in Lemur's conf.".format(var=var))
+
+
+# https://bitbucket.org/zzzeek/sqlalchemy/wiki/UsageRecipes/WindowedRangeQuery
+def column_windows(session, column, windowsize):
+    """Return a series of WHERE clauses against
+    a given column that break it into windows.
+
+    Result is an iterable of tuples, consisting of
+    ((start, end), whereclause), where (start, end) are the ids.
+
+    Requires a database that supports window functions,
+    i.e. Postgresql, SQL Server, Oracle.
+
+    Enhance this yourself !  Add a "where" argument
+    so that windows of just a subset of rows can
+    be computed.
+
+    """
+    def int_for_range(start_id, end_id):
+        if end_id:
+            return and_(
+                column >= start_id,
+                column < end_id
+            )
+        else:
+            return column >= start_id
+
+    q = session.query(
+        column,
+        func.row_number().over(order_by=column).label('rownum')
+    ).from_self(column)
+
+    if windowsize > 1:
+        q = q.filter(sqlalchemy.text("rownum %% %d=1" % windowsize))
+
+    intervals = [id for id, in q]
+
+    while intervals:
+        start = intervals.pop(0)
+        if intervals:
+            end = intervals[0]
+        else:
+            end = None
+        yield int_for_range(start, end)
+
+
+def windowed_query(q, column, windowsize):
+    """"Break a Query into windows on a given column."""
+
+    for whereclause in column_windows(
+            q.session,
+            column, windowsize):
+        for row in q.filter(whereclause).order_by(column):
+            yield row
+
+
+def truthiness(s):
+    """If input string resembles something truthy then return True, else False."""
+
+    return s.lower() in ('true', 'yes', 'on', 't', '1')

lemur/common/validators.py

@@ -0,0 +1,143 @@
+import re
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.x509 import NameOID
+from flask import current_app
+from marshmallow.exceptions import ValidationError
+
+from lemur.auth.permissions import SensitiveDomainPermission
+from lemur.common.utils import parse_certificate, is_weekend
+from lemur.domains import service as domain_service
+
+
+def public_certificate(body):
+    """
+    Determines if specified string is valid public certificate.
+
+    :param body:
+    :return:
+    """
+    try:
+        parse_certificate(body)
+    except Exception as e:
+        current_app.logger.exception(e)
+        raise ValidationError('Public certificate presented is not valid.')
+
+
+def private_key(key):
+    """
+    User to validate that a given string is a RSA private key
+
+    :param key:
+    :return: :raise ValueError:
+    """
+    try:
+        if isinstance(key, bytes):
+            serialization.load_pem_private_key(key, None, backend=default_backend())
+        else:
+            serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
+    except Exception:
+        raise ValidationError('Private key presented is not valid.')
+
+
+def common_name(value):
+    """If the common name could be a domain name, apply domain validation rules."""
+    # Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client
+    # certificates). As a simple heuristic, we assume that human-readable names always include a space.
+    # However, to avoid confusion for humans, we also don't count spaces at the beginning or end of the string.
+    if ' ' not in value.strip():
+        return sensitive_domain(value)
+
+
+def sensitive_domain(domain):
+    """
+    Checks if user has the admin role, the domain does not match sensitive domains and whitelisted domain patterns.
+    :param domain: domain name (str)
+    :return:
+    """
+    if SensitiveDomainPermission().can():
+        # User has permission, no need to check anything
+        return
+
+    whitelist = current_app.config.get('LEMUR_WHITELISTED_DOMAINS', [])
+    if whitelist and not any(re.match(pattern, domain) for pattern in whitelist):
+        raise ValidationError('Domain {0} does not match whitelisted domain patterns. '
+                              'Contact an administrator to issue the certificate.'.format(domain))
+
+    if any(d.sensitive for d in domain_service.get_by_name(domain)):
+        raise ValidationError('Domain {0} has been marked as sensitive. '
+                              'Contact an administrator to issue the certificate.'.format(domain))
+
+
+def encoding(oid_encoding):
+    """
+    Determines if the specified oid type is valid.
+    :param oid_encoding:
+    :return:
+    """
+    valid_types = ['b64asn1', 'string', 'ia5string']
+    if oid_encoding.lower() not in [o_type.lower() for o_type in valid_types]:
+        raise ValidationError('Invalid Oid Encoding: {0} choose from {1}'.format(oid_encoding, ",".join(valid_types)))
+
+
+def sub_alt_type(alt_type):
+    """
+    Determines if the specified subject alternate type is valid.
+    :param alt_type:
+    :return:
+    """
+    valid_types = ['DNSName', 'IPAddress', 'uniFormResourceIdentifier', 'directoryName', 'rfc822Name', 'registrationID',
+                   'otherName', 'x400Address', 'EDIPartyName']
+    if alt_type.lower() not in [a_type.lower() for a_type in valid_types]:
+        raise ValidationError('Invalid SubAltName Type: {0} choose from {1}'.format(type, ",".join(valid_types)))
+
+
+def csr(data):
+    """
+    Determines if the CSR is valid and allowed.
+    :param data:
+    :return:
+    """
+    try:
+        request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
+    except Exception:
+        raise ValidationError('CSR presented is not valid.')
+
+    # Validate common name and SubjectAltNames
+    for name in request.subject.get_attributes_for_oid(NameOID.COMMON_NAME):
+        common_name(name.value)
+
+    try:
+        alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
+
+        for name in alt_names.value.get_values_for_type(x509.DNSName):
+            sensitive_domain(name)
+    except x509.ExtensionNotFound:
+        pass
+
+
+def dates(data):
+    if not data.get('validity_start') and data.get('validity_end'):
+        raise ValidationError('If validity start is specified so must validity end.')
+
+    if not data.get('validity_end') and data.get('validity_start'):
+        raise ValidationError('If validity end is specified so must validity start.')
+
+    if data.get('validity_start') and data.get('validity_end'):
+        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+            if is_weekend(data.get('validity_end')):
+                raise ValidationError('Validity end must not land on a weekend.')
+
+        if not data['validity_start'] < data['validity_end']:
+            raise ValidationError('Validity start must be before validity end.')
+
+        if data.get('authority'):
+            if data.get('validity_start').date() < data['authority'].authority_certificate.not_before.date():
+                raise ValidationError('Validity start must not be before {0}'.format(data['authority'].authority_certificate.not_before))
+
+            if data.get('validity_end').date() > data['authority'].authority_certificate.not_after.date():
+                raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
+
+    return data

lemur/constants.py

@@ -1,8 +1,34 @@
 """
 .. module: lemur.constants
-    :copyright: (c) 2015 by Netflix Inc.
+    :copyright: (c) 2018 by Netflix Inc.
     :license: Apache, see LICENSE for more details.
 """
 SAN_NAMING_TEMPLATE = "SAN-{subject}-{issuer}-{not_before}-{not_after}"
 DEFAULT_NAMING_TEMPLATE = "{subject}-{issuer}-{not_before}-{not_after}"
 NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
+
+SUCCESS_METRIC_STATUS = 'success'
+FAILURE_METRIC_STATUS = 'failure'
+
+CERTIFICATE_KEY_TYPES = [
+    'RSA2048',
+    'RSA4096',
+    'ECCPRIME192V1',
+    'ECCPRIME256V1',
+    'ECCSECP192R1',
+    'ECCSECP224R1',
+    'ECCSECP256R1',
+    'ECCSECP384R1',
+    'ECCSECP521R1',
+    'ECCSECP256K1',
+    'ECCSECT163K1',
+    'ECCSECT233K1',
+    'ECCSECT283K1',
+    'ECCSECT409K1',
+    'ECCSECT571K1',
+    'ECCSECT163R2',
+    'ECCSECT233R1',
+    'ECCSECT283R1',
+    'ECCSECT409R1',
+    'ECCSECT571R2'
+]

lemur/database.py

@@ -4,14 +4,15 @@
     :synopsis: This module contains all of the database related methods
     needed for lemur to interact with a datastore
 
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from sqlalchemy import exc
+from inflection import underscore
+from sqlalchemy import exc, func
 from sqlalchemy.sql import and_, or_
-from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.orm import make_transient
 
 from lemur.extensions import db
 from lemur.exceptions import AttrNotFound, DuplicateError
@@ -75,6 +76,16 @@ def add(model):
     db.session.add(model)
 
 
+def get_model_column(model, field):
+    if field in getattr(model, 'sensitive_fields', ()):
+        raise AttrNotFound(field)
+    column = model.__table__.columns._data.get(field, None)
+    if column is None:
+        raise AttrNotFound(field)
+
+    return column
+
+
 def find_all(query, model, kwargs):
     """
     Returns a query object that ensures that all kwargs
@@ -91,7 +102,7 @@ def find_all(query, model, kwargs):
         if not isinstance(value, list):
             value = value.split(',')
 
-        conditions.append(getattr(model, attr).in_(value))
+        conditions.append(get_model_column(model, attr).in_(value))
 
     return query.filter(and_(*conditions))
 
@@ -108,7 +119,7 @@ def find_any(query, model, kwargs):
     """
     or_args = []
     for attr, value in kwargs.items():
-        or_args.append(or_(getattr(model, attr) == value))
+        or_args.append(or_(get_model_column(model, attr) == value))
     exprs = or_(*or_args)
     return query.filter(exprs)
 
@@ -123,10 +134,7 @@ def get(model, value, field="id"):
     :return:
     """
     query = session_query(model)
-    try:
+    return query.filter(get_model_column(model, field) == value).scalar()
-        return query.filter(getattr(model, field) == value).one()
-    except NoResultFound as e:
-        return
 
 
 def get_all(model, value, field="id"):
@@ -139,7 +147,7 @@ def get_all(model, value, field="id"):
     :return:
     """
     query = session_query(model)
-    return query.filter(getattr(model, field) == value)
+    return query.filter(get_model_column(model, field) == value)
 
 
 def create(model):
@@ -191,7 +199,8 @@ def filter(query, model, terms):
     :param terms:
     :return:
     """
-    return query.filter(getattr(model, terms[0]).ilike('%{}%'.format(terms[1])))
+    column = get_model_column(model, underscore(terms[0]))
+    return query.filter(column.ilike('%{}%'.format(terms[1])))
 
 
 def sort(query, model, field, direction):
@@ -204,13 +213,8 @@ def sort(query, model, field, direction):
     :param field:
     :param direction:
     """
-    try:
+    column = get_model_column(model, underscore(field))
-        field = getattr(model, field)
+    return query.order_by(column.desc() if direction == 'desc' else column.asc())
-        direction = getattr(field, direction)
-        query = query.order_by(direction())
-        return query
-    except AttributeError:
-        raise AttrNotFound(field)
 
 
 def paginate(query, page, count):
@@ -237,9 +241,6 @@ def update_list(model, model_attr, item_model, items):
     """
     ids = []
 
-    for i in items:
-        ids.append(i['id'])
-
     for i in getattr(model, model_attr):
         if i.id not in ids:
             getattr(model, model_attr).remove(i)
@@ -254,6 +255,29 @@ def update_list(model, model_attr, item_model, items):
     return model
 
 
+def clone(model):
+    """
+    Clones the given model and removes it's primary key
+    :param model:
+    :return:
+    """
+    db.session.expunge(model)
+    make_transient(model)
+    model.id = None
+    return model
+
+
+def get_count(q):
+    """
+    Count the number of rows in a table. More efficient than count(*)
+    :param q:
+    :return:
+    """
+    count_q = q.statement.with_only_columns([func.count()]).order_by(None)
+    count = q.session.execute(count_q).scalar()
+    return count
+
+
 def sort_and_page(query, model, args):
     """
     Helper that allows us to combine sorting and paging
@@ -268,9 +292,17 @@ def sort_and_page(query, model, args):
     page = args.pop('page')
     count = args.pop('count')
 
+    if args.get('user'):
+        user = args.pop('user')
+
     query = find_all(query, model, args)
 
     if sort_by and sort_dir:
         query = sort(query, model, sort_by, sort_dir)
 
-    return paginate(query, page, count)
+    total = get_count(query)
+
+    # offset calculated at zero
+    page -= 1
+    items = query.offset(count * page).limit(count).all()
+    return dict(items=items, total=total)

lemur/decorators.py

@@ -1,56 +0,0 @@
-"""
-.. module: lemur.decorators
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-"""
-from builtins import str
-
-from datetime import timedelta
-from flask import make_response, request, current_app
-
-from functools import update_wrapper
-
-
-# this is only used for dev
-def crossdomain(origin=None, methods=None, headers=None,
-                max_age=21600, attach_to_all=True,
-                automatic_options=True):  # pragma: no cover
-    if methods is not None:
-        methods = ', '.join(sorted(x.upper() for x in methods))
-
-    if headers is not None and not isinstance(headers, str):
-        headers = ', '.join(x.upper() for x in headers)
-
-    if not isinstance(origin, str):
-        origin = ', '.join(origin)
-
-    if isinstance(max_age, timedelta):
-        max_age = max_age.total_seconds()
-
-    def get_methods():
-        if methods is not None:
-            return methods
-
-        options_resp = current_app.make_default_options_response()
-        return options_resp.headers['allow']
-
-    def decorator(f):
-        def wrapped_function(*args, **kwargs):
-            if automatic_options and request.method == 'OPTIONS':
-                resp = current_app.make_default_options_response()
-            else:
-                resp = make_response(f(*args, **kwargs))
-            if not attach_to_all and request.method != 'OPTIONS':
-                return resp
-
-            h = resp.headers
-            h['Access-Control-Allow-Origin'] = origin
-            h['Access-Control-Allow-Methods'] = get_methods()
-            h['Access-Control-Max-Age'] = str(max_age)
-            h['Access-Control-Allow-Headers'] = "Origin, X-Requested-With, Content-Type, Accept, Authorization "
-            h['Access-Control-Allow-Credentials'] = 'true'
-            return resp
-
-        f.provide_automatic_options = False
-        return update_wrapper(wrapped_function, f)
-    return decorator

lemur/default.conf.py

@@ -3,15 +3,13 @@
 import os
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
-ADMINS = frozenset([''])
-
 THREADS_PER_PAGE = 8
 
 # General
 
 # These will need to be set to `True` if you are developing locally
 CORS = False
-debug = False
+DEBUG = False
 
 # Logging
 

lemur/defaults/schemas.py

@@ -0,0 +1,23 @@
+"""
+.. module: lemur.defaults.schemas
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+from lemur.common.schema import LemurOutputSchema
+from lemur.authorities.schemas import AuthorityNestedOutputSchema
+
+
+class DefaultOutputSchema(LemurOutputSchema):
+    authority = fields.Nested(AuthorityNestedOutputSchema)
+    country = fields.String()
+    state = fields.String()
+    location = fields.String()
+    organization = fields.String()
+    organizational_unit = fields.String()
+    issuer_plugin = fields.String()
+
+
+default_output_schema = DefaultOutputSchema()

lemur/defaults/views.py

@@ -1,13 +1,17 @@
 """
-.. module: lemur.status.views
+.. module: lemur.defaults.views
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 """
 from flask import current_app, Blueprint
-from flask.ext.restful import Api
+from flask_restful import Api
 
+from lemur.common.schema import validate_schema
+from lemur.authorities.service import get_by_name
 from lemur.auth.service import AuthenticatedResource
 
+from lemur.defaults.schemas import default_output_schema
+
 
 mod = Blueprint('default', __name__)
 api = Api(mod)
@@ -18,6 +22,7 @@ class LemurDefaults(AuthenticatedResource):
     def __init__(self):
         super(LemurDefaults)
 
+    @validate_schema(None, default_output_schema)
     def get(self):
         """
         .. http:get:: /defaults
@@ -45,19 +50,26 @@ class LemurDefaults(AuthenticatedResource):
                  "state": "CA",
                  "location": "Los Gatos",
                  "organization": "Netflix",
-                 "organizationalUnit": "Operations"
+                 "organizationalUnit": "Operations",
+                 "dnsProviders": [{"name": "test", ...}, {...}],
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
+
+        default_authority = get_by_name(current_app.config.get('LEMUR_DEFAULT_AUTHORITY'))
+
         return dict(
             country=current_app.config.get('LEMUR_DEFAULT_COUNTRY'),
             state=current_app.config.get('LEMUR_DEFAULT_STATE'),
             location=current_app.config.get('LEMUR_DEFAULT_LOCATION'),
             organization=current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'),
-            organizationalUnit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT')
+            organizational_unit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'),
+            issuer_plugin=current_app.config.get('LEMUR_DEFAULT_ISSUER_PLUGIN'),
+            authority=default_authority,
         )
 
+
 api.add_resource(LemurDefaults, '/defaults', endpoint='default')

lemur/deployment/service.py

@@ -0,0 +1,15 @@
+from lemur import database
+
+
+def rotate_certificate(endpoint, new_cert):
+    """
+    Rotates a certificate on a given endpoint.
+
+    :param endpoint:
+    :param new_cert:
+    :return:
+    """
+    # ensure that certificate is available for rotation
+    endpoint.source.plugin.update_endpoint(endpoint, new_cert)
+    endpoint.certificate = new_cert
+    database.update(endpoint)

lemur/destinations/models.py

@@ -1,11 +1,10 @@
 """
 .. module: lemur.destinations.models
     :platform: unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import copy
 from sqlalchemy import Column, Integer, String, Text
 from sqlalchemy_utils import JSONType
 from lemur.database import db
@@ -23,7 +22,7 @@ class Destination(db.Model):
 
     @property
     def plugin(self):
-        p = plugins.get(self.plugin_name)
+        return plugins.get(self.plugin_name)
-        c = copy.deepcopy(p)
+
-        c.options = self.options
+    def __repr__(self):
-        return c
+        return "Destination(label={label})".format(label=self.label)

lemur/destinations/schemas.py

@@ -0,0 +1,43 @@
+"""
+.. module: lemur.destinations.schemas
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+
+from marshmallow import fields, post_dump
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.schemas import PluginInputSchema, PluginOutputSchema
+
+
+class DestinationInputSchema(LemurInputSchema):
+    id = fields.Integer()
+    label = fields.String(required=True)
+    description = fields.String(required=True)
+    active = fields.Boolean()
+    plugin = fields.Nested(PluginInputSchema, required=True)
+
+
+class DestinationOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    label = fields.String()
+    description = fields.String()
+    active = fields.Boolean()
+    plugin = fields.Nested(PluginOutputSchema)
+    options = fields.List(fields.Dict())
+
+    @post_dump
+    def fill_object(self, data):
+        if data:
+            data['plugin']['pluginOptions'] = data['options']
+        return data
+
+
+class DestinationNestedOutputSchema(DestinationOutputSchema):
+    __envelope__ = False
+
+
+destination_input_schema = DestinationInputSchema()
+destinations_output_schema = DestinationOutputSchema(many=True)
+destination_output_schema = DestinationOutputSchema()

lemur/destinations/service.py

@@ -1,7 +1,7 @@
 """
 .. module: lemur.destinations.service
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
@@ -22,6 +22,11 @@ def create(label, plugin_name, options, description=None):
     :rtype : Destination
     :return: New destination
     """
+    # remove any sub-plugin objects before try to save the json options
+    for option in options:
+        if 'plugin' in option['type']:
+            del option['value']['plugin_object']
+
     destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
     return database.create(destination)
 
@@ -56,7 +61,7 @@ def delete(destination_id):
 
 def get(destination_id):
     """
-    Retrieves an destination by it's lemur assigned ID.
+    Retrieves an destination by its lemur assigned ID.
 
     :param destination_id: Lemur assigned ID
     :rtype : Destination
@@ -67,7 +72,7 @@ def get(destination_id):
 
 def get_by_label(label):
     """
-    Retrieves a destination by it's label
+    Retrieves a destination by its label
 
     :param label:
     :return:
@@ -86,10 +91,6 @@ def get_all():
 
 
 def render(args):
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
     certificate_id = args.pop('certificate_id', None)
 
@@ -103,12 +104,7 @@ def render(args):
         terms = filt.split(';')
         query = database.filter(query, Destination, terms)
 
-    query = database.find_all(query, Destination, args)
+    return database.sort_and_page(query, Destination, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Destination, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)
 
 
 def stats(**kwargs):

lemur/destinations/views.py

@@ -2,39 +2,33 @@
 .. module: lemur.destinations.views
     :platform: Unix
     :synopsis: This module contains all of the accounts view code.
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import Blueprint
-from flask.ext.restful import Api, reqparse, fields
+from flask_restful import Api, reqparse
 from lemur.destinations import service
 
 from lemur.auth.service import AuthenticatedResource
 from lemur.auth.permissions import admin_permission
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.common.utils import paginated_parser
+
+from lemur.common.schema import validate_schema
+from lemur.destinations.schemas import destinations_output_schema, destination_input_schema, destination_output_schema
 
 
 mod = Blueprint('destinations', __name__)
 api = Api(mod)
 
 
-FIELDS = {
-    'description': fields.String,
-    'destinationOptions': fields.Raw(attribute='options'),
-    'pluginName': fields.String(attribute='plugin_name'),
-    'label': fields.String,
-    'id': fields.Integer,
-}
-
-
 class DestinationsList(AuthenticatedResource):
     """ Defines the 'destinations' endpoint """
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(DestinationsList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, destinations_output_schema)
     def get(self):
         """
         .. http:get:: /destinations
@@ -58,32 +52,40 @@ class DestinationsList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
+                "items": [{
-                    {
+                    "description": "test",
-                        "destinationOptions": [
+                    "options": [{
-                            {
+                        "name": "accountNumber",
-                                "name": "accountNumber",
+                        "required": true,
-                                "required": true,
+                        "value": "111111111111111",
-                                "value": 111111111112,
+                        "helpMessage": "Must be a valid AWS account number!",
-                                "helpMessage": "Must be a valid AWS account number!",
+                        "validation": "/^[0-9]{12,12}$/",
-                                "validation": "/^[0-9]{12,12}$/",
+                        "type": "str"
-                                "type": "int"
+                    }],
-                            }
+                    "id": 4,
-                        ],
+                    "plugin": {
-                        "pluginName": "aws-destination",
+                        "pluginOptions": [{
-                        "id": 3,
+                            "name": "accountNumber",
-                        "description": "test",
+                            "required": true,
-                        "label": "test"
+                            "value": "111111111111111",
-                    }
+                            "helpMessage": "Must be a valid AWS account number!",
-                ],
+                            "validation": "/^[0-9]{12,12}$/",
+                            "type": "str"
+                        }],
+                        "description": "Allow the uploading of certificates to AWS IAM",
+                        "slug": "aws-destination",
+                        "title": "AWS"
+                    },
+                    "label": "test546"
+                }
                 "total": 1
               }
 
            :query sortBy: field to sort on
-           :query sortDir: acs or desc
+           :query sortDir: asc or desc
            :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
+           :query filter: key value pair format is k;v
-           :query limit: limit number. default is 10
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
@@ -92,8 +94,8 @@ class DestinationsList(AuthenticatedResource):
         return service.render(args)
 
     @admin_permission.require(http_exception=403)
-    @marshal_items(FIELDS)
+    @validate_schema(destination_input_schema, destination_output_schema)
-    def post(self):
+    def post(self, data=None):
         """
         .. http:post:: /destinations
 
@@ -108,20 +110,30 @@ class DestinationsList(AuthenticatedResource):
               Accept: application/json, text/javascript
 
               {
-                "destinationOptions": [
+                "description": "test33",
-                    {
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "34324324",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "34324324",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
+                        "type": "str"
-                    }
+                    }],
-                ],
+                    "description": "Allow the uploading of certificates to AWS IAM",
-                "pluginName": "aws-destination",
+                    "slug": "aws-destination",
-                "id": 3,
+                    "title": "AWS"
-                "description": "test",
+                },
-                "label": "test"
+                "label": "test546"
               }
 
            **Example response**:
@@ -133,20 +145,30 @@ class DestinationsList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "destinationOptions": [
+                "description": "test33",
-                    {
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "34324324",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "111111111111111",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
+                        "type": "str"
-                    }
+                    }],
-                ],
+                    "description": "Allow the uploading of certificates to AWS IAM",
-                "pluginName": "aws-destination",
+                    "slug": "aws-destination",
-                "id": 3,
+                    "title": "AWS"
-                "description": "test",
+                },
-                "label": "test"
+                "label": "test546"
               }
 
            :arg label: human readable account label
@@ -154,12 +176,7 @@ class DestinationsList(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
+        return service.create(data['label'], data['plugin']['slug'], data['plugin']['plugin_options'], data['description'])
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-
-        args = self.reqparse.parse_args()
-        return service.create(args['label'], args['plugin']['slug'], args['plugin']['pluginOptions'], args['description'])
 
 
 class Destinations(AuthenticatedResource):
@@ -167,7 +184,7 @@ class Destinations(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Destinations, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, destination_output_schema)
     def get(self, destination_id):
         """
         .. http:get:: /destinations/1
@@ -191,20 +208,30 @@ class Destinations(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "destinationOptions": [
+                "description": "test",
-                    {
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "111111111111111",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "111111111111111",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
+                        "type": "str"
-                    }
+                    }],
-                ],
+                    "description": "Allow the uploading of certificates to AWS IAM",
-                "pluginName": "aws-destination",
+                    "slug": "aws-destination",
-                "id": 3,
+                    "title": "AWS"
-                "description": "test",
+                },
-                "label": "test"
+                "label": "test546"
               }
 
            :reqheader Authorization: OAuth token to authenticate
@@ -213,8 +240,8 @@ class Destinations(AuthenticatedResource):
         return service.get(destination_id)
 
     @admin_permission.require(http_exception=403)
-    @marshal_items(FIELDS)
+    @validate_schema(destination_input_schema, destination_output_schema)
-    def put(self, destination_id):
+    def put(self, destination_id, data=None):
         """
         .. http:put:: /destinations/1
 
@@ -228,23 +255,35 @@ class Destinations(AuthenticatedResource):
               Host: example.com
               Accept: application/json, text/javascript
 
+
               {
-                "destinationOptions": [
+                "description": "test33",
-                    {
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "34324324",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "34324324",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
+                        "type": "str"
-                    }
+                    }],
-                ],
+                    "description": "Allow the uploading of certificates to AWS IAM",
-                "pluginName": "aws-destination",
+                    "slug": "aws-destination",
-                "id": 3,
+                    "title": "AWS"
-                "description": "test",
+                },
-                "label": "test"
+                "label": "test546"
               }
 
+
            **Example response**:
 
            .. sourcecode:: http
@@ -254,20 +293,30 @@ class Destinations(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "destinationOptions": [
+                "description": "test",
-                    {
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "111111111111111",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "111111111111111",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
+                        "type": "str"
-                    }
+                    }],
-                ],
+                    "description": "Allow the uploading of certificates to AWS IAM",
-                "pluginName": "aws-destination",
+                    "slug": "aws-destination",
-                "id": 3,
+                    "title": "AWS"
-                "description": "test",
+                },
-                "label": "test"
+                "label": "test546"
               }
 
            :arg accountNumber: aws account number
@@ -276,12 +325,7 @@ class Destinations(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
+        return service.update(destination_id, data['label'], data['plugin']['plugin_options'], data['description'])
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-
-        args = self.reqparse.parse_args()
-        return service.update(destination_id, args['label'], args['plugin']['pluginOptions'], args['description'])
 
     @admin_permission.require(http_exception=403)
     def delete(self, destination_id):
@@ -294,7 +338,7 @@ class CertificateDestinations(AuthenticatedResource):
     def __init__(self):
         super(CertificateDestinations, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, destination_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/destinations
@@ -318,32 +362,40 @@ class CertificateDestinations(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
+                "items": [{
-                    {
+                    "description": "test",
-                        "destinationOptions": [
+                    "options": [{
-                            {
+                        "name": "accountNumber",
-                                "name": "accountNumber",
+                        "required": true,
-                                "required": true,
+                        "value": "111111111111111",
-                                "value": 111111111112,
+                        "helpMessage": "Must be a valid AWS account number!",
-                                "helpMessage": "Must be a valid AWS account number!",
+                        "validation": "/^[0-9]{12,12}$/",
-                                "validation": "/^[0-9]{12,12}$/",
+                        "type": "str"
-                                "type": "int"
+                    }],
-                            }
+                    "id": 4,
-                        ],
+                    "plugin": {
-                        "pluginName": "aws-destination",
+                        "pluginOptions": [{
-                        "id": 3,
+                            "name": "accountNumber",
-                        "description": "test",
+                            "required": true,
-                        "label": "test"
+                            "value": "111111111111111",
-                    }
+                            "helpMessage": "Must be a valid AWS account number!",
-                ],
+                            "validation": "/^[0-9]{12,12}$/",
+                            "type": "str"
+                        }],
+                        "description": "Allow the uploading of certificates to AWS IAM",
+                        "slug": "aws-destination",
+                        "title": "AWS"
+                    },
+                    "label": "test546"
+                }
                 "total": 1
               }
 
            :query sortBy: field to sort on
-           :query sortDir: acs or desc
+           :query sortDir: asc or desc
-           :query page: int. default is 1
+           :query page: int default is 1
-           :query filter: key value pair. format is k=v;
+           :query filter: key value pair format is k;v
-           :query limit: limit number. default is 10
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """

lemur/dns_providers/models.py

@@ -0,0 +1,37 @@
+from sqlalchemy import Column, Integer, String, text, Text
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy_utils import ArrowType
+
+from lemur.database import db
+from lemur.plugins.base import plugins
+from lemur.utils import Vault
+
+
+class DnsProvider(db.Model):
+    __tablename__ = 'dns_providers'
+    id = Column(
+        Integer(),
+        primary_key=True,
+    )
+    name = Column(String(length=256), unique=True, nullable=True)
+    description = Column(Text(), nullable=True)
+    provider_type = Column(String(length=256), nullable=True)
+    credentials = Column(Vault, nullable=True)
+    api_endpoint = Column(String(length=256), nullable=True)
+    date_created = Column(ArrowType(), server_default=text('now()'), nullable=False)
+    status = Column(String(length=128), nullable=True)
+    options = Column(JSON, nullable=True)
+    domains = Column(JSON, nullable=True)
+
+    def __init__(self, name, description, provider_type, credentials):
+        self.name = name
+        self.description = description
+        self.provider_type = provider_type
+        self.credentials = credentials
+
+    @property
+    def plugin(self):
+        return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "DnsProvider(name={name})".format(name=self.name)

lemur/dns_providers/schemas.py

@@ -0,0 +1,27 @@
+from lemur.common.fields import ArrowDateTime
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+
+from marshmallow import fields
+
+
+class DnsProvidersNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    name = fields.String()
+    providerType = fields.String()
+    description = fields.String()
+    credentials = fields.String()
+    api_endpoint = fields.String()
+    date_created = ArrowDateTime()
+
+
+class DnsProvidersNestedInputSchema(LemurInputSchema):
+    __envelope__ = False
+    name = fields.String()
+    description = fields.String()
+    provider_type = fields.Dict()
+
+
+dns_provider_output_schema = DnsProvidersNestedOutputSchema()
+
+dns_provider_input_schema = DnsProvidersNestedInputSchema()

lemur/dns_providers/service.py

@@ -0,0 +1,111 @@
+import json
+
+from flask import current_app
+from lemur import database
+from lemur.dns_providers.models import DnsProvider
+
+
+def render(args):
+    """
+    Helper that helps us render the REST Api responses.
+    :param args:
+    :return:
+    """
+    query = database.session_query(DnsProvider)
+
+    return database.sort_and_page(query, DnsProvider, args)
+
+
+def get(dns_provider_id):
+    provider = database.get(DnsProvider, dns_provider_id)
+    return provider
+
+
+def get_friendly(dns_provider_id):
+    """
+    Retrieves a dns provider by its lemur assigned ID.
+
+    :param dns_provider_id: Lemur assigned ID
+    :rtype : DnsProvider
+    :return:
+    """
+    dns_provider = get(dns_provider_id)
+    dns_provider_friendly = {
+        "name": dns_provider.name,
+        "description": dns_provider.description,
+        "providerType": dns_provider.provider_type,
+        "options": dns_provider.options,
+        "credentials": dns_provider.credentials,
+    }
+
+    if dns_provider.provider_type == "route53":
+        dns_provider_friendly["account_id"] = json.loads(dns_provider.credentials).get("account_id")
+    return dns_provider_friendly
+
+
+def delete(dns_provider_id):
+    """
+    Deletes a DNS provider.
+
+    :param dns_provider_id: Lemur assigned ID
+    """
+    database.delete(get(dns_provider_id))
+
+
+def get_types():
+    provider_config = current_app.config.get(
+        'ACME_DNS_PROVIDER_TYPES',
+        {"items": [
+            {
+                'name': 'route53',
+                'requirements': [
+                    {
+                        'name': 'account_id',
+                        'type': 'int',
+                        'required': True,
+                        'helpMessage': 'AWS Account number'
+                    },
+                ]
+            },
+            {
+                'name': 'cloudflare',
+                'requirements': [
+                    {
+                        'name': 'email',
+                        'type': 'str',
+                        'required': True,
+                        'helpMessage': 'Cloudflare Email'
+                    },
+                    {
+                        'name': 'key',
+                        'type': 'str',
+                        'required': True,
+                        'helpMessage': 'Cloudflare Key'
+                    },
+                ]
+            },
+            {
+                'name': 'dyn',
+            },
+        ]}
+    )
+    if not provider_config:
+        raise Exception("No DNS Provider configuration specified.")
+    provider_config["total"] = len(provider_config.get("items"))
+    return provider_config
+
+
+def create(data):
+    provider_name = data.get("name")
+
+    credentials = {}
+    for item in data.get("provider_type", {}).get("requirements", []):
+        credentials[item["name"]] = item["value"]
+    dns_provider = DnsProvider(
+        name=provider_name,
+        description=data.get("description"),
+        provider_type=data.get("provider_type").get("name"),
+        credentials=json.dumps(credentials),
+    )
+    created = database.create(dns_provider)
+    return created.id

lemur/dns_providers/views.py

@@ -0,0 +1,170 @@
+"""
+.. module: lemur.dns)providers.views
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
+"""
+from flask import Blueprint, g
+from flask_restful import reqparse, Api
+
+from lemur.auth.permissions import admin_permission
+from lemur.auth.service import AuthenticatedResource
+from lemur.common.schema import validate_schema
+from lemur.common.utils import paginated_parser
+from lemur.dns_providers import service
+from lemur.dns_providers.schemas import dns_provider_output_schema, dns_provider_input_schema
+
+mod = Blueprint('dns_providers', __name__)
+api = Api(mod)
+
+
+class DnsProvidersList(AuthenticatedResource):
+    """ Defines the 'dns_providers' endpoint """
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(DnsProvidersList, self).__init__()
+
+    @validate_schema(None, dns_provider_output_schema)
+    def get(self):
+        """
+        .. http:get:: /dns_providers
+
+           The current list of DNS Providers
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /dns_providers HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [{
+                    "id": 1,
+                    "name": "test",
+                    "description": "test",
+                    "provider_type": "dyn",
+                    "status": "active",
+                }],
+                "total": 1
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int. default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number. default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        parser = paginated_parser.copy()
+        parser.add_argument('dns_provider_id', type=int, location='args')
+        parser.add_argument('name', type=str, location='args')
+        parser.add_argument('type', type=str, location='args')
+
+        args = parser.parse_args()
+        args['user'] = g.user
+        return service.render(args)
+
+    @validate_schema(dns_provider_input_schema, None)
+    @admin_permission.require(http_exception=403)
+    def post(self, data=None):
+        """
+        Creates a DNS Provider
+
+        **Example request**:
+        {
+          "providerType": {
+            "name": "route53",
+            "requirements": [
+              {
+                "name": "account_id",
+                "type": "int",
+                "required": true,
+                "helpMessage": "AWS Account number",
+                "value": 12345
+              }
+            ],
+            "route": "dns_provider_options",
+            "reqParams": null,
+            "restangularized": true,
+            "fromServer": true,
+            "parentResource": null,
+            "restangularCollection": false
+          },
+          "name": "provider_name",
+          "description": "provider_description"
+        }
+
+        **Example request 2**
+        {
+          "providerType": {
+            "name": "cloudflare",
+            "requirements": [
+              {
+                "name": "email",
+                "type": "str",
+                "required": true,
+                "helpMessage": "Cloudflare Email",
+                "value": "test@example.com"
+              },
+              {
+                "name": "key",
+                "type": "str",
+                "required": true,
+                "helpMessage": "Cloudflare Key",
+                "value": "secretkey"
+              }
+            ],
+            "route": "dns_provider_options",
+            "reqParams": null,
+            "restangularized": true,
+            "fromServer": true,
+            "parentResource": null,
+            "restangularCollection": false
+          },
+          "name": "provider_name",
+          "description": "provider_description"
+        }
+        :return:
+        """
+        return service.create(data)
+
+
+class DnsProviders(AuthenticatedResource):
+    @validate_schema(None, dns_provider_output_schema)
+    def get(self, dns_provider_id):
+        return service.get_friendly(dns_provider_id)
+
+    @admin_permission.require(http_exception=403)
+    def delete(self, dns_provider_id):
+        service.delete(dns_provider_id)
+        return {'result': True}
+
+
+class DnsProviderOptions(AuthenticatedResource):
+    """ Defines the 'dns_provider_types' endpoint """
+
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(DnsProviderOptions, self).__init__()
+
+    def get(self):
+        return service.get_types()
+
+
+api.add_resource(DnsProvidersList, '/dns_providers', endpoint='dns_providers')
+api.add_resource(DnsProviders, '/dns_providers/<int:dns_provider_id>', endpoint='dns_provider')
+api.add_resource(DnsProviderOptions, '/dns_provider_options', endpoint='dns_provider_options')

lemur/domains/models.py

@@ -1,13 +1,13 @@
 """
 .. module: lemur.domains.models
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, Integer, String, Boolean
 
 from lemur.database import db
 
@@ -16,11 +16,7 @@ class Domain(db.Model):
     __tablename__ = 'domains'
     id = Column(Integer, primary_key=True)
     name = Column(String(256))
+    sensitive = Column(Boolean, default=False)
 
-    def as_dict(self):
+    def __repr__(self):
-        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+        return "Domain(name={name})".format(name=self.name)
-
-    def serialize(self):
-        blob = self.as_dict()
-        blob['certificates'] = [x.id for x in self.certificate]
-        return blob

lemur/domains/schemas.py

@@ -0,0 +1,35 @@
+"""
+.. module: lemur.domains.schemas
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.schemas import AssociatedCertificateSchema
+
+# from lemur.certificates.schemas import CertificateNestedOutputSchema
+
+
+class DomainInputSchema(LemurInputSchema):
+    id = fields.Integer()
+    name = fields.String(required=True)
+    sensitive = fields.Boolean(missing=False)
+    certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[])
+
+
+class DomainOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    name = fields.String()
+    sensitive = fields.Boolean()
+    # certificates = fields.Nested(CertificateNestedOutputSchema, many=True, missing=[])
+
+
+class DomainNestedOutputSchema(DomainOutputSchema):
+    __envelope__ = False
+
+
+domain_input_schema = DomainInputSchema()
+domain_output_schema = DomainOutputSchema()
+domains_output_schema = DomainOutputSchema(many=True)

lemur/domains/service.py

@@ -1,7 +1,7 @@
 """
 .. module: lemur.domains.service
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
@@ -32,6 +32,43 @@ def get_all():
     return database.find_all(query, Domain, {}).all()
 
 
+def get_by_name(name):
+    """
+    Fetches domain by its name
+
+    :param name:
+    :return:
+    """
+    return database.get_all(Domain, name, field="name").all()
+
+
+def create(name, sensitive):
+    """
+    Create a new domain
+
+    :param name:
+    :param sensitive:
+    :return:
+    """
+    domain = Domain(name=name, sensitive=sensitive)
+    return database.create(domain)
+
+
+def update(domain_id, name, sensitive):
+    """
+    Update an existing domain
+
+    :param domain_id:
+    :param name:
+    :param sensitive:
+    :return:
+    """
+    domain = get(domain_id)
+    domain.name = name
+    domain.sensitive = sensitive
+    database.update(domain)
+
+
 def render(args):
     """
     Helper to parse REST Api requests
@@ -39,12 +76,7 @@ def render(args):
     :param args:
     :return:
     """
-    query = database.session_query(Domain).join(Certificate, Domain.certificate)
+    query = database.session_query(Domain)
-
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
     certificate_id = args.pop('certificate_id', None)
 
@@ -53,11 +85,7 @@ def render(args):
         query = database.filter(query, Domain, terms)
 
     if certificate_id:
+        query = query.join(Certificate, Domain.certificates)
         query = query.filter(Certificate.id == certificate_id)
 
-    query = database.find_all(query, Domain, args)
+    return database.sort_and_page(query, Domain, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Domain, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)

lemur/domains/views.py

@@ -1,24 +1,23 @@
 """
 .. module: lemur.domains.views
     :platform: Unix
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
 from flask import Blueprint
-from flask.ext.restful import reqparse, Api, fields
+from flask_restful import reqparse, Api
 
 from lemur.domains import service
 from lemur.auth.service import AuthenticatedResource
+from lemur.auth.permissions import SensitiveDomainPermission
 
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.common.schema import validate_schema
+from lemur.common.utils import paginated_parser
 
-FIELDS = {
+from lemur.domains.schemas import domain_input_schema, domain_output_schema, domains_output_schema
-    'id': fields.Integer,
-    'name': fields.String
-}
 
 mod = Blueprint('domains', __name__)
 api = Api(mod)
@@ -29,7 +28,7 @@ class DomainsList(AuthenticatedResource):
     def __init__(self):
         super(DomainsList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, domains_output_schema)
     def get(self):
         """
         .. http:get:: /domains
@@ -57,20 +56,22 @@ class DomainsList(AuthenticatedResource):
                     {
                       "id": 1,
                       "name": "www.example.com",
+                      "sensitive": false
                     },
                     {
                       "id": 2,
                       "name": "www.example2.com",
+                      "sensitive": false
                     }
                   ]
                 "total": 2
               }
 
            :query sortBy: field to sort on
-           :query sortDir: acs or desc
+           :query sortDir: asc or desc
-           :query page: int. default is 1
+           :query page: int default is 1
-           :query filter: key value pair. format is k=v;
+           :query filter: key value pair format is k;v
-           :query limit: limit number. default is 10
+           :query count: count number. default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
@@ -79,13 +80,58 @@ class DomainsList(AuthenticatedResource):
         args = parser.parse_args()
         return service.render(args)
 
+    @validate_schema(domain_input_schema, domain_output_schema)
+    def post(self, data=None):
+        """
+        .. http:post:: /domains
+
+           The current domain list
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /domains HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                "name": "www.example.com",
+                "sensitive": false
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "id": 1,
+                "name": "www.example.com",
+                "sensitive": false
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        return service.create(data['name'], data['sensitive'])
+
 
 class Domains(AuthenticatedResource):
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Domains, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, domain_output_schema)
     def get(self, domain_id):
         """
         .. http:get:: /domains/1
@@ -111,6 +157,7 @@ class Domains(AuthenticatedResource):
               {
                   "id": 1,
                   "name": "www.example.com",
+                  "sensitive": false
               }
 
            :reqheader Authorization: OAuth token to authenticate
@@ -119,13 +166,56 @@ class Domains(AuthenticatedResource):
         """
         return service.get(domain_id)
 
+    @validate_schema(domain_input_schema, domain_output_schema)
+    def put(self, domain_id, data=None):
+        """
+        .. http:get:: /domains/1
+
+           update one domain
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /domains HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                  "name": "www.example.com",
+                  "sensitive": false
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "id": 1,
+                  "name": "www.example.com",
+                  "sensitive": false
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if SensitiveDomainPermission().can():
+            return service.update(domain_id, data['name'], data['sensitive'])
+
+        return dict(message='You are not authorized to modify this domain'), 403
+
 
 class CertificateDomains(AuthenticatedResource):
     """ Defines the 'domains' endpoint """
     def __init__(self):
         super(CertificateDomains, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, domains_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/domains
@@ -153,20 +243,22 @@ class CertificateDomains(AuthenticatedResource):
                     {
                       "id": 1,
                       "name": "www.example.com",
+                      "sensitive": false
                     },
                     {
                       "id": 2,
                       "name": "www.example2.com",
+                      "sensitive": false
                     }
                   ]
                 "total": 2
               }
 
            :query sortBy: field to sort on
-           :query sortDir: acs or desc
+           :query sortDir: asc or desc
-           :query page: int. default is 1
+           :query page: int default is 1
-           :query filter: key value pair. format is k=v;
+           :query filter: key value pair format is k;v
-           :query limit: limit number. default is 10
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated

lemur/endpoints/cli.py

@@ -0,0 +1,43 @@
+"""
+.. module: lemur.endpoints.cli
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask_script import Manager
+
+import arrow
+from datetime import timedelta
+
+from sqlalchemy import cast
+from sqlalchemy_utils import ArrowType
+
+from lemur import database
+from lemur.extensions import metrics, sentry
+from lemur.endpoints.models import Endpoint
+
+
+manager = Manager(usage="Handles all endpoint related tasks.")
+
+
+@manager.option('-ttl', '--time-to-live', type=int, dest='ttl', default=2, help='Time in hours, which endpoint has not been refreshed to remove the endpoint.')
+def expire(ttl):
+    """
+    Removed all endpoints that have not been recently updated.
+    """
+    print("[+] Staring expiration of old endpoints.")
+
+    try:
+        now = arrow.utcnow()
+        expiration = now - timedelta(hours=ttl)
+        endpoints = database.session_query(Endpoint).filter(cast(Endpoint.last_updated, ArrowType) <= expiration)
+
+        for endpoint in endpoints:
+            print("[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(name=endpoint.name, last_updated=endpoint.last_updated))
+            database.delete(endpoint)
+            metrics.send('endpoint_expired', 'counter', 1)
+
+        print("[+] Finished expiration.")
+    except Exception as e:
+        sentry.captureException()

lemur/endpoints/models.py

@@ -0,0 +1,93 @@
+"""
+.. module: lemur.endpoints.models
+    :platform: unix
+    :synopsis: This module contains all of the models need to create an authority within Lemur.
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+import arrow
+from sqlalchemy.orm import relationship
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import Column, Integer, String, Boolean, ForeignKey
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy.sql.expression import case
+
+from sqlalchemy_utils import ArrowType
+
+from lemur.database import db
+
+from lemur.models import policies_ciphers
+
+
+BAD_CIPHERS = [
+    'Protocol-SSLv3',
+    'Protocol-SSLv2',
+    'Protocol-TLSv1'
+]
+
+
+class Cipher(db.Model):
+    __tablename__ = 'ciphers'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(128), nullable=False)
+
+    @hybrid_property
+    def deprecated(self):
+        return self.name in BAD_CIPHERS
+
+    @deprecated.expression
+    def deprecated(cls):
+        return case(
+            [
+                (cls.name in BAD_CIPHERS, True)
+            ],
+            else_=False
+        )
+
+
+class Policy(db.Model):
+    ___tablename__ = 'policies'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(128), nullable=True)
+    ciphers = relationship('Cipher', secondary=policies_ciphers, backref='policy')
+
+
+class Endpoint(db.Model):
+    __tablename__ = 'endpoints'
+    id = Column(Integer, primary_key=True)
+    owner = Column(String(128))
+    name = Column(String(128))
+    dnsname = Column(String(256))
+    type = Column(String(128))
+    active = Column(Boolean, default=True)
+    port = Column(Integer)
+    policy_id = Column(Integer, ForeignKey('policy.id'))
+    policy = relationship('Policy', backref='endpoint')
+    certificate_id = Column(Integer, ForeignKey('certificates.id'))
+    source_id = Column(Integer, ForeignKey('sources.id'))
+    sensitive = Column(Boolean, default=False)
+    source = relationship('Source', back_populates='endpoints')
+    last_updated = Column(ArrowType, default=arrow.utcnow, nullable=False)
+    date_created = Column(ArrowType, default=arrow.utcnow, onupdate=arrow.utcnow, nullable=False)
+
+    replaced = association_proxy('certificate', 'replaced')
+
+    @property
+    def issues(self):
+        issues = []
+
+        for cipher in self.policy.ciphers:
+            if cipher.deprecated:
+                issues.append({'name': 'deprecated cipher', 'value': '{0} has been deprecated consider removing it.'.format(cipher.name)})
+
+        if self.certificate.expired:
+            issues.append({'name': 'expired certificate', 'value': 'There is an expired certificate attached to this endpoint consider replacing it.'})
+
+        if self.certificate.revoked:
+            issues.append({'name': 'revoked', 'value': 'There is a revoked certificate attached to this endpoint consider replacing it.'})
+
+        return issues
+
+    def __repr__(self):
+        return "Endpoint(name={name})".format(name=self.name)

lemur/endpoints/schemas.py

@@ -0,0 +1,44 @@
+"""
+.. module: lemur.endpoints.schemas
+    :platform: unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+
+from lemur.common.schema import LemurOutputSchema
+from lemur.certificates.schemas import CertificateNestedOutputSchema
+
+
+class CipherNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    deprecated = fields.Boolean()
+    name = fields.String()
+
+
+class PolicyNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    name = fields.String()
+    ciphers = fields.Nested(CipherNestedOutputSchema, many=True)
+
+
+class EndpointOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    dnsname = fields.String()
+    owner = fields.Email()
+    type = fields.String()
+    port = fields.Integer()
+    active = fields.Boolean()
+    certificate = fields.Nested(CertificateNestedOutputSchema)
+    policy = fields.Nested(PolicyNestedOutputSchema)
+
+    issues = fields.List(fields.Dict())
+
+
+endpoint_output_schema = EndpointOutputSchema()
+endpoints_output_schema = EndpointOutputSchema(many=True)

lemur/endpoints/service.py

@@ -0,0 +1,178 @@
+"""
+.. module: lemur.endpoints.service
+    :platform: Unix
+    :synopsis: This module contains all of the services level functions used to
+    administer endpoints in Lemur
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+
+"""
+import arrow
+
+from sqlalchemy import func
+
+from lemur import database
+from lemur.common.utils import truthiness
+from lemur.endpoints.models import Endpoint, Policy, Cipher
+from lemur.extensions import metrics
+
+
+def get_all():
+    """
+    Get all endpoints that are currently in Lemur.
+
+    :rtype : List
+    :return:
+    """
+    query = database.session_query(Endpoint)
+    return database.find_all(query, Endpoint, {}).all()
+
+
+def get(endpoint_id):
+    """
+    Retrieves an endpoint given it's ID
+
+    :param endpoint_id:
+    :return:
+    """
+    return database.get(Endpoint, endpoint_id)
+
+
+def get_by_name(name):
+    """
+    Retrieves an endpoint given it's name.
+
+    :param name:
+    :return:
+    """
+    return database.get(Endpoint, name, field='name')
+
+
+def get_by_dnsname(dnsname):
+    """
+    Retrieves an endpoint given it's name.
+
+    :param dnsname:
+    :return:
+    """
+    return database.get(Endpoint, dnsname, field='dnsname')
+
+
+def get_by_dnsname_and_port(dnsname, port):
+    """
+    Retrieves and endpoint by it's dnsname and port.
+    :param dnsname:
+    :param port:
+    :return:
+    """
+    return Endpoint.query.filter(Endpoint.dnsname == dnsname).filter(Endpoint.port == port).scalar()
+
+
+def get_by_source(source_label):
+    """
+    Retrieves all endpoints for a given source.
+    :param source_label:
+    :return:
+    """
+    return Endpoint.query.filter(Endpoint.source.label == source_label).all()  # noqa
+
+
+def get_all_pending_rotation():
+    """
+    Retrieves all endpoints which have certificates deployed
+    that have been replaced.
+    :return:
+    """
+    return Endpoint.query.filter(Endpoint.replaced.any()).all()
+
+
+def create(**kwargs):
+    """
+    Creates a new endpoint.
+    :param kwargs:
+    :return:
+    """
+    endpoint = Endpoint(**kwargs)
+    database.create(endpoint)
+    metrics.send('endpoint_added', 'counter', 1, metric_tags={'source': endpoint.source.label})
+    return endpoint
+
+
+def get_or_create_policy(**kwargs):
+    policy = database.get(Policy, kwargs['name'], field='name')
+
+    if not policy:
+        policy = Policy(**kwargs)
+        database.create(policy)
+
+    return policy
+
+
+def get_or_create_cipher(**kwargs):
+    cipher = database.get(Cipher, kwargs['name'], field='name')
+
+    if not cipher:
+        cipher = Cipher(**kwargs)
+        database.create(cipher)
+
+    return cipher
+
+
+def update(endpoint_id, **kwargs):
+    endpoint = database.get(Endpoint, endpoint_id)
+
+    endpoint.policy = kwargs['policy']
+    endpoint.certificate = kwargs['certificate']
+    endpoint.source = kwargs['source']
+    endpoint.last_updated = arrow.utcnow()
+    metrics.send('endpoint_updated', 'counter', 1, metric_tags={'source': endpoint.source.label})
+    database.update(endpoint)
+    return endpoint
+
+
+def render(args):
+    """
+    Helper that helps us render the REST Api responses.
+    :param args:
+    :return:
+    """
+    query = database.session_query(Endpoint)
+    filt = args.pop('filter')
+
+    if filt:
+        terms = filt.split(';')
+        if 'active' in filt:  # this is really weird but strcmp seems to not work here??
+            query = query.filter(Endpoint.active == truthiness(terms[1]))
+        elif 'port' in filt:
+            if terms[1] != 'null':  # ng-table adds 'null' if a number is removed
+                query = query.filter(Endpoint.port == terms[1])
+        elif 'ciphers' in filt:
+            query = query.filter(
+                Cipher.name == terms[1]
+            )
+        else:
+            query = database.filter(query, Endpoint, terms)
+
+    return database.sort_and_page(query, Endpoint, args)
+
+
+def stats(**kwargs):
+    """
+    Helper that defines some useful statistics about endpoints.
+
+    :param kwargs:
+    :return:
+    """
+    attr = getattr(Endpoint, kwargs.get('metric'))
+    query = database.db.session.query(attr, func.count(attr))
+
+    items = query.group_by(attr).all()
+
+    keys = []
+    values = []
+    for key, count in items:
+        keys.append(key)
+        values.append(count)
+
+    return {'labels': keys, 'values': values}

lemur/endpoints/views.py

@@ -0,0 +1,107 @@
+"""
+.. module: lemur.endpoints.views
+    :platform: Unix
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import Blueprint, g
+from flask_restful import reqparse, Api
+
+from lemur.common.utils import paginated_parser
+from lemur.common.schema import validate_schema
+from lemur.auth.service import AuthenticatedResource
+
+from lemur.endpoints import service
+from lemur.endpoints.schemas import endpoint_output_schema, endpoints_output_schema
+
+
+mod = Blueprint('endpoints', __name__)
+api = Api(mod)
+
+
+class EndpointsList(AuthenticatedResource):
+    """ Defines the 'endpoints' endpoint """
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(EndpointsList, self).__init__()
+
+    @validate_schema(None, endpoints_output_schema)
+    def get(self):
+        """
+        .. http:get:: /endpoints
+
+           The current list of endpoints
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /endpoints HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+
+           :query sortBy: field to sort on
+           :query sortDir: asc or desc
+           :query page: int default is 1
+           :query filter: key value pair. format is k;v
+           :query limit: limit number default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+           :note: this will only show certificates that the current user is authorized to use
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        args['user'] = g.current_user
+        return service.render(args)
+
+
+class Endpoints(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(Endpoints, self).__init__()
+
+    @validate_schema(None, endpoint_output_schema)
+    def get(self, endpoint_id):
+        """
+        .. http:get:: /endpoints/1
+
+           One endpoint
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /endpoints/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        return service.get(endpoint_id)
+
+
+api.add_resource(EndpointsList, '/endpoints', endpoint='endpoints')
+api.add_resource(Endpoints, '/endpoints/<int:endpoint_id>', endpoint='endpoint')

lemur/exceptions.py

@@ -1,14 +1,14 @@
 """
 .. module: lemur.exceptions
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 """
 from flask import current_app
 
 
 class LemurException(Exception):
-    def __init__(self):
+    def __init__(self, *args, **kwargs):
-        current_app.logger.error(self)
+        current_app.logger.exception(self)
 
 
 class DuplicateError(LemurException):
@@ -19,51 +19,26 @@ class DuplicateError(LemurException):
         return repr("Duplicate found! Could not create: {0}".format(self.key))
 
 
-class AuthenticationFailedException(LemurException):
-    def __init__(self, remote_ip, user_agent):
-        self.remote_ip = remote_ip
-        self.user_agent = user_agent
-
-    def __str__(self):
-        return repr("Failed login from: {} {}".format(self.remote_ip, self.user_agent))
-
-
-class IntegrityError(LemurException):
-    def __init__(self, message):
-        self.message = message
-
-    def __str__(self):
-        return repr(self.message)
-
-
 class InvalidListener(LemurException):
     def __str__(self):
         return repr("Invalid listener, ensure you select a certificate if you are using a secure protocol")
 
 
-class CertificateUnavailable(LemurException):
-    def __str__(self):
-        return repr("The certificate requested is not available")
-
-
 class AttrNotFound(LemurException):
     def __init__(self, field):
         self.field = field
 
     def __str__(self):
-        return repr("The field '{0}' is not sortable".format(self.field))
+        return repr("The field '{0}' is not sortable or filterable".format(self.field))
 
 
-class NoPersistanceFound(Exception):
+class InvalidConfiguration(Exception):
-    def __str__(self):
+    pass
-        return repr("No peristence method found, Lemur cannot persist sensitive information")
 
 
-class NoEncryptionKeyFound(Exception):
+class InvalidAuthority(Exception):
-    def __str__(self):
+    pass
-        return repr("Aborting... Lemur cannot locate db encryption key, is ENCRYPTION_KEY set?")
 
 
-class InvalidToken(Exception):
+class UnknownProvider(Exception):
-    def __str__(self):
+    pass
-        return repr("Invalid token")

lemur/extensions.py

@@ -1,19 +1,31 @@
 """
 .. module: lemur.extensions
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 """
-from flask.ext.sqlalchemy import SQLAlchemy
+from flask_sqlalchemy import SQLAlchemy
 db = SQLAlchemy()
 
-from flask.ext.migrate import Migrate
+from flask_migrate import Migrate
 migrate = Migrate()
 
-from flask.ext.bcrypt import Bcrypt
+from flask_bcrypt import Bcrypt
 bcrypt = Bcrypt()
 
-from flask.ext.principal import Principal
+from flask_principal import Principal
-principal = Principal()
+principal = Principal(use_sessions=False)
 
 from flask_mail import Mail
 smtp_mail = Mail()
+
+from lemur.metrics import Metrics
+metrics = Metrics()
+
+from raven.contrib.flask import Sentry
+sentry = Sentry()
+
+from blinker import Namespace
+signals = Namespace()
+
+from flask_cors import CORS
+cors = CORS()

lemur/factory.py

@@ -4,7 +4,7 @@
     :synopsis: This module contains all the needed functions to allow
     the factory app creation.
 
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
@@ -14,12 +14,14 @@ import imp
 import errno
 import pkg_resources
 
-from logging import Formatter
+from logging import Formatter, StreamHandler
 from logging.handlers import RotatingFileHandler
 
 from flask import Flask
+
+from lemur.certificates.hooks import activate_debug_dump
 from lemur.common.health import mod as health
-from lemur.extensions import db, migrate, principal, smtp_mail
+from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry, cors
 
 
 DEFAULT_BLUEPRINTS = (
@@ -73,7 +75,8 @@ def from_file(file_path, silent=False):
     d.__file__ = file_path
     try:
         with open(file_path) as config_file:
-            exec(compile(config_file.read(), file_path, 'exec'), d.__dict__)
+            exec(compile(config_file.read(),  # nosec: config file safe
+                 file_path, 'exec'), d.__dict__)
     except IOError as e:
         if silent and e.errno in (errno.ENOENT, errno.EISDIR):
             return False
@@ -90,15 +93,22 @@ def configure_app(app, config=None):
     :param config:
     :return:
     """
-    try:
+    # respect the config first
-        app.config.from_envvar("LEMUR_CONF")
+    if config and config != 'None':
-    except RuntimeError:
+        app.config['CONFIG_PATH'] = config
-        if config and config != 'None':
+        app.config.from_object(from_file(config))
-            app.config.from_object(from_file(config))
+    else:
-        elif os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
+        try:
-            app.config.from_object(from_file(os.path.expanduser("~/.lemur/lemur.conf.py")))
+            app.config.from_envvar("LEMUR_CONF")
-        else:
+        except RuntimeError:
-            app.config.from_object(from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'default.conf.py')))
+            # look in default paths
+            if os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
+                app.config.from_object(from_file(os.path.expanduser("~/.lemur/lemur.conf.py")))
+            else:
+                app.config.from_object(from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'default.conf.py')))
+
+    # we don't use this
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 
 
 def configure_extensions(app):
@@ -112,6 +122,12 @@ def configure_extensions(app):
     migrate.init_app(app, db)
     principal.init_app(app)
     smtp_mail.init_app(app)
+    metrics.init_app(app)
+    sentry.init_app(app)
+
+    if app.config['CORS']:
+        app.config['CORS_HEADERS'] = 'Content-Type'
+        cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True)
 
 
 def configure_blueprints(app, blueprints):
@@ -143,14 +159,22 @@ def configure_logging(app):
     app.logger.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
     app.logger.addHandler(handler)
 
+    stream_handler = StreamHandler()
+    stream_handler.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
+    app.logger.addHandler(stream_handler)
+
+    if app.config.get('DEBUG_DUMP', False):
+        activate_debug_dump()
+
 
 def install_plugins(app):
     """
     Installs new issuers that are not currently bundled with Lemur.
 
-    :param settings:
+    :param app:
     :return:
     """
+    from lemur.plugins import plugins
     from lemur.plugins.base import register
     # entry_points={
     #    'lemur.plugins': [
@@ -165,3 +189,13 @@ def install_plugins(app):
             app.logger.error("Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc()))
         else:
             register(plugin)
+
+    # ensure that we have some way to notify
+    with app.app_context():
+        slug = app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
+        try:
+            plugins.get(slug)
+        except KeyError:
+            raise Exception("Unable to location notification plugin: {slug}. Ensure that "
+                            "LEMUR_DEFAULT_NOTIFICATION_PLUGIN is set to a valid and installed notification plugin."
+                            .format(slug=slug))

lemur/logs/models.py

@@ -0,0 +1,23 @@
+"""
+.. module: lemur.logs.models
+    :platform: unix
+    :synopsis: This module contains all of the models related private key audit log.
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from sqlalchemy import Column, Integer, ForeignKey, PassiveDefault, func, Enum
+
+from sqlalchemy_utils.types.arrow import ArrowType
+
+from lemur.database import db
+
+
+class Log(db.Model):
+    __tablename__ = 'logs'
+    id = Column(Integer, primary_key=True)
+    certificate_id = Column(Integer, ForeignKey('certificates.id'))
+    log_type = Column(Enum('key_view', 'create_cert', 'update_cert', 'revoke_cert', name='log_type'), nullable=False)
+    logged_at = Column(ArrowType(), PassiveDefault(func.now()), nullable=False)
+    user_id = Column(Integer, ForeignKey('users.id'), nullable=False)