Closes: #469 (#510)

* Adding coverage.

* Attempting to adding coverage.

* Adding coveragerc.

.coveragerc

@@ -0,0 +1,2 @@
+[report]
+include = lemur/*.py

.gitattributes

@@ -1 +1,2 @@
-* text=auto
+* text=auto
+version.py  export-subst

.gitignore

@@ -1,3 +1,4 @@
+/.cache
 .coverage
 .tox
 .DS_Store

.jshintrc

@@ -8,7 +8,7 @@
   "eqeqeq": true,
   "immed": true,
   "indent": 2,
-  "latedef": true,
+  "latedef": false,
   "newcap": false,
   "noarg": true,
   "quotmark": "single",
@@ -22,6 +22,8 @@
     "angular": false,
     "moment": false,
     "toaster": false,
+    "d3": false,
+    "self": false,
     "_": false
   }
 }

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: 18d7035de5388cc7775be57f529c154bf541aab9
+    hooks:
+    -   id: trailing-whitespace
+    -   id: flake8

.travis.yml

@@ -1,6 +1,9 @@
-sudo: false
-
 language: python
+sudo: required
+dist: trusty
+
+node_js:
+  - "4.2"
 
 addons:
   postgresql: "9.4"
@@ -9,10 +12,8 @@ matrix:
   include:
     - python: "2.7"
       env: TOXENV=py27
-    - python: "3.3"
-      env: TOXENV=py33
-    - python: "3.4"
-      env: TOXENV=py34
+    - python: "3.5"
+      env: TOXENV=py35
 
 cache:
   directories:
@@ -22,18 +23,26 @@ cache:
 env:
   global:
     - PIP_DOWNLOAD_CACHE=".pip_download_cache"
-
-install:
-  - make dev-postgres
+    # do not load /etc/boto.cfg with Python 3 incompatible plugin
+    # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
+    - BOTO_CONFIG=/doesnotexist
 
 before_script:
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
+  - npm config set registry https://registry.npmjs.org
   - npm install -g bower
+  - pip install --upgrade setuptools
+
+install:
+  - pip install coveralls
 
 script:
   - make test
 
+after_success:
+  - coveralls
+
 notifications:
   email:
-    kglisson@netflix.com
+    kglisson@netflix.com

CHANGELOG.rst

@@ -1,14 +1,134 @@
 Changelog
 =========
 
-0.2.0 - `master` _
-~~~~~~~~~~~~~~~~~~~
+0.5 - `master`
+~~~~~~~~~~~~~~
 
-.. note:: This version not yet released and is under active development
+.. note:: This version is not yet released and is under active development
+
+
+0.4 - ``
+~~~~~~~~
+
+There have been quite a few issues closed in this release. Some notables:
+
+* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated
+AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for
+future enhancements of Lemur's certificate 'deployment' capabilities.
+
+* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability
+to restrict certificate expiration dates to weekdays.
+
+Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!)
+
+This will most likely be the last release to support python2.7 moving Lemur to target python3 exclusively. Please comment
+on issue #340 if this negatively affects your usage of Lemur.
+
+Upgrading
+---------
+
+See the full list of issues closed in `0.4 <https://github.com/Netflix/lemur/milestone/3>`_.
+
+.. note:: This release will need a slight migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+0.3.0 - `2016-06-06`
+~~~~~~~~~~~~~~~~~~~~
+
+This is quite a large upgrade, it is highly advised you backup your database before attempting to upgrade as this release
+requires the migration of database structure as well as data.
+
+
+Upgrading
+---------
+
+Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
+Source Plugin Owners
+--------------------
+
+The dictionary returned from a source plugin has changed keys from `public_certificate` to `body` and `intermediate_certificate` to chain.
+
+
+Issuer Plugin Owners
+--------------------
+
+This release may break your plugins, the keys in `issuer_options` have been changed from `camelCase` to `under_score`.
+This change was made to break a undue reliance on downstream options maintains a more pythonic naming convention. Renaming
+these keys should be fairly trivial, additionally pull requests have been submitted to affected plugins to help ease the transition.
+
+.. note:: This change only affects issuer plugins and does not affect any other types of plugins.
+
+
+* Closed `#63 <https://github.com/Netflix/lemur/issues/63>`_ - Validates all endpoints with Marshmallow schemas, this allows for
+    stricter input validation and better error messages when validation fails.
+* Closed `#146 <https://github.com/Netflix/lemur/issues/146>`_ - Moved authority type to first pane of authority creation wizard.
+* Closed `#147 <https://github.com/Netflix/lemur/issues/147>`_ - Added and refactored the relationship between authorities and their
+    root certificates. Displays the certificates (and chains) next the the authority in question.
+* Closed `#199 <https://github.com/Netflix/lemur/issues/199>`_ - Ensures that the dates submitted to Lemur during authority and
+    certificate creation are actually dates.
+* Closed `#230 <https://github.com/Netflix/lemur/issues/230>`_ - Migrated authority dropdown to a ui-select based dropdown, this
+    should be easier to determine what authorities are available and when an authority has actually been selected.
+* Closed `#254 <https://github.com/Netflix/lemur/issues/254>`_ - Forces certificate names to be generally unique. If a certificate name
+    (generated or otherwise) is found to be a duplicate we increment by appending a counter.
+* Closed `#254 <https://github.com/Netflix/lemur/issues/275>`_ - Switched to using Fernet generated passphrases for exported items.
+    These are more sounds that pseudo random passphrases generated before and have the nice property of being in base64.
+* Closed `#278 <https://github.com/Netflix/lemur/issues/278>`_ - Added ability to specify a custom name to certificate creation, previously
+    this was only available in the certificate import wizard.
+* Closed `#281 <https://github.com/Netflix/lemur/issues/281>`_ - Fixed an issue where notifications could not be removed from a certificate
+    via the UI.
+* Closed `#289 <https://github.com/Netflix/lemur/issues/289>`_ - Fixed and issue where intermediates were not being properly exported.
+* Closed `#315 <https://github.com/Netflix/lemur/issues/315>`_ - Made how roles are associated with certificates and authorities much more
+    explict, including adding the ability to add roles directly to certificates and authorities on creation.
+
+
+
+0.2.2 - 2016-02-05
+~~~~~~~~~~~~~~~~~~
+
+* Closed `#234 <https://github.com/Netflix/lemur/issues/234>`_ - Allows export plugins to define whether they need
+    private key material (default is True)
+* Closed `#231 <https://github.com/Netflix/lemur/issues/231>`_ - Authorities were not respecting 'owning' roles and their
+    users
+* Closed `#228 <https://github.com/Netflix/lemur/issues/228>`_ - Fixed documentation with correct filter values
+* Closed `#226 <https://github.com/Netflix/lemur/issues/226>`_ - Fixes issue were `import_certificate` was requiring
+    replacement certificates to be specified
+* Closed `#224 <https://github.com/Netflix/lemur/issues/224>`_ - Fixed an issue where NPM might not be globally available (thanks AlexClineBB!)
+* Closed `#221 <https://github.com/Netflix/lemur/issues/234>`_ - Fixes several reported issues where older migration scripts were
+    missing tables, this change removes pre 0.2 migration scripts
+* Closed `#218 <https://github.com/Netflix/lemur/issues/234>`_ - Fixed an issue where export passphrases would not validate
+
+
+0.2.1 - 2015-12-14
+~~~~~~~~~~~~~~~~~~
+
+* Fixed bug with search not refreshing values
+* Cleaned up documentation, including working supervisor example (thanks rpicard!)
+* Closed #165 - Fixed an issue with email templates
+* Closed #188 - Added ability to submit third party CSR
+* Closed #176 - Java-export should allow user to specify truststore/keystore
+* Closed #176 - Extended support for exporting certificate in P12 format
+
+
+0.2.0 - 2015-12-02
+~~~~~~~~~~~~~~~~~~
+
+* Closed #120 - Error messages not displaying long enough
+* Closed #121 - Certificate create form should not be valid until a Certificate Authority object is available
+* Closed #122 - Certificate API should allow for the specification of preceding certificates
+    You can now target a certificate(s) for replacement. When specified the replaced certificate will be marked as
+    'inactive'. This means that there will be no notifications for that certificate.
+* Closed #139 - SubCA autogenerated descriptions for their certs are incorrect
+* Closed #140 - Permalink does not change with filtering
+* Closed #144 - Should be able to search certificates by domains covered, included wildcards
+* Closed #165 - Cleaned up expiration notification template
+* Closed #160 - Cleaned up quickstart documentation (thanks forkd!)
+* Closed #144 - Now able to search by all domains in a given certificate, not just by common name
 
 
 0.1.5 - 2015-10-26
-~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 
 * **SECURITY ISSUE**: Switched from use a AES static key to Fernet encryption.
   Affects all versions prior to 0.1.5. If upgrading this will require a data migration.

LICENSE

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/

MANIFEST.in

@@ -1,4 +1,4 @@
-include setup.py package.json bower.json gulpfile.js README.rst MANIFEST.in LICENSE AUTHORS
+include setup.py version.py package.json bower.json gulpfile.js README.rst MANIFEST.in LICENSE AUTHORS
 recursive-include lemur/plugins/lemur_email/templates *
 recursive-include lemur/static *
 global-exclude *~

Makefile

@@ -1,9 +1,16 @@
 NPM_ROOT = ./node_modules
 STATIC_DIR = src/lemur/static/app
 
+USER := $(shell whoami)
+
 develop: update-submodules setup-git
 	@echo "--> Installing dependencies"
+ifeq ($(USER), root)
+	@echo "WARNING: It looks like you are installing Lemur as root. This is not generally advised."
+	npm install --unsafe-perm
+else
 	npm install
+endif
 	pip install "setuptools>=0.9.8"
 	# order matters here, base package must install first
 	pip install -e .
@@ -41,7 +48,7 @@ test: develop lint test-python
 
 testloop: develop
 	pip install pytest-xdist
-	py.test tests -f
+	coverage run --source lemur -m py.test
 
 test-cli:
 	@echo "--> Testing CLI"
@@ -60,7 +67,7 @@ test-js:
 
 test-python:
 	@echo "--> Running Python tests"
-	py.test lemur/tests || exit 1
+	coverage run --source lemur -m py.test
 	@echo ""
 
 lint: lint-python lint-js
@@ -82,4 +89,4 @@ coverage: develop
 publish:
 	python setup.py sdist bdist_wheel upload
 
-.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish
+.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish

OSSMETADATA

@@ -0,0 +1 @@
+osslifecycle=active

README.rst

@@ -5,20 +5,15 @@ Lemur
    :alt: Join the chat at https://gitter.im/Netflix/lemur
    :target: https://gitter.im/Netflix/lemur?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: https://img.shields.io/pypi/v/lemur.svg
-    :target: https://pypi.python.org/pypi/lemur/
-    :alt: Latest Version
-
 .. image:: https://readthedocs.org/projects/lemur/badge/?version=latest
     :target: https://lemur.readthedocs.org
     :alt: Latest Docs
 
+.. image:: https://img.shields.io/badge/NetflixOSS-active-brightgreen.svg
+
 .. image:: https://travis-ci.org/Netflix/lemur.svg
     :target: https://travis-ci.org/Netflix/lemur
 
-.. image:: https://badge.waffle.io/Netflix/lemur.png?label=ready&title=Ready
-    :target: https://waffle.io/Netflix/lemur
-    :alt: 'Stories in Ready'
 
 Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs
 and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults.

bower.json

@@ -6,42 +6,46 @@
   },
   "private": true,
   "dependencies": {
-    "angular": "1.3",
-    "json3": "~3.3",
-    "es5-shim": "~4.0",
-    "jquery": "~2.1",
-    "angular-resource": "1.2.15",
-    "angular-cookies": "1.2.15",
-    "angular-sanitize": "1.2.15",
-    "angular-route": "1.2.15",
-    "angular-strap": "~2.0.2",
-    "restangular": "~1.4.0",
-    "ng-table": "~0.5.4",
-    "ngAnimate": "*",
-    "moment": "~2.6.0",
-    "angular-animate": "~1.4.0",
-    "angular-loading-bar": "~0.6.0",
-    "fontawesome": "~4.2.0",
+    "jquery": "~2.2.0",
     "angular-wizard": "~0.4.0",
-    "bootswatch": "3.3.1+2",
+    "angular": "1.4.9",
+    "json3": "~3.3",
+    "es5-shim": "~4.5.0",
+    "bootstrap": "~3.3.6",
+    "angular-bootstrap": "~1.1.1",
+    "angular-animate": "~1.4.9",
+    "restangular": "~1.5.1",
+    "ng-table": "~0.8.3",
+    "moment": "~2.11.1",
+    "angular-loading-bar": "~0.8.0",
+    "angular-moment": "~0.10.3",
+    "moment-range": "~2.1.0",
     "angular-spinkit": "~0.3.3",
-    "angular-bootstrap": "~0.12.0",
-    "angular-ui-switch": "~0.1.0",
-    "angular-chart.js": "~0.7.1",
-    "satellizer": "~0.9.4",
-    "angularjs-toaster": "~0.4.14",
-    "ngletteravatar": "~3.0.1",
+    "angular-clipboard": "~1.3.0",
+    "angularjs-toaster": "~1.0.0",
+    "angular-chart.js": "~0.8.8",
+    "ngletteravatar": "~4.0.0",
+    "bootswatch": "~3.3.6",
+    "fontawesome": "~4.5.0",
+    "satellizer": "~0.13.4",
     "angular-ui-router": "~0.2.15",
-    "angular-clipboard": "~1.1.1"
-  },
-  "devDependencies": {
-    "angular-mocks": "~1.3",
-    "angular-scenario": "~1.3",
-    "ngletteravatar": "~3.0.1"
+    "font-awesome": "~4.5.0",
+    "lodash": "~4.0.1",
+    "underscore": "~1.8.3",
+    "angular-smart-table": "~2.1.6",
+    "angular-strap": ">= 2.2.2",
+    "angular-underscore": "^0.5.0",
+    "angular-translate": "^2.9.0",
+    "angular-ui-switch": "~0.1.0",
+    "angular-sanitize": "^1.5.0",
+    "angular-file-saver": "~1.0.1",
+    "angular-ui-select": "~0.17.1",
+    "d3": "^3.5.17"
   },
   "resolutions": {
-    "bootstrap": "~3.3.1",
-    "angular": "1.3"
+    "moment": ">=2.8.0 <2.11.0",
+    "lodash": ">=1.3.0 <2.5.0",
+    "angular": "1.4.9"
   },
   "ignore": [
     "**/.*",

docs/administration.rst

@@ -24,7 +24,6 @@ Basic Configuration
 
         LOG_FILE = "/logs/lemur/lemur-test.log"
 
-
 .. data:: debug
     :noindex:
 
@@ -34,7 +33,6 @@ Basic Configuration
 
         debug = False
 
-
     .. warning::
         This should never be used in a production environment as it exposes Lemur to
         remote code execution through the debug console.
@@ -53,7 +51,7 @@ Basic Configuration
         CORS = False
 
 
-.. data:: SQLACHEMY_DATABASE_URI
+.. data:: SQLALCHEMY_DATABASE_URI
     :noindex:
 
         If you have ever used sqlalchemy before this is the standard connection string used. Lemur uses a postgres database and the connection string would look something like:
@@ -63,6 +61,11 @@ Basic Configuration
         SQLALCHEMY_DATABASE_URI = 'postgresql://<user>:<password>@<hostname>:5432/lemur'
 
 
+.. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION
+    :noindex:
+
+        Specifies whether to allow certificates created by Lemur to expire on weekends. Default is True.
+
 .. data:: LEMUR_RESTRICTED_DOMAINS
     :noindex:
 
@@ -109,7 +112,7 @@ Basic Configuration
 Certificate Default Options
 ---------------------------
 
-Lemur allows you to find tune your certificates to your organization. The following defaults are presented in the UI
+Lemur allows you to fine tune your certificates to your organization. The following defaults are presented in the UI
 and are used when Lemur creates the CSR for your certificates.
 
 
@@ -145,7 +148,7 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_ORGANIZATION = "Netflix"
 
 
-.. data:: LEMUR_DEFAULT_ORGANIZATION_UNIT
+.. data:: LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
     :noindex:
 
     ::
@@ -153,6 +156,14 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = "Operations"
 
 
+.. data:: LEMUR_DEFAULT_ISSUER_PLUGIN
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_ISSUER_PLUGIN = "verisign-issuer"
+
+
 Notification Options
 --------------------
 
@@ -163,7 +174,7 @@ to be sent to subscribers.
 Templates for expiration emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
 Notifications are sent to the certificate creator, owner and security team as specified by the `LEMUR_SECURITY_TEAM_EMAIL` configuration parameter.
 
-Certificates marked as in-active will **not** be notified of upcoming expiration. This enables a user to essentially
+Certificates marked as inactive will **not** be notified of upcoming expiration. This enables a user to essentially
 silence the expiration. If a certificate is active and is expiring the above will be notified according to the `LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS` or
 30, 15, 2 days before expiration if no intervals are set.
 
@@ -173,47 +184,137 @@ Lemur supports sending certification expiration notifications through SES and SM
 .. data:: LEMUR_EMAIL_SENDER
     :noindex:
 
-            Specifies which service will be delivering notification emails. Valid values are `SMTP` or `SES`
+    Specifies which service will be delivering notification emails. Valid values are `SMTP` or `SES`
 
-.. note::
-    If using STMP as your provider you will need to define additional configuration options as specified by Flask-Mail.
-    See: `Flask-Mail <https://pythonhosted.org/Flask-Mail>`_
+    .. note::
+        If using SMTP as your provider you will need to define additional configuration options as specified by Flask-Mail.
+        See: `Flask-Mail <https://pythonhosted.org/Flask-Mail>`_
+
+        If you are using SES the email specified by the `LEMUR_MAIL` configuration will need to be verified by AWS before
+        you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
 
-    If you are using SES the email specified by the `LEMUR_MAIL` configuration will need to be verified by AWS before
-    you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
 
 .. data:: LEMUR_MAIL
     :noindex:
 
-            Lemur sender's email
+        Lemur sender's email
 
         ::
 
-        LEMUR_MAIL = 'lemur.example.com'
+            LEMUR_MAIL = 'lemur.example.com'
 
 
 .. data:: LEMUR_SECURITY_TEAM_EMAIL
     :noindex:
 
-            This is an email or list of emails that should be notified when a certificate is expiring. It is also the contact email address for any discovered certificate.
+        This is an email or list of emails that should be notified when a certificate is expiring. It is also the contact email address for any discovered certificate.
 
         ::
 
-        LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
+            LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
 
 
 .. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS
     :noindex:
 
-            Lemur notification intervals
+        Lemur notification intervals
 
         ::
 
-        LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
+            LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
 
 
-Authority Options
------------------
+Authentication Options
+----------------------
+Lemur currently supports Basic Authentication, Ping OAuth2, and Google out of the box. Additional flows can be added relatively easily.
+If you are not using an authentication provider you do not need to configure any of these options.
+
+For more information about how to use social logins, see: `Satellizer <https://github.com/sahat/satellizer>`_
+
+.. data:: ACTIVE_PROVIDERS
+    :noindex:
+
+        ::
+
+            ACTIVE_PROVIDERS = ["ping", "google"]
+
+.. data:: PING_SECRET
+    :noindex:
+
+        ::
+
+            PING_SECRET = 'somethingsecret'
+
+.. data:: PING_ACCESS_TOKEN_URL
+    :noindex:
+
+        ::
+
+            PING_ACCESS_TOKEN_URL = "https://<yourpingserver>/as/token.oauth2"
+
+
+.. data:: PING_USER_API_URL
+    :noindex:
+
+        ::
+
+            PING_USER_API_URL = "https://<yourpingserver>/idp/userinfo.openid"
+
+.. data:: PING_JWKS_URL
+    :noindex:
+
+        ::
+
+            PING_JWKS_URL = "https://<yourpingserver>/pf/JWKS"
+
+.. data:: PING_NAME
+    :noindex:
+
+        ::
+
+            PING_NAME = "Example Oauth2 Provider"
+
+.. data:: PING_CLIENT_ID
+    :noindex:
+
+        ::
+
+            PING_CLIENT_ID = "client-id"
+
+.. data:: PING_REDIRECT_URI
+    :noindex:
+
+        ::
+
+            PING_REDIRECT_URI = "https://<yourlemurserver>/api/1/auth/ping"
+
+.. data:: PING_AUTH_ENDPOINT
+    :noindex:
+
+        ::
+
+            PING_AUTH_ENDPOINT = "https://<yourpingserver>/oauth2/authorize"
+
+.. data:: GOOGLE_CLIENT_ID
+    :noindex:
+
+        ::
+
+            GOOGLE_CLIENT_ID = "client-id"
+
+.. data:: GOOGLE_SECRET
+    :noindex:
+
+        ::
+
+            GOOGLE_SECRET = "somethingsecret"
+
+
+Plugin Specific Options
+-----------------------
+
+Verisign Issuer Plugin
+^^^^^^^^^^^^^^^^^^^^^^
 
 Authorities will each have their own configuration options. There is currently just one plugin bundled with Lemur,
 Verisign/Symantec. Additional plugins may define additional options. Refer to the plugin's own documentation
@@ -260,53 +361,79 @@ for those plugins.
         This is the root to be used for your CA chain
 
 
-Authentication
---------------
-Lemur currently supports Basic Authentication and Ping OAuth2 out of the box. Additional flows can be added relatively easily.
-If you are not using Ping you do not need to configure any of these options.
+Digicert Issuer Plugin
+~~~~~~~~~~~~~~~~~~~~~~
 
-For more information about how to use social logins, see: `Satellizer <https://github.com/sahat/satellizer>`_
+The following configuration properties are required to use the Digicert issuer plugin.
 
-.. data:: PING_SECRET
+
+.. data:: DIGICERT_URL
     :noindex:
 
-    ::
+            This is the url for the Digicert API
 
-        PING_SECRET = 'somethingsecret'
 
-.. data:: PING_ACCESS_TOKEN_URL
+.. data:: DIGICERT_API_KEY
     :noindex:
 
-    ::
-
-        PING_ACCESS_TOKEN_URL = "https://<yourpingserver>/as/token.oauth2"
+            This is the Digicert API key
 
 
-.. data:: PING_USER_API_URL
+.. data:: DIGICERT_ORG_ID
     :noindex:
 
-    ::
+            This is the Digicert organization ID tied to your API key
 
-        PING_USER_API_URL = "https://<yourpingserver>/idp/userinfo.openid"
 
-.. data:: PING_JWKS_URL
+.. data:: DIGICERT_INTERMEDIATE
     :noindex:
 
-    ::
+            This is the intermediate to be used for your CA chain
 
-        PING_JWKS_URL = "https://<yourpingserver>/pf/JWKS"
+
+.. data:: DIGICERT_ROOT
+    :noindex:
+
+            This is the root to be used for your CA chain
+
+
+.. data:: DIGICERT_DEFAULT_VALIDITY
+    :noindex:
+
+            This is the default validity (in years), if no end date is specified. (Default: 1)
 
 
 
-AWS Plugin Configuration
-========================
+CFSSL Issuer Plugin
+^^^^^^^^^^^^^^^^^^^
+
+The following configuration properties are required to use the the CFSSL issuer plugin.
+
+.. data:: CFSSL_URL
+    :noindex:
+
+        This is the URL for the CFSSL API
+
+.. data:: CFSSL_ROOT
+    :noindex:
+
+        This is the root to be used for your CA chain
+
+.. data:: CFSSL_INTERMEDIATE
+    :noindex:
+
+        This is the intermediate to be used for your CA chain
+
+
+AWS Source/Destination Plugin
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In order for Lemur to manage its own account and other accounts we must ensure it has the correct AWS permissions.
 
 .. note:: AWS usage is completely optional. Lemur can upload, find and manage TLS certificates in AWS. But is not required to do so.
 
 Setting up IAM roles
---------------------
+""""""""""""""""""""
 
 Lemur's AWS plugin uses boto heavily to talk to all the AWS resources it manages. By default it uses the on-instance credentials to make the necessary calls.
 
@@ -411,7 +538,8 @@ IAM-ServerCertificate
 
 
 Setting up STS access
----------------------
+"""""""""""""""""""""
+
 Once we have setup our accounts we need to ensure that we create a trust relationship so that LemurInstanceProfile can assume the Lemur role.
 
 In the AWS console select the Lemur IAM role and select the Trust Relationships tab and click Edit Trust Relationship
@@ -438,7 +566,7 @@ Below is an example policy:
 
 
 Adding N+1 accounts
--------------------
+"""""""""""""""""""
 
 To add another account we go to the new account and create a new Lemur IAM role with the same policy as above.
 
@@ -466,7 +594,7 @@ An example policy:
     }
 
 Setting up SES
---------------
+""""""""""""""
 
 Lemur has built in support for sending it's certificate notifications via Amazon's simple email service (SES). To force
 Lemur to use SES ensure you are the running as the IAM role defined above and that you have followed the steps outlined
@@ -481,18 +609,6 @@ Will be the sender of all notifications, so ensure that it is verified with AWS.
 SES if the default notification gateway and will be used unless SMTP settings are configured in the application configuration
 settings.
 
-Upgrading Lemur
-===============
-
-Lemur provides an easy way to upgrade between versions. Simply download the newest
-version of Lemur from pypi and then apply any schema changes with the following command.
-
-.. code-block:: bash
-
-    $ lemur db upgrade
-
-.. note:: Internally, this uses `Alembic <https://alembic.readthedocs.org/en/latest/>`_ to manage database migrations.
-
 .. _CommandLineInterface:
 
 Command Line Interface
@@ -562,24 +678,33 @@ All commands default to `~/.lemur/lemur.conf.py` if a configuration is not speci
 
     Traverses every certificate that Lemur is aware of and attempts to understand its validity.
     It utilizes both OCSP and CRL. If Lemur is unable to come to a conclusion about a certificates
-    validity its status is marked 'unknown'
+    validity its status is marked 'unknown'.
 
 
 .. data:: sync
 
     Sync attempts to discover certificates in the environment that were not created by Lemur. If you wish to only sync
-    a few sources you can pass a comma delimited list of sources to sync
+    a few sources you can pass a comma delimited list of sources to sync.
 
     ::
 
-        lemur sync source1,source2
+        lemur sync -s source1,source2
 
 
-    Additionally you can also list the available sources that Lemur can sync
+    Additionally you can also list the available sources that Lemur can sync.
 
     ::
 
-        lemur sync -list
+        lemur sync
+
+
+.. data:: notify
+
+    Will traverse all current notifications and see if any of them need to be triggered.
+
+    ::
+
+        lemur notify
 
 
 Sub-commands
@@ -602,32 +727,6 @@ and to get help on sub-commands
         lemur certificates --help
 
 
-Identity and Access Management
-==============================
-
-Lemur uses a Role Based Access Control (RBAC) mechanism to control which users have access to which resources. When a
-user is first created in Lemur they can be assigned one or more roles. These roles are typically dynamically created
-depending on a external identity provider (Google, LDAP, etc.,) or are hardcoded within Lemur and associated with special
-meaning.
-
-Within Lemur there are three main permissions: AdminPermission, CreatorPermission, OwnerPermission. Sub-permissions such
-as ViewPrivateKeyPermission are compositions of these three main Permissions.
-
-Lets take a look at how these permissions are used:
-
-Each `Authority` has a set of roles associated with it. If a user is also associated with the same roles
-that the `Authority` is associated with, Lemur allows that user to user/view/update that `Authority`.
-
-This RBAC is also used when determining which users can access which certificate private key. Lemur's current permission
-structure is setup such that if the user is a `Creator` or `Owner` of a given certificate they are allow to view that
-private key. Owners can also be a role name, such that any user with the same role as owner will be allowed to view the
-private key information.
-
-These permissions are applied to the user upon login and refreshed on every request.
-
-.. seealso::
-    `Flask-Principal <https://pythonhosted.org/Flask-Principal>`_
-
 
 Upgrading Lemur
 ===============
@@ -657,4 +756,207 @@ After you have the latest version of the Lemur code base you must run any needed
         lemur db upgrade
 
 
-This will ensure that any needed tables or columns are created or destroyed.
+This will ensure that any needed tables or columns are created or destroyed.
+
+.. note::
+    Internally, this uses `Alembic <https://alembic.readthedocs.org/en/latest/>`_ to manage database migrations.
+
+.. note::
+    By default Alembic looks for the `migrations` folder in the current working directory.The migrations folder is
+    located under `<LEMUR_HOME>/lemur/migrations` if you are running the lemur command from any location besides
+    `<LEMUR_HOME>/lemur` you will need to pass the `-d` flag to specify the absolute file path to the `migrations` folder.
+
+Plugins
+=======
+
+There are several interfaces currently available to extend Lemur. These are a work in
+progress and the API is not frozen.
+
+Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec.
+
+Verisign/Symantec
+-----------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for the VICE 2.0 API
+
+
+Cryptography
+------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Issuer
+:Description:
+    Toy certificate authority that creates self-signed certificate authorities.
+    Allows for the creation of arbitrary authorities and end-entity certificates.
+    This is *not* recommended for production use.
+
+
+Acme
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Issuer
+:Description:
+    Adds support for the ACME protocol (including LetsEncrypt) with domain validation being handled Route53.
+
+
+Atlas
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Metric
+:Description:
+    Adds basic support for the `Atlas <https://github.com/Netflix/atlas/wiki>`_ telemetry system.
+
+
+Email
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Notification
+:Description:
+    Adds support for basic email notifications via SES.
+
+
+Slack
+-----
+
+:Authors:
+    Harm Weites <harm@weites.com>
+:Type:
+    Notification
+:Description:
+    Adds support for slack notifications.
+
+
+AWS
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Source
+:Description:
+    Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment.
+
+
+AWS
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Destination
+:Description:
+    Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment.
+
+
+Kubernetes
+----------
+
+:Authors:
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Destination
+:Description:
+    Allows Lemur to upload generated certificates to the Kubernetes certificate store.
+
+
+Java
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Generates java compatible .jks keystores and truststores from Lemur managed certificates.
+
+
+Openssl
+-------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Leverages Openssl to support additional export formats (pkcs12)
+
+
+CFSSL
+-----
+
+:Authors:
+    Charles Hendrie <chad.hendrie@thomsonreuters.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for generating certificates from the private certificate authority CFSSL
+
+
+3rd Party Plugins
+=================
+
+The following plugins are available and maintained by members of the Lemur community:
+
+Digicert
+--------
+
+:Authors:
+    Chris Dorros
+:Type:
+    Issuer
+:Description:
+    Adds support for basic Digicert
+:Links:
+    https://github.com/opendns/lemur-digicert
+
+
+Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
+get it added.
+
+Want to create your own extension? See :doc:`../developer/plugins/index` to get started.
+
+
+Identity and Access Management
+==============================
+
+Lemur uses a Role Based Access Control (RBAC) mechanism to control which users have access to which resources. When a
+user is first created in Lemur they can be assigned one or more roles. These roles are typically dynamically created
+depending on a external identity provider (Google, LDAP, etc.,) or are hardcoded within Lemur and associated with special
+meaning.
+
+Within Lemur there are three main permissions: AdminPermission, CreatorPermission, OwnerPermission. Sub-permissions such
+as ViewPrivateKeyPermission are compositions of these three main Permissions.
+
+Lets take a look at how these permissions are used:
+
+Each `Authority` has a set of roles associated with it. If a user is also associated with the same roles
+that the `Authority` is associated with, Lemur allows that user to user/view/update that `Authority`.
+
+This RBAC is also used when determining which users can access which certificate private key. Lemur's current permission
+structure is setup such that if the user is a `Creator` or `Owner` of a given certificate they are allow to view that
+private key. Owners can also be a role name, such that any user with the same role as owner will be allowed to view the
+private key information.
+
+These permissions are applied to the user upon login and refreshed on every request.
+
+.. seealso::
+
+    `Flask-Principal <https://pythonhosted.org/Flask-Principal>`_

docs/conf.py

@@ -11,7 +11,6 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
-
 import sys
 import os
 
@@ -54,10 +53,12 @@ copyright = u'2015, Netflix Inc.'
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-# The short X.Y version.
-version = '0.1'
-# The full version, including alpha/beta/rc tags.
-release = '0.1.3'
+base_dir = os.path.join(os.path.dirname(__file__), os.pardir)
+about = {}
+with open(os.path.join(base_dir, "lemur", "__about__.py")) as f:
+    exec(f.read(), about)
+
+version = release = about["__version__"]
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -100,9 +101,13 @@ pygments_style = 'sphinx'
 
 # -- Options for HTML output ----------------------------------------------
 
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-html_theme = 'default'
+# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/developer/index.rst

@@ -144,6 +144,17 @@ If you've made changes and need to compile them by hand for any reason, you can
 
 The minified and processed files should be committed alongside the unprocessed changes.
 
+It's also important to note that Lemur's frontend and API are not tied together. The API does not serve any of the static assets, we rely on nginx or some other file server to server all of the static assets.
+During development that means we need an additional server to serve those static files for the GUI.
+
+This is accomplished with a Gulp task:
+
+::
+
+    ./node_modules/.bin/gulp serve
+
+The gulp task compiles all the JS/CSS/HTML files and opens the Lemur welcome page in your default browsers. Additionally any changes to made to the JS/CSS/HTML with be reloaded in your browsers.
+
 Developing with Flask
 ----------------------
 
@@ -180,19 +191,92 @@ You can see a list of open pull requests (pending changes) by visiting https://g
 
 Pull requests should be against **master** and pass all TravisCI checks
 
-Plugins
-=======
+
+Writing a Plugin
+================
 
 .. toctree::
-    :maxdepth: 1
+    :maxdepth: 2
 
     plugins/index
 
+
+REST API
+========
+
+Lemur's front end is entirely API driven. Any action that you can accomplish via the UI can also be accomplished by the
+API. The following is documents and provides examples on how to make requests to the Lemur API.
+
+Authentication
+--------------
+
+.. automodule:: lemur.auth.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Destinations
+------------
+
+.. automodule:: lemur.destinations.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Notifications
+-------------
+
+.. automodule:: lemur.notifications.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Users
+-----
+
+.. automodule:: lemur.users.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Roles
+-----
+
+.. automodule:: lemur.roles.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Certificates
+------------
+
+.. automodule:: lemur.certificates.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Authorities
+-----------
+
+.. automodule:: lemur.authorities.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Domains
+-------
+
+.. automodule:: lemur.domains.views
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
 Internals
 =========
 
 .. toctree::
-    :maxdepth: 1
+    :maxdepth: 2
 
     internals/lemur
 

docs/developer/internals/lemur.plugins.lemur_cfssl.rst

@@ -0,0 +1,20 @@
+lemur_cfssl Package
+===================
+
+:mod:`lemur_cfssl` Package
+--------------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.rst

@@ -27,6 +27,6 @@ Subpackages
     lemur.plugins.base
     lemur.plugins.bases
     lemur.plugins.lemur_aws
-    lemur.plugins.lemur_cloudca
+    lemur.plugins.lemur_cfssl
     lemur.plugins.lemur_email
     lemur.plugins.lemur_verisign

docs/developer/internals/lemur.rst

@@ -96,5 +96,4 @@ Subpackages
     lemur.notifications
     lemur.plugins
     lemur.roles
-    lemur.status
     lemur.users

docs/developer/internals/lemur.status.rst

@@ -1,11 +0,0 @@
-status Package
-==============
-
-:mod:`views` Module
--------------------
-
-.. automodule:: lemur.status.views
-    :noindex:
-    :members:
-    :undoc-members:
-    :show-inheritance:

docs/developer/plugins/index.rst

@@ -1,6 +1,3 @@
-Writing a Plugin
-================
-
 Several interfaces exist for extending Lemur:
 
 * Issuer (lemur.plugins.base.issuer)
@@ -140,6 +137,12 @@ Destination
 Destination plugins allow you to propagate certificates managed by Lemur to additional third parties. This provides flexibility when
 different orchestration systems have their own way of manage certificates or there is an existing system you wish to integrate with Lemur.
 
+By default destination plugins have a private key requirement. If your plugin does not require a certificates private key mark `requires_key = False`
+in the plugins base class like so::
+
+    class MyDestinationPlugin(DestinationPlugin):
+        requires_key = False
+
 The DestinationPlugin requires only one function to be implemented::
 
     def upload(self, cert, private_key, cert_chain, options, **kwargs):
@@ -214,8 +217,8 @@ certificate Lemur does not know about and adding the certificate to it's invento
 
 The `SourcePlugin` object has one default option of `pollRate`. This controls the number of seconds which to get new certificates.
 
- .. warning::
-    Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. A lock file is generated to guarantee that 
+.. warning::
+    Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. A lock file is generated to guarantee that
     only one sync is running at a time. It also means that the minimum resolution of a source plugin poll rate is effectively 15min. You can always specify a faster cron
     job if you need a higher resolution sync job.
 
@@ -226,10 +229,31 @@ The `SourcePlugin` object requires implementation of one function::
           #  request.get("some source of certificates")
 
 
-.. Note::
+.. note::
     Often times to facilitate code re-use it makes sense put source and destination plugins into one package.
 
 
+Export
+------
+
+Formats, formats and more formats. That's the current PKI landscape. See the always relevant `xkcd <https://xkcd.com/927/>`_.
+Thankfully Lemur supports the ability to output your certificates into whatever format you want. This integration comes by the way
+of Export plugins. Support is still new and evolving, the goal of these plugins is to return raw data in a new format that
+can then be used by any number of applications. Included in Lemur is the `JavaExportPlugin` which currently supports generating
+a Java Key Store (JKS) file for use in Java based applications.
+
+
+The `ExportPlugin` object requires the implementation of one function::
+
+    def export(self, body, chain, key, options, **kwargs):
+        # sys.call('openssl hokuspocus')
+        # return "extension", passphrase, raw
+
+
+.. note::
+    Support of various formats sometimes relies on external tools system calls. Always be mindful of sanitizing any input to these calls.
+
+
 Testing
 =======
 
@@ -259,11 +283,7 @@ The ``conftest.py`` file is our main entry-point for py.test. We need to configu
 
 .. code-block:: python
 
-   from __future__ import absolute_import
-
-   pytest_plugins = [
-       'lemur.utils.pytest'
-   ]
+   from lemur.tests.conftest import *  # noqa
 
 
 Test Cases
@@ -273,14 +293,18 @@ You can now inherit from Lemur's core test classes. These are Django-based and e
 
 .. code-block:: python
 
-   # test_myextension.py
-   from __future__ import absolute_import
+    import pytest
+    from lemur.tests.vectors import INTERNAL_CERTIFICATE_A_STR, INTERNAL_PRIVATE_KEY_A_STR
 
-   from lemur.testutils import TestCase
+    def test_export_keystore(app):
+        from lemur.plugins.base import plugins
+        p = plugins.get('java-keystore-jks')
+        options = [{'name': 'passphrase', 'value': 'test1234'}]
+        with pytest.raises(Exception):
+            p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
 
-   class MyExtensionTest(TestCase):
-       def test_simple(self):
-          assert 1 != 2
+        raw = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
+        assert raw != b""
 
 
 Running Tests
@@ -292,13 +316,14 @@ Running tests follows the py.test standard. As long as your test files and metho
 
     $ py.test -v
     ============================== test session starts ==============================
-    platform darwin -- Python 2.7.9 -- py-1.4.26 -- pytest-2.6.4/python2.7
-    plugins: django
-    collected 1 items
+    platform darwin -- Python 2.7.10, pytest-2.8.5, py-1.4.30, pluggy-0.3.1
+    cachedir: .cache
+    plugins: flask-0.10.0
+    collected 346 items
 
-    tests/test_myextension.py::MyExtensionTest::test_simple PASSED
+    lemur/plugins/lemur_acme/tests/test_acme.py::test_get_certificates PASSED
 
     =========================== 1 passed in 0.35 seconds ============================
 
 
-.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above. View the source: # TODO
+.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above.

docs/developer/rest.rst

@@ -1,66 +0,0 @@
-Lemur's front end is entirely API driven. Any action that you can accomplish via the UI can also be accomplished by the
-UI. The following is documents and provides examples on how to make requests to the Lemur API.
-
-Authentication
---------------
-
-.. automodule:: lemur.auth.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Destinations
-------------
-
-.. automodule:: lemur.destinations.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Notifications
--------------
-
-.. automodule:: lemur.notifications.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Users
------
-
-.. automodule:: lemur.users.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Roles
------
-
-.. automodule:: lemur.roles.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Certificates
-------------
-
-.. automodule:: lemur.certificates.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Authorities
------------
-
-.. automodule:: lemur.authorities.views
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-Domains
--------
-
-.. automodule:: lemur.domains.views
-    :members:
-    :undoc-members:
-    :show-inheritance:

docs/doing-a-release.rst

@@ -0,0 +1,53 @@
+Doing a release
+===============
+
+Doing a release of ``lemur`` requires a few steps.
+
+Bumping the version number
+--------------------------
+
+The next step in doing a release is bumping the version number in the
+software.
+
+* Update the version number in ``lemur/__about__.py``.
+* Set the release date in the :doc:`/changelog`.
+* Do a commit indicating this.
+* Send a pull request with this.
+* Wait for it to be merged.
+
+Performing the release
+----------------------
+
+The commit that merged the version number bump is now the official release
+commit for this release. You will need to have ``gpg`` installed and a ``gpg``
+key in order to do a release. Once this has happened:
+
+* Run ``invoke release {version}``.
+
+The release should now be available on PyPI and a tag should be available in
+the repository.
+
+Verifying the release
+---------------------
+
+You should verify that ``pip install lemur`` works correctly:
+
+.. code-block:: pycon
+
+    >>> import lemur
+    >>> lemur.__version__
+    '...'
+
+Verify that this is the version you just released.
+
+Post-release tasks
+------------------
+
+* Update the version number to the next major (e.g. ``0.5.dev1``) in
+  ``lemur/__about__.py`` and
+* Add new :doc:`/changelog` entry with next version and note that it is under
+  active development
+* Send a pull request with these items
+* Check for any outstanding code undergoing a deprecation cycle by looking in
+  ``lemur.utils`` for ``DeprecatedIn**`` definitions. If any exist open
+  a ticket to increment them for the next release.

docs/faq.rst

@@ -14,6 +14,11 @@ I am seeing Lemur's javascript load in my browser but not the CSS.
   :doc:`production/index` for example configurations.
 
 
+After installing Lemur I am unable to login
+  Ensure that you are trying to login with the credentials you entered during `lemur init`. These are separate
+  from the postgres database credentials.
+
+
 Running 'lemur db upgrade' seems stuck.
   Most likely, the upgrade is stuck because an existing query on the database is holding onto a lock that the
   migration needs.

docs/index.rst

@@ -27,8 +27,7 @@ Administration
 .. toctree::
     :maxdepth: 2
 
-    administration/index
-    plugins/index
+    administration
 
 Developers
 ----------
@@ -38,14 +37,21 @@ Developers
 
     developer/index
 
-
-REST API
+Security
 --------
 
 .. toctree::
     :maxdepth: 2
 
-    developer/rest
+    security
+
+Doing a Release
+---------------
+
+.. toctree::
+    :maxdepth: 1
+
+    doing-a-release
 
 FAQ
 ----

docs/plugins/index.rst

@@ -1,20 +0,0 @@
-Plugins
-=======
-
-There are several interfaces currently available to extend Lemur. These are a work in
-progress and the API is not frozen.
-
-Bundled Plugins
----------------
-
-Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec and CloudCA services.
-
-3rd Party Extensions
---------------------
-
-The following extensions are available and maintained by members of the Lemur community:
-
-Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
-get it added.
-
-Want to create your own extension? See :doc:`../developer/plugins/index` to get started.

docs/production/index.rst

@@ -110,7 +110,7 @@ You can make some adjustments to get a better user experience::
        error_log   /var/log/nginx/log/lemur.error.log;
 
        location /api {
-            proxy_pass  http://127.0.0.1:5000;
+            proxy_pass  http://127.0.0.1:8000;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
             proxy_redirect off;
             proxy_buffering off;
@@ -176,7 +176,7 @@ sensitive nature of Lemur and what it controls makes this essential. This is a s
        resolver <IP DNS resolver>;
 
        location /api {
-            proxy_pass  http://127.0.0.1:5000;
+            proxy_pass  http://127.0.0.1:8000;
             proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
             proxy_redirect off;
             proxy_buffering off;
@@ -257,13 +257,12 @@ Create a configuration file named supervisor.ini::
     nodaemon=false
     minfds=1024
     minprocs=200
-    user=lemur
 
     [program:lemur]
     command=python /path/to/lemur/manage.py manage.py start
 
     directory=/path/to/lemur/
-    environment=PYTHONPATH='/path/to/lemur/'
+    environment=PYTHONPATH='/path/to/lemur/',LEMUR_CONF='/home/lemur/.lemur/lemur.conf.py'
     user=lemur
     autostart=true
     autorestart=true
@@ -296,3 +295,25 @@ Then you can manage the process by running::
 It will start a shell from which you can start/stop/restart the service.
 
 You can read all errors that might occur from /tmp/lemur.log.
+
+
+Periodic Tasks
+==============
+
+Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
+a cron job that runs the commands.
+
+There are currently three commands that could/should be run on a periodic basis:
+
+- `notify`
+- `check_revoked`
+- `sync`
+
+How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
+`sync` is typically run every 15 minutes.
+
+Example cron entries::
+
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify
+    */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur sync -s all
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur check_revoked

docs/quickstart/index.rst

@@ -1,88 +1,93 @@
 Quickstart
 **********
 
-This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.
-This guide assumes a clean Ubuntu 14.04 instance, commands may differ based on the OS and configuration being used.
+This guide will step you through setting up a Python-based virtualenv, installing the required packages, and configuring the basic web service.  This guide assumes a clean Ubuntu 14.04 instance, commands may differ based on the OS and configuration being used.
 
 Pressed for time? See the Lemur docker file on `Github <https://github.com/Netflix/lemur-docker>`_.
 
+
 Dependencies
 ------------
 
 Some basic prerequisites which you'll need in order to run Lemur:
 
-* A UNIX-based operating system. We test on Ubuntu, develop on OS X
+* A UNIX-based operating system (we test on Ubuntu, develop on OS X)
 * Python 2.7
 * PostgreSQL
 * Nginx
 
-.. note:: Lemur was built with in AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB),
-    are largely handled for us. Lemur does **not** require AWS to function. Our guides and documentation try to be
-    be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
+.. note:: Lemur was built with in AWS in mind. This means that things such as databases (RDS), mail (SES), and TLS (ELB), are largely handled for us.  Lemur does **not** require AWS to function. Our guides and documentation try to be be as generic as possible and are not intended to document every step of launching Lemur into a given environment.
+
+
+Installing Build Dependencies
+-----------------------------
+
+If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies:
+
+.. code-block:: bash
+
+    $ sudo apt-get update
+    $ sudo apt-get install nodejs-legacy python-pip python-dev libpq-dev build-essential libssl-dev libffi-dev nginx git supervisor npm postgresql
+
+.. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver.  npm is needed if you're installing Lemur from the source (e.g., from git).
+
+Now, install Python ``virtualenv`` package:
+
+.. code-block:: bash
+
+    $ sudo pip install -U virtualenv
 
 
 Setting up an Environment
 -------------------------
 
-The first thing you'll need is the Python ``virtualenv`` package. You probably already
-have this, but if not, you can install it with::
+In this guide, Lemur will be installed in ``/www``, so you need to create that structure first:
 
-  pip install -U virtualenv
+.. code-block:: bash
 
-Once that's done, choose a location for the environment, and create it with the ``virtualenv``
-command. For our guide, we're going to choose ``/www/lemur/``::
+    $ sudo mkdir /www
+    $ cd /www
 
-  virtualenv /www/lemur/
+Clone Lemur inside the just created directory and give yourself write permission (we assume ``lemur`` is the user):
 
-Finally, activate your virtualenv::
+.. code-block:: bash
 
-  source /www/lemur/bin/activate
+    $ sudo git clone https://github.com/Netflix/lemur
+    $ sudo chown -R lemur lemur/
 
-.. note:: Activating the environment adjusts your PATH, so that things like pip now
-          install into the virtualenv by default.
+Create the virtual environment, activate it and enter the Lemur's directory:
 
+.. code-block:: bash
 
-Installing build dependencies
------------------------------
+    $ virtualenv lemur
+    $ source /www/lemur/bin/activate
+    $ cd lemur
 
-If installing Lemur on truely bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's
-dependencies::
-
-    $ sudo apt-get update
-    $ sudo apt-get install nodejs-legacy python-pip libpq-dev python-dev build-essential libssl-dev libffi-dev nginx git supervisor
-
-And optionally if your database is going to be on the same host as the webserver::
-
-    $ sudo apt-get install postgresql
+.. note:: Activating the environment adjusts your PATH, so that things like pip now install into the virtualenv by default.
 
 
 Installing from Source
 ~~~~~~~~~~~~~~~~~~~~~~
 
-If you're installing the Lemur source (e.g. from git), you'll also need to install **npm**.
-
 Once your system is prepared, ensure that you are in the virtualenv:
 
 .. code-block:: bash
 
   $ which python
 
-
 And then run:
 
 .. code-block:: bash
 
   $ make develop
 
-.. Note:: This command will install npm dependencies as well as compile static assets.
+.. note:: This command will install npm dependencies as well as compile static assets.
 
 
 Creating a configuration
 ------------------------
 
-Before we run Lemur we must create a valid configuration file for it.
-
-The Lemur cli comes with a simple command to get you up and running quickly.
+Before we run Lemur, we must create a valid configuration file for it.  The Lemur command line interface comes with a simple command to get you up and running quickly.
 
 Simply run:
 
@@ -90,89 +95,82 @@ Simply run:
 
   $ lemur create_config
 
-.. Note:: This command will create a default configuration under `~/.lemur/lemur.conf.py` you
-    can specify this location by passing the `config_path` parameter to the `create_config` command.
+.. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command.
+
+You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment.
 
-You can specify `-c` or `--config` to any Lemur command to specify the current environment
-you are working in. Lemur will also look under the environmental variable `LEMUR_CONF` should
-that be easier to setup in your environment.
 
 Update your configuration
 -------------------------
 
-Once created you will need to update the configuration file with information about your environment,
-such as which database to talk to, where keys are stored etc..
+Once created, you will need to update the configuration file with information about your environment, such as which database to talk to, where keys are stored etc.
+
+.. note:: If you are unfamiliar with with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
+      ``postgresql://userame:password@<database-fqdn>:<database-port>/<database-name>``
 
-.. Note:: If you are unfamiliar with with the SQLALCHEMY_DATABASE_URI string it can be broken up like so:
-      postgresql://userame:password@databasefqdn:databaseport/databasename
 
 Setup Postgres
 --------------
 
-For production a dedicated database is recommended, for this guide we will assume postgres has been installed and is on
-the same machine that Lemur is installed on.
+For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on.
 
-First, set a password for the postgres user.  For this guide, we will use **lemur** as an example but you should use the database password generated for by Lemur::
+First, set a password for the postgres user.  For this guide, we will use ``lemur`` as an example but you should use the database password generated by Lemur:
 
-     $ sudo -u postgres psql postgres
-     # \password postgres
-     Enter new password: lemur
-     Enter it again: lemur
+.. code-block:: bash
 
-Type CTRL-D to exit psql once you have changed the password.
+    $ sudo -u postgres -i
+    # \password postgres
+    Enter new password: lemur
+    Enter it again: lemur
 
-Next, we will create our new database::
+Once successful, type CTRL-D to exit the Postgres shell.
 
-     $ sudo -u postgres createdb lemur
+Next, we will create our new database:
+
+.. code-block:: bash
+
+    $ sudo -u postgres createdb lemur
 
 .. _InitializingLemur:
 
+.. note::
+    For this guide we assume you will use the `postgres` user to connect to your database, when deploying to a VM or container this is often all you will need. If you have a shared database it is recommend you give Lemur it's own user.
+
 Initializing Lemur
 ------------------
 
-Lemur provides a helpful command that will initialize your database for you. It creates a default user (lemur) that is
-used by Lemur to help associate certificates that do not currently have an owner. This is most commonly the case when
-Lemur has discovered certificates from a third party source. This is also a default user that can be used to
-administer Lemur.
+Lemur provides a helpful command that will initialize your database for you. It creates a default user (``lemur``) that is used by Lemur to help associate certificates that do not currently have an owner. This is most commonly the case when Lemur has discovered certificates from a third party source.  This is also a default user that can be used to administer Lemur.
 
-In addition to creating a new user, Lemur also creates a few default email notifications. These notifications are based
-on a few configuration options such as `LEMUR_SECURITY_TEAM_EMAIL`. They basically guarantee that every cerificate within
-Lemur will send one expiration notification to the security team.
+In addition to creating a new user, Lemur also creates a few default email notifications.  These notifications are based on a few configuration options such as ``LEMUR_SECURITY_TEAM_EMAIL``.  They basically guarantee that every certificate within Lemur will send one expiration notification to the security team.
 
-Additional notifications can be created through the UI or API.
-See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+Additional notifications can be created through the UI or API.  See :ref:`Creating Notifications <CreatingNotifications>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
 
-**Make note of the password used as this will be used during first login to the Lemur UI**
-
-.. code-block:: bash
-
-    $ lemur db init
+**Make note of the password used as this will be used during first login to the Lemur UI.**
 
 .. code-block:: bash
 
+    $ cd /www/lemur/lemur
     $ lemur init
 
-.. note:: It is recommended that once the 'lemur' user is created that you create individual users for every day access.
-    There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account
-    for them or be enrolled automatically through SSO. This can be done through the CLI or UI.
-    See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details
+
+.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.
+
 
 Setup a Reverse Proxy
 ---------------------
 
-By default, Lemur runs on port 5000. Even if you change this, under normal conditions you won't be able to bind to
-port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a
-simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
+By default, Lemur runs on port 8000.  Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx.
+
 
 Proxying with Nginx
 ~~~~~~~~~~~~~~~~~~~
 
-You'll use the builtin HttpProxyModule within Nginx to handle proxying
+You'll use the builtin ``HttpProxyModule`` within Nginx to handle proxying.  Edit the ``/etc/nginx/sites-available/default`` file according to the lines below
 
 ::
 
    location /api {
-        proxy_pass  http://127.0.0.1:5000;
+        proxy_pass  http://127.0.0.1:8000;
         proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
         proxy_redirect off;
         proxy_buffering off;
@@ -180,23 +178,29 @@ You'll use the builtin HttpProxyModule within Nginx to handle proxying
         proxy_set_header        X-Real-IP       $remote_addr;
         proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
     }
-    
+
     location / {
         root /www/lemur/lemur/static/dist;
         include mime.types;
         index index.html;
     }
 
-See :doc:`../production/index` for more details on using Nginx.
+.. note:: See :doc:`../production/index` for more details on using Nginx.
+
+After making these changes, restart Nginx service to apply them:
+
+.. code-block:: bash
+
+    $ sudo service nginx restart
 
 
 Starting the Web Service
 ------------------------
 
-Lemur provides a built-in webserver (powered by gunicorn and eventlet) to get you off the ground quickly.
+Lemur provides a built-in web server (powered by gunicorn and eventlet) to get you off the ground quickly.
 
-To start the webserver, you simply use ``lemur start``. If you opted to use an alternative configuration path
-you can pass that via the --config option.
+To start the web server, you simply use ``lemur start``. If you opted to use an alternative configuration path
+you can pass that via the ``--config`` option.
 
 .. note::
     You can login with the default user created during :ref:`Initializing Lemur <InitializingLemur>` or any other
@@ -204,23 +208,23 @@ you can pass that via the --config option.
 
 ::
 
-  # Lemur's server runs on port 5000 by default. Make sure your client reflects
+  # Lemur's server runs on port 8000 by default. Make sure your client reflects
   # the correct host and port!
-  lemur --config=/etc/lemur.conf.py start -b 127.0.0.1:5000
+  lemur --config=/etc/lemur.conf.py start -b 127.0.0.1:8000
+
+You should now be able to test the web service by visiting ``http://localhost:8000/``.
 
-You should now be able to test the web service by visiting `http://localhost:5000/`.
 
 Running Lemur as a Service
----------------------------
+--------------------------
+
+We recommend using whatever software you are most familiar with for managing Lemur processes.  One option is `Supervisor <http://supervisord.org/>`_.
 
-We recommend using whatever software you are most familiar with for managing Lemur processes. One option is
-`Supervisor <http://supervisord.org/>`_.
 
 Configure ``supervisord``
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Configuring Supervisor couldn't be more simple. Just point it to the ``lemur`` executable in your virtualenv's bin/
-folder and you're good to go.
+Configuring Supervisor couldn't be more simple. Just point it to the ``lemur`` executable in your virtualenv's ``bin/`` folder and you're good to go.
 
 ::
 
@@ -235,44 +239,43 @@ folder and you're good to go.
 
 See :ref:`Using Supervisor <UsingSupervisor>` for more details on using Supervisor.
 
+
 Syncing
 -------
 
-Lemur uses periodic sync tasks to make sure it is up-to-date with its environment. As always things can change outside
-of Lemur, but we do our best to reconcile those changes.
+Lemur uses periodic sync tasks to make sure it is up-to-date with its environment. Things change outside of Lemur we do our best to reconcile those changes. The recommended method is to use CRON:
 
 .. code-block:: bash
 
   $ crontab -e
-  * 3 * * * lemur sync --all
-  * 3 * * * lemur check_revoked
+  */15 * * * * lemur sync -s all
+  0 22 * * * lemur check_revoked
+  0 22 * * * lemur notify
+
 
 Additional Utilities
 --------------------
 
-If you're familiar with Python you'll quickly find yourself at home, and even more so if you've used Flask. The
-``lemur`` command is just a simple wrapper around Flask's ``manage.py``, which means you get all of the
-power and flexibility that goes with it.
+If you're familiar with Python you'll quickly find yourself at home, and even more so if you've used Flask.  The ``lemur`` command is just a simple wrapper around Flask's ``manage.py``, which means you get all of the power and flexibility that goes with it.
+
+Some of the features which you'll likely find useful are listed below.
 
-Some of the features which you'll likely find useful are:
 
 lock
 ~~~~
 
-Encrypts sensitive key material - This is most useful for storing encrypted secrets in source code.
+Encrypts sensitive key material - this is most useful for storing encrypted secrets in source code.
+
 
 unlock
 ~~~~~~
 
-Decrypts sensitive key material - Used to decrypt the secrets stored in source during deployment.
+Decrypts sensitive key material - used to decrypt the secrets stored in source during deployment.
 
 
 What's Next?
 ------------
 
-Get familiar with how Lemur works by reviewing the :doc:`../guide/index`. When you're ready
-see :doc:`../production/index` for more details on how to configure Lemur for production.
-
-The above just gets you going, but for production there are several different security considerations to take into account.
-Remember, Lemur is handling sensitive data and security is imperative.
+Get familiar with how Lemur works by reviewing the :doc:`../guide/index`. When you're ready see :doc:`../production/index` for more details on how to configure Lemur for production.
 
+The above just gets you going, but for production there are several different security considerations to take into account.  Remember, Lemur is handling sensitive data and security is imperative.

docs/requirements.txt

@@ -1,29 +1,54 @@
-Jinja2>=2.3
-Pygments>=1.2
-Sphinx>=1.3
-docutils>=0.7
-markupsafe
-sphinxcontrib-httpdomain
+alabaster==0.7.8
+alembic==0.8.6
+aniso8601==1.1.0
+arrow==0.7.0
+Babel==2.3.4
+bcrypt==2.0.0
+beautifulsoup4==4.4.1
+blinker==1.4
+boto==2.38.0
+cffi==1.7.0
+cryptography==1.3.2
+docutils==0.12
+enum34==1.1.6
 Flask==0.10.1
-Flask-RESTful==0.3.3
-Flask-SQLAlchemy==2.0
-Flask-Script==2.0.5
-Flask-Migrate==1.4.0
-Flask-Bcrypt==0.6.2
-Flask-Principal==0.4.0
+Flask-Bcrypt==0.7.1
 Flask-Mail==0.9.1
-SQLAlchemy-Utils==0.30.11
-BeautifulSoup4
-requests==2.7.0
+Flask-Migrate==1.7.0
+Flask-Principal==0.4.0
+Flask-RESTful==0.3.3
+Flask-Script==2.0.5
+Flask-SQLAlchemy==2.1
+future==0.15.2
+gunicorn==19.4.1
+idna==2.1
+imagesize==0.7.1
+inflection==0.3.1
+ipaddress==1.0.16
+itsdangerous==0.24
+Jinja2==2.8
+lockfile==0.12.2
+Mako==1.0.4
+MarkupSafe==0.23
+marshmallow==2.4.0
+marshmallow-sqlalchemy==0.8.0
 psycopg2==2.6.1
-arrow==0.5.4
-boto==2.38.0  # we might make this optional
-six==1.9.0
-gunicorn==19.3.0
+pyasn1==0.1.9
+pycparser==2.14
 pycrypto==2.6.1
-cryptography==1.0.1
-pyopenssl==0.15.1
-pyjwt==1.0.1
+Pygments==2.1.3
+PyJWT==1.4.0
+pyOpenSSL==0.15.1
+python-dateutil==2.5.3
+python-editor==1.0.1
+pytz==2016.4
+requests==2.9.1
+six==1.10.0
+snowballstemmer==1.2.1
+Sphinx==1.4.4
+sphinx-rtd-theme==0.1.9
+sphinxcontrib-httpdomain==1.5.0
+SQLAlchemy==1.0.13
+SQLAlchemy-Utils==0.31.4
+Werkzeug==0.11.10
 xmltodict==0.9.2
-lockfile==0.10.2
-future==0.15.0

docs/security.rst

@@ -44,7 +44,7 @@ containing:
   new releases, and publicly disclose the issue.
 
 Simultaneously, the reporter of the issue will receive notification of the date
-on which we plan to take the issue public.
+on which we plan to make the issue public.
 
 On the day of disclosure, we will take the following steps:
 
@@ -59,7 +59,7 @@ known exploit in the wild, for example – the time between advance notification
 and public disclosure may be shortened considerably.
 
 The list of people and organizations who receives advanced notification of
-security issues is not and will not be made public. This list generally
+security issues is not, and will not, be made public. This list generally
 consists of high profile downstream distributors and is entirely at the
 discretion of the ``lemur`` team.
 

gulp/build.js

@@ -79,8 +79,9 @@ gulp.task('dev:styles', function () {
     'bower_components/angular-loading-bar/src/loading-bar.css',
     'bower_components/angular-ui-switch/angular-ui-switch.css',
     'bower_components/angular-wizard/dist/angular-wizard.css',
-    'bower_components/ng-table/ng-table.css',
+    'bower_components/ng-table/dist/ng-table.css',
     'bower_components/angularjs-toaster/toaster.css',
+    'bower_components/angular-ui-select/dist/select.css',
     'lemur/static/app/styles/lemur.css'
   ];
 
@@ -238,8 +239,8 @@ gulp.task('build:images', function () {
 
 gulp.task('package:strip', function () {
   return gulp.src(['lemur/static/dist/scripts/main*'])
-    .pipe(replace('http:\/\/localhost:5000', ''))
     .pipe(replace('http:\/\/localhost:3000', ''))
+    .pipe(replace('http:\/\/localhost:8000', ''))
     .pipe(useref())
     .pipe(revReplace())
     .pipe(gulp.dest('lemur/static/dist/scripts'))

hooks/pre-commit

@@ -17,13 +17,12 @@ if 'VIRTUAL_ENV' in os.environ:
 
 
 def py_lint(files_modified):
-    from flake8.main import DEFAULT_CONFIG
     from flake8.engine import get_style_guide
 
     # remove non-py files and files which no longer exist
     files_modified = filter(lambda x: x.endswith('.py'), files_modified)
 
-    flake8_style = get_style_guide(parse_argv=True, config_file=DEFAULT_CONFIG)
+    flake8_style = get_style_guide(parse_argv=True)
     report = flake8_style.check_files(files_modified)
 
     return report.total_errors != 0

lemur/__about__.py

@@ -0,0 +1,18 @@
+from __future__ import absolute_import, division, print_function
+
+__all__ = [
+    "__title__", "__summary__", "__uri__", "__version__", "__author__",
+    "__email__", "__license__", "__copyright__",
+]
+
+__title__ = "lemur"
+__summary__ = ("Certificate management and orchestration service")
+__uri__ = "https://github.com/Netflix/lemur"
+
+__version__ = "0.4.0"
+
+__author__ = "The Lemur developers"
+__email__ = "security@netflix.com"
+
+__license__ = "Apache License, Version 2.0"
+__copyright__ = "Copyright 2015 {0}".format(__author__)

lemur/__init__.py

@@ -8,7 +8,10 @@
 
 
 """
+from __future__ import absolute_import, division, print_function
+
 from lemur import factory
+from lemur.extensions import metrics
 
 from lemur.users.views import mod as users_bp
 from lemur.roles.views import mod as roles_bp
@@ -21,7 +24,18 @@ from lemur.defaults.views import mod as defaults_bp
 from lemur.plugins.views import mod as plugins_bp
 from lemur.notifications.views import mod as notifications_bp
 from lemur.sources.views import mod as sources_bp
+from lemur.endpoints.views import mod as endpoints_bp
 
+from lemur.__about__ import (
+    __author__, __copyright__, __email__, __license__, __summary__, __title__,
+    __uri__, __version__
+)
+
+
+__all__ = [
+    "__title__", "__summary__", "__uri__", "__version__", "__author__",
+    "__email__", "__license__", "__copyright__",
+]
 
 LEMUR_BLUEPRINTS = (
     users_bp,
@@ -34,7 +48,8 @@ LEMUR_BLUEPRINTS = (
     defaults_bp,
     plugins_bp,
     notifications_bp,
-    sources_bp
+    sources_bp,
+    endpoints_bp
 )
 
 
@@ -50,7 +65,8 @@ def configure_hook(app):
     :param app:
     :return:
     """
-    from flask.ext.principal import PermissionDenied
+    from flask import jsonify
+    from werkzeug.exceptions import default_exceptions
     from lemur.decorators import crossdomain
     if app.config.get('CORS'):
         @app.after_request
@@ -58,8 +74,13 @@ def configure_hook(app):
         def after(response):
             return response
 
-    @app.errorhandler(PermissionDenied)
-    def handle_invalid_usage(error):
-        response = {'message': 'You are not allow to access this resource'}
-        response.status_code = 403
-        return response
+    def make_json_handler(code):
+        def json_handler(error):
+            metrics.send('{}_status_code'.format(code), 'counter', 1)
+            response = jsonify(message=str(error))
+            response.status_code = code
+            return response
+        return json_handler
+
+    for code, value in default_exceptions.items():
+        app.error_handler_spec[None][code] = make_json_handler(code)

lemur/auth/permissions.py

@@ -18,27 +18,32 @@ admin_permission = Permission(RoleNeed('admin'))
 CertificateCreator = namedtuple('certificate', ['method', 'value'])
 CertificateCreatorNeed = partial(CertificateCreator, 'key')
 
-
-class ViewKeyPermission(Permission):
-    def __init__(self, certificate_id, owner):
-        c_need = CertificateCreatorNeed(certificate_id)
-        super(ViewKeyPermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
+CertificateOwner = namedtuple('certificate', ['method', 'value'])
+CertificateOwnerNeed = partial(CertificateOwner, 'role')
 
 
-class UpdateCertificatePermission(Permission):
-    def __init__(self, certificate_id, owner):
-        c_need = CertificateCreatorNeed(certificate_id)
-        super(UpdateCertificatePermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
+class SensitiveDomainPermission(Permission):
+    def __init__(self):
+        super(SensitiveDomainPermission, self).__init__(RoleNeed('admin'))
 
 
-RoleUser = namedtuple('role', ['method', 'value'])
-ViewRoleCredentialsNeed = partial(RoleUser, 'roleView')
+class CertificatePermission(Permission):
+    def __init__(self, certificate_id, owner, roles):
+        needs = [RoleNeed('admin'), CertificateCreatorNeed(certificate_id), RoleNeed(owner)]
+        for r in roles:
+            needs.append(CertificateOwnerNeed(str(r)))
+
+        super(CertificatePermission, self).__init__(*needs)
 
 
-class ViewRoleCredentialsPermission(Permission):
+RoleMember = namedtuple('role', ['method', 'value'])
+RoleMemberNeed = partial(RoleMember, 'member')
+
+
+class RoleMemberPermission(Permission):
     def __init__(self, role_id):
-        need = ViewRoleCredentialsNeed(role_id)
-        super(ViewRoleCredentialsPermission, self).__init__(need, RoleNeed('admin'))
+        needs = [RoleNeed('admin'), RoleMemberNeed(role_id)]
+        super(RoleMemberPermission, self).__init__(*needs)
 
 
 AuthorityCreator = namedtuple('authority', ['method', 'value'])

lemur/auth/service.py

@@ -8,11 +8,9 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from __future__ import unicode_literals
-from builtins import bytes
+import sys
 import jwt
 import json
-import base64
 import binascii
 
 from functools import wraps
@@ -31,20 +29,7 @@ from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicNumbers
 
 from lemur.users import service as user_service
 from lemur.auth.permissions import CertificateCreatorNeed, \
-    AuthorityCreatorNeed, ViewRoleCredentialsNeed
-
-
-def base64url_decode(data):
-    rem = len(data) % 4
-
-    if rem > 0:
-        data += '=' * (4 - rem)
-
-    return base64.urlsafe_b64decode(bytes(data.encode('latin-1')))
-
-
-def base64url_encode(data):
-    return base64.urlsafe_b64encode(data).replace('=', '')
+    AuthorityCreatorNeed, RoleMemberNeed
 
 
 def get_rsa_public_key(n, e):
@@ -55,8 +40,13 @@ def get_rsa_public_key(n, e):
     :param e:
     :return: a RSA Public Key in PEM format
     """
-    n = int(binascii.hexlify(base64url_decode(n)), 16)
-    e = int(binascii.hexlify(base64url_decode(e)), 16)
+    if sys.version_info >= (3, 0):
+        n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, 'utf-8'))), 16)
+        e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, 'utf-8'))), 16)
+    else:
+        n = int(binascii.hexlify(jwt.utils.base64url_decode(str(n))), 16)
+        e = int(binascii.hexlify(jwt.utils.base64url_decode(str(e))), 16)
+
     pub = RSAPublicNumbers(e, n).public_key(default_backend())
     return pub.public_bytes(
         encoding=serialization.Encoding.PEM,
@@ -75,8 +65,8 @@ def create_token(user):
     expiration_delta = timedelta(days=int(current_app.config.get('LEMUR_TOKEN_EXPIRATION', 1)))
     payload = {
         'sub': user.id,
-        'iat': datetime.now(),
-        'exp': datetime.now() + expiration_delta
+        'iat': datetime.utcnow(),
+        'exp': datetime.utcnow() + expiration_delta
     }
     token = jwt.encode(payload, current_app.config['LEMUR_TOKEN_SECRET'])
     return token.decode('unicode_escape')
@@ -138,13 +128,13 @@ def fetch_token_header(token):
         raise jwt.DecodeError('Not enough segments')
 
     try:
-        return json.loads(base64url_decode(header_segment))
+        if sys.version_info >= (3, 0):
+            return json.loads(jwt.utils.base64url_decode(header_segment).decode('utf-8'))
+        else:
+            return json.loads(jwt.utils.base64url_decode(header_segment))
     except TypeError as e:
         current_app.logger.exception(e)
         raise jwt.DecodeError('Invalid header padding')
-    except binascii.Error as e:
-        current_app.logger.exception(e)
-        raise jwt.DecodeError('Invalid header padding')
 
 
 @identity_loaded.connect
@@ -165,8 +155,8 @@ def on_identity_loaded(sender, identity):
     # identity with the roles that the user provides
     if hasattr(user, 'roles'):
         for role in user.roles:
-            identity.provides.add(ViewRoleCredentialsNeed(role.id))
             identity.provides.add(RoleNeed(role.name))
+            identity.provides.add(RoleMemberNeed(role.id))
 
     # apply ownership for authorities
     if hasattr(user, 'authorities'):

lemur/auth/views.py

@@ -5,15 +5,17 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import sys
 import jwt
 import base64
 import requests
 
-from flask import g, Blueprint, current_app
+from flask import Blueprint, current_app
 
 from flask.ext.restful import reqparse, Resource, Api
 from flask.ext.principal import Identity, identity_changed
 
+from lemur.extensions import metrics
 from lemur.common.utils import get_psuedo_random_string
 
 from lemur.users import service as user_service
@@ -96,13 +98,13 @@ class Login(Resource):
             # Tell Flask-Principal the identity changed
             identity_changed.send(current_app._get_current_object(),
                                   identity=Identity(user.id))
+
+            metrics.send('successful_login', 'counter', 1)
             return dict(token=create_token(user))
 
+        metrics.send('invalid_login', 'counter', 1)
         return dict(message='The supplied credentials are invalid'), 401
 
-    def get(self):
-        return {'username': g.current_user.username, 'roles': [r.name for r in g.current_user.roles]}
-
 
 class Ping(Resource):
     """
@@ -139,8 +141,14 @@ class Ping(Resource):
         user_api_url = current_app.config.get('PING_USER_API_URL')
 
         # the secret and cliendId will be given to you when you signup for the provider
-        basic = base64.b64encode('{0}:{1}'.format(args['clientId'], current_app.config.get("PING_SECRET")))
-        headers = {'Authorization': 'Basic {0}'.format(basic)}
+        token = '{0}:{1}'.format(args['clientId'], current_app.config.get("PING_SECRET"))
+
+        if sys.version_info >= (3, 0):
+            basic = base64.b64encode(bytes(token, 'utf-8'))
+            headers = {'authorization': 'basic {0}'.format(basic.decode('utf-8'))}
+        else:
+            basic = base64.b64encode(token, 'utf-8')
+            headers = {'authorization': 'basic {0}'.format(basic)}
 
         # exchange authorization code for access token.
 
@@ -164,7 +172,10 @@ class Ping(Resource):
 
         # validate your token based on the key it was signed with
         try:
-            jwt.decode(id_token, secret, algorithms=[algo], audience=args['clientId'])
+            if sys.version_info >= (3, 0):
+                jwt.decode(id_token, secret.decode('utf-8'), algorithms=[algo], audience=args['clientId'])
+            else:
+                jwt.decode(id_token, secret, algorithms=[algo], audience=args['clientId'])
         except jwt.DecodeError:
             return dict(message='Token is invalid'), 403
         except jwt.ExpiredSignatureError:
@@ -179,6 +190,7 @@ class Ping(Resource):
         profile = r.json()
 
         user = user_service.get_by_email(profile['email'])
+        metrics.send('successful_login', 'counter', 1)
 
         # update their google 'roles'
         roles = []
@@ -189,10 +201,13 @@ class Ping(Resource):
                 role = role_service.create(group, description='This is a google group based role created by Lemur')
             roles.append(role)
 
-        # if we get an sso user create them an account
-        # we still pick a random password in case sso is down
-        if not user:
+        role = role_service.get_by_name(profile['email'])
+        if not role:
+            role = role_service.create(profile['email'], description='This is a user specific role')
+        roles.append(role)
 
+        # if we get an sso user create them an account
+        if not user:
             # every user is an operator (tied to a default role)
             if current_app.config.get('LEMUR_DEFAULT_ROLE'):
                 v = role_service.get_by_name(current_app.config.get('LEMUR_DEFAULT_ROLE'))
@@ -230,5 +245,78 @@ class Ping(Resource):
         return dict(token=create_token(user))
 
 
+class Google(Resource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(Google, self).__init__()
+
+    def post(self):
+        access_token_url = 'https://accounts.google.com/o/oauth2/token'
+        people_api_url = 'https://www.googleapis.com/plus/v1/people/me/openIdConnect'
+
+        self.reqparse.add_argument('clientId', type=str, required=True, location='json')
+        self.reqparse.add_argument('redirectUri', type=str, required=True, location='json')
+        self.reqparse.add_argument('code', type=str, required=True, location='json')
+
+        args = self.reqparse.parse_args()
+
+        # Step 1. Exchange authorization code for access token
+        payload = {
+            'client_id': args['clientId'],
+            'grant_type': 'authorization_code',
+            'redirect_uri': args['redirectUri'],
+            'code': args['code'],
+            'client_secret': current_app.config.get('GOOGLE_SECRET')
+        }
+
+        r = requests.post(access_token_url, data=payload)
+        token = r.json()
+
+        # Step 2. Retrieve information about the current user
+        headers = {'Authorization': 'Bearer {0}'.format(token['access_token'])}
+
+        r = requests.get(people_api_url, headers=headers)
+        profile = r.json()
+
+        user = user_service.get_by_email(profile['email'])
+
+        if user:
+            metrics.send('successful_login', 'counter', 1)
+            return dict(token=create_token(user))
+
+
+class Providers(Resource):
+    def get(self):
+        active_providers = []
+
+        for provider in current_app.config.get("ACTIVE_PROVIDERS", []):
+            provider = provider.lower()
+
+            if provider == "google":
+                active_providers.append({
+                    'name': 'google',
+                    'clientId': current_app.config.get("GOOGLE_CLIENT_ID"),
+                    'url': api.url_for(Google)
+                })
+
+            elif provider == "ping":
+                active_providers.append({
+                    'name': current_app.config.get("PING_NAME"),
+                    'url': current_app.config.get('PING_REDIRECT_URI'),
+                    'redirectUri': current_app.config.get("PING_REDIRECT_URI"),
+                    'clientId': current_app.config.get("PING_CLIENT_ID"),
+                    'responseType': 'code',
+                    'scope': ['openid', 'email', 'profile', 'address'],
+                    'scopeDelimiter': ' ',
+                    'authorizationEndpoint': current_app.config.get("PING_AUTH_ENDPOINT"),
+                    'requiredUrlParams': ['scope'],
+                    'type': '2.0'
+                })
+
+        return active_providers
+
+
 api.add_resource(Login, '/auth/login', endpoint='login')
 api.add_resource(Ping, '/auth/ping', endpoint='ping')
+api.add_resource(Google, '/auth/google', endpoint='google')
+api.add_resource(Providers, '/auth/providers', endpoint='providers')

lemur/authorities/models.py

@@ -6,53 +6,43 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
-
 from sqlalchemy.orm import relationship
 from sqlalchemy import Column, Integer, String, Text, func, ForeignKey, DateTime, PassiveDefault, Boolean
 from sqlalchemy.dialects.postgresql import JSON
 
 from lemur.database import db
-from lemur.certificates.models import get_cn, get_not_after, get_not_before
+from lemur.plugins.base import plugins
+from lemur.models import roles_authorities
 
 
 class Authority(db.Model):
     __tablename__ = 'authorities'
     id = Column(Integer, primary_key=True)
-    owner = Column(String(128))
+    owner = Column(String(128), nullable=False)
     name = Column(String(128), unique=True)
     body = Column(Text())
     chain = Column(Text())
-    bits = Column(Integer())
-    cn = Column(String(128))
-    not_before = Column(DateTime)
-    not_after = Column(DateTime)
     active = Column(Boolean, default=True)
-    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
     plugin_name = Column(String(64))
     description = Column(Text)
     options = Column(JSON)
-    roles = relationship('Role', backref=db.backref('authority'), lazy='dynamic')
+    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
+    roles = relationship('Role', secondary=roles_authorities, passive_deletes=True, backref=db.backref('authority'), lazy='dynamic')
     user_id = Column(Integer, ForeignKey('users.id'))
-    certificates = relationship("Certificate", backref='authority')
+    authority_certificate = relationship("Certificate", backref='root_authority', uselist=False, foreign_keys='Certificate.root_authority_id')
+    certificates = relationship("Certificate", backref='authority', foreign_keys='Certificate.authority_id')
 
-    def __init__(self, name, owner, plugin_name, body, roles=None, chain=None, description=None):
-        self.name = name
-        self.body = body
-        self.chain = chain
-        self.owner = owner
-        self.plugin_name = plugin_name
-        cert = x509.load_pem_x509_certificate(str(body), default_backend())
-        self.cn = get_cn(cert)
-        self.not_before = get_not_before(cert)
-        self.not_after = get_not_after(cert)
-        self.roles = roles
-        self.description = description
+    def __init__(self, **kwargs):
+        self.owner = kwargs['owner']
+        self.roles = kwargs.get('roles', [])
+        self.name = kwargs.get('name')
+        self.description = kwargs.get('description')
+        self.authority_certificate = kwargs['authority_certificate']
+        self.plugin_name = kwargs['plugin']['slug']
 
-    def as_dict(self):
-        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+    @property
+    def plugin(self):
+        return plugins.get(self.plugin_name)
 
-    def serialize(self):
-        blob = self.as_dict()
-        return blob
+    def __repr__(self):
+        return "Authority(name={name})".format(name=self.name)

lemur/authorities/schemas.py

@@ -0,0 +1,119 @@
+"""
+.. module: lemur.authorities.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import current_app
+
+from marshmallow import fields, validates_schema, pre_load
+from marshmallow import validate
+from marshmallow.exceptions import ValidationError
+
+from lemur.schemas import PluginInputSchema, PluginOutputSchema, ExtensionSchema, AssociatedAuthoritySchema, AssociatedRoleSchema
+from lemur.users.schemas import UserNestedOutputSchema
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.common import validators, missing
+
+from lemur.common.fields import ArrowDateTime
+
+
+class AuthorityInputSchema(LemurInputSchema):
+    name = fields.String(required=True)
+    owner = fields.Email(required=True)
+    description = fields.String()
+    common_name = fields.String(required=True, validate=validators.sensitive_domain)
+
+    validity_start = ArrowDateTime()
+    validity_end = ArrowDateTime()
+    validity_years = fields.Integer()
+
+    # certificate body fields
+    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
+    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
+    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
+    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
+    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+
+    plugin = fields.Nested(PluginInputSchema)
+
+    # signing related options
+    type = fields.String(validate=validate.OneOf(['root', 'subca']), missing='root')
+    parent = fields.Nested(AssociatedAuthoritySchema)
+    signing_algorithm = fields.String(validate=validate.OneOf(['sha256WithRSA', 'sha1WithRSA']), missing='sha256WithRSA')
+    key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')
+    key_name = fields.String()
+    sensitivity = fields.String(validate=validate.OneOf(['medium', 'high']), missing='medium')
+    serial_number = fields.Integer()
+    first_serial = fields.Integer(missing=1)
+
+    extensions = fields.Nested(ExtensionSchema)
+
+    roles = fields.Nested(AssociatedRoleSchema(many=True))
+
+    @validates_schema
+    def validate_dates(self, data):
+        validators.dates(data)
+
+    @validates_schema
+    def validate_subca(self, data):
+        if data['type'] == 'subca':
+            if not data.get('parent'):
+                raise ValidationError("If generating a subca parent 'authority' must be specified.")
+
+    @pre_load
+    def ensure_dates(self, data):
+        return missing.convert_validity_years(data)
+
+
+class AuthorityUpdateSchema(LemurInputSchema):
+    owner = fields.Email(required=True)
+    description = fields.String()
+    active = fields.Boolean()
+    roles = fields.Nested(AssociatedRoleSchema(many=True))
+
+
+class RootAuthorityCertificateOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    active = fields.Boolean()
+    bits = fields.Integer()
+    body = fields.String()
+    chain = fields.String()
+    description = fields.String()
+    name = fields.String()
+    cn = fields.String()
+    not_after = fields.DateTime()
+    not_before = fields.DateTime()
+    owner = fields.Email()
+    status = fields.Boolean()
+    user = fields.Nested(UserNestedOutputSchema)
+
+
+class AuthorityOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    plugin = fields.Nested(PluginOutputSchema)
+    active = fields.Boolean()
+    options = fields.Dict()
+    roles = fields.List(fields.Nested(AssociatedRoleSchema))
+    authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema)
+
+
+class AuthorityNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    plugin = fields.Nested(PluginOutputSchema)
+    active = fields.Boolean()
+
+
+authority_update_schema = AuthorityUpdateSchema()
+authority_input_schema = AuthorityInputSchema()
+authority_output_schema = AuthorityOutputSchema()
+authorities_output_schema = AuthorityOutputSchema(many=True)

lemur/authorities/service.py

@@ -9,17 +9,13 @@
 
 """
 from flask import g
-from flask import current_app
 
 from lemur import database
+from lemur.extensions import metrics
 from lemur.authorities.models import Authority
 from lemur.roles import service as role_service
-from lemur.notifications import service as notification_service
 
-from lemur.roles.models import Role
-from lemur.certificates.models import Certificate
-
-from lemur.plugins.base import plugins
+from lemur.certificates.service import upload
 
 
 def update(authority_id, description=None, owner=None, active=None, roles=None):
@@ -28,12 +24,12 @@ def update(authority_id, description=None, owner=None, active=None, roles=None):
 
     :param authority_id:
     :param roles: roles that are allowed to use this authority
-    :rtype : Authority
     :return:
     """
     authority = get(authority_id)
+
     if roles:
-        authority = database.update_list(authority, 'roles', Role, roles)
+        authority.roles = roles
 
     if active:
         authority.active = active
@@ -43,38 +39,39 @@ def update(authority_id, description=None, owner=None, active=None, roles=None):
     return database.update(authority)
 
 
-def create(kwargs):
+def mint(**kwargs):
     """
-    Create a new authority.
+    Creates the authority based on the plugin provided.
+    """
+    issuer = kwargs['plugin']['plugin_object']
+    values = issuer.create_authority(kwargs)
 
-    :rtype : Authority
+    # support older plugins
+    if len(values) == 3:
+        body, chain, roles = values
+        private_key = None
+    elif len(values) == 4:
+        body, private_key, chain, roles = values
+
+    roles = create_authority_roles(roles, kwargs['owner'], kwargs['plugin']['plugin_object'].title)
+    return body, private_key, chain, roles
+
+
+def create_authority_roles(roles, owner, plugin_title):
+    """
+    Creates all of the necessary authority roles.
+    :param roles:
     :return:
     """
-
-    issuer = plugins.get(kwargs.get('pluginName'))
-
-    kwargs['creator'] = g.current_user.email
-    cert_body, intermediate, issuer_roles = issuer.create_authority(kwargs)
-
-    cert = Certificate(cert_body, chain=intermediate)
-    cert.owner = kwargs['ownerEmail']
-    cert.description = "This is the ROOT certificate for the {0} certificate authority".format(kwargs.get('caName'))
-    cert.user = g.current_user
-
-    cert.notifications = notification_service.create_default_expiration_notifications(
-        'DEFAULT_SECURITY',
-        current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
-    )
-
-    # we create and attach any roles that the issuer gives us
     role_objs = []
-    for r in issuer_roles:
-
-        role = role_service.create(
-            r['name'],
-            password=r['password'],
-            description="{0} auto generated role".format(kwargs.get('pluginName')),
-            username=r['username'])
+    for r in roles:
+        role = role_service.get_by_name(r['name'])
+        if not role:
+            role = role_service.create(
+                r['name'],
+                password=r['password'],
+                description="Auto generated role for {0}".format(plugin_title),
+                username=r['username'])
 
         # the user creating the authority should be able to administer it
         if role.username == 'admin':
@@ -82,21 +79,44 @@ def create(kwargs):
 
         role_objs.append(role)
 
-    authority = Authority(
-        kwargs.get('caName'),
-        kwargs['ownerEmail'],
-        kwargs['pluginName'],
-        cert_body,
-        description=kwargs['caDescription'],
-        chain=intermediate,
-        roles=role_objs
-    )
+    # create an role for the owner and assign it
+    owner_role = role_service.get_by_name(owner)
+    if not owner_role:
+        owner_role = role_service.create(
+            owner,
+            description="Auto generated role based on owner: {0}".format(owner)
+        )
 
-    database.update(cert)
+    role_objs.append(owner_role)
+    return role_objs
+
+
+def create(**kwargs):
+    """
+    Creates a new authority.
+    """
+    kwargs['creator'] = g.user.email
+    body, private_key, chain, roles = mint(**kwargs)
+
+    g.user.roles = list(set(list(g.user.roles) + roles))
+
+    kwargs['body'] = body
+    kwargs['private_key'] = private_key
+    kwargs['chain'] = chain
+
+    if kwargs.get('roles'):
+        kwargs['roles'] += roles
+    else:
+        kwargs['roles'] = roles
+
+    cert = upload(**kwargs)
+    kwargs['authority_certificate'] = cert
+
+    authority = Authority(**kwargs)
     authority = database.create(authority)
+    g.user.authorities.append(authority)
 
-    g.current_user.authorities.append(authority)
-
+    metrics.send('authority_created', 'counter', 1, metric_tags=dict(owner=authority.owner))
     return authority
 
 
@@ -115,7 +135,6 @@ def get(authority_id):
     """
     Retrieves an authority given it's ID
 
-    :rtype : Authority
     :param authority_id:
     :return:
     """
@@ -127,7 +146,6 @@ def get_by_name(authority_name):
     Retrieves an authority given it's name.
 
     :param authority_name:
-    :rtype : Authority
     :return:
     """
     return database.get(Authority, authority_name, field='name')
@@ -141,14 +159,9 @@ def get_authority_role(ca_name):
     :param ca_name:
     """
     if g.current_user.is_admin:
-        authority = get_by_name(ca_name)
-        # TODO we should pick admin ca roles for admin
-        return authority.roles[0]
+        return role_service.get_by_name("{0}_admin".format(ca_name))
     else:
-        for role in g.current_user.roles:
-            if role.authority:
-                if role.authority.name == ca_name:
-                    return role
+        return role_service.get_by_name("{0}_operator".format(ca_name))
 
 
 def render(args):
@@ -158,10 +171,6 @@ def render(args):
     :return:
     """
     query = database.session_query(Authority)
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
 
     if filt:
@@ -174,14 +183,12 @@ def render(args):
     # we make sure that a user can only use an authority they either own are are a member of - admins can see all
     if not g.current_user.is_admin:
         authority_ids = []
+        for authority in g.current_user.authorities:
+            authority_ids.append(authority.id)
+
         for role in g.current_user.roles:
-            if role.authority:
-                authority_ids.append(role.authority.id)
+            for authority in role.authorities:
+                authority_ids.append(authority.id)
         query = query.filter(Authority.id.in_(authority_ids))
 
-    query = database.find_all(query, Authority, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Authority, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)
+    return database.sort_and_page(query, Authority, args)

lemur/authorities/views.py

@@ -5,32 +5,19 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from flask import Blueprint, g
-from flask.ext.restful import reqparse, fields, Api
+from flask import Blueprint
+from flask.ext.restful import reqparse, Api
 
-from lemur.authorities import service
-from lemur.roles import service as role_service
-from lemur.certificates import service as certificate_service
+from lemur.common.utils import paginated_parser
+from lemur.common.schema import validate_schema
 from lemur.auth.service import AuthenticatedResource
-
 from lemur.auth.permissions import AuthorityPermission
 
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.certificates import service as certificate_service
 
+from lemur.authorities import service
+from lemur.authorities.schemas import authority_input_schema, authority_output_schema, authorities_output_schema, authority_update_schema
 
-FIELDS = {
-    'name': fields.String,
-    'owner': fields.String,
-    'description': fields.String,
-    'options': fields.Raw,
-    'pluginName': fields.String,
-    'body': fields.String,
-    'chain': fields.String,
-    'active': fields.Boolean,
-    'notBefore': fields.DateTime(dt_format='iso8601', attribute='not_before'),
-    'notAfter': fields.DateTime(dt_format='iso8601', attribute='not_after'),
-    'id': fields.Integer,
-}
 
 mod = Blueprint('authorities', __name__)
 api = Api(mod)
@@ -42,7 +29,7 @@ class AuthoritiesList(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(AuthoritiesList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, authorities_output_schema)
     def get(self):
         """
         .. http:get:: /authorities
@@ -66,28 +53,52 @@ class AuthoritiesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
-                    {
-                      "id": 1,
-                      "name": "authority1",
-                      "description": "this is authority1",
-                      "pluginName": null,
-                      "chain": "-----Begin ...",
-                      "body": "-----Begin ...",
-                      "active": true,
-                      "notBefore": "2015-06-05T17:09:39",
-                      "notAfter": "2015-06-10T17:09:39"
-                      "options": null
-                    }
-                  ]
+                "items": [{
+                    "name": "TestAuthority",
+                    "roles": [{
+                        "id": 123,
+                        "name": "secure@example.com"
+                    }, {
+                        "id": 564,
+                        "name": "TestAuthority_admin"
+                    }, {
+                        "id": 565,
+                        "name": "TestAuthority_operator"
+                    }],
+                    "options": null,
+                    "active": true,
+                    "authorityCertificate": {
+                        "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
+                        "status": true,
+                        "cn": "AcommonName",
+                        "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                        "chain": "",
+                        "notBefore": "2016-06-02T00:00:15+00:00",
+                        "notAfter": "2023-06-02T23:59:15+00:00",
+                        "owner": "secure@example.com",
+                        "user": {
+                            "username": "joe@example.com",
+                            "active": true,
+                            "email": "joe@example.com",
+                            "id": 3
+                        },
+                        "active": true,
+                        "bits": 2048,
+                        "id": 2235,
+                        "name": "TestAuthority"
+                    },
+                    "owner": "secure@example.com",
+                    "id": 43,
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority."
+                }
                 "total": 1
               }
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
-           :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query page: int default is 1
+           :query filter: key value pair. format is k;v
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
@@ -98,8 +109,8 @@ class AuthoritiesList(AuthenticatedResource):
         args = parser.parse_args()
         return service.render(args)
 
-    @marshal_items(FIELDS)
-    def post(self):
+    @validate_schema(authority_input_schema, authority_output_schema)
+    def post(self, data=None):
         """
         .. http:post:: /authorities
 
@@ -113,31 +124,30 @@ class AuthoritiesList(AuthenticatedResource):
               Host: example.com
               Accept: application/json, text/javascript
 
-              {
-                "caDN": {
-                  "country": "US",
-                  "state": "CA",
-                  "location": "A Location",
-                  "organization": "ExampleInc",
-                  "organizationalUnit": "Operations",
-                  "commonName": "a common name"
-                },
-                "caType": "root",
-                "caSigningAlgo": "sha256WithRSA",
-                "caSensitivity": "medium",
+             {
+                "country": "US",
+                "state": "California",
+                "location": "Los Gatos",
+                "organization": "Netflix",
+                "organizationalUnit": "Operations",
+                "type": "root",
+                "signingAlgorithm": "sha256WithRSA",
+                "sensitivity": "medium",
                 "keyType": "RSA2048",
-                "pluginName": "cloudca",
-                "validityStart": "2015-06-11T07:00:00.000Z",
-                "validityEnd": "2015-06-13T07:00:00.000Z",
-                "caName": "DoctestCA",
-                "ownerEmail": "jimbob@example.com",
-                "caDescription": "Example CA",
-                "extensions": {
-                  "subAltNames": {
-                    "names": []
-                  }
+                "plugin": {
+                    "slug": "cloudca-issuer",
                 },
-              }
+                "name": "TimeTestAuthority5",
+                "owner": "secure@example.com",
+                "description": "test",
+                "commonName": "AcommonName",
+                "validityYears": "20",
+                "extensions": {
+                    "subAltNames": {
+                        "names": []
+                    },
+                    "custom": []
+             }
 
            **Example response**:
 
@@ -148,57 +158,67 @@ class AuthoritiesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
-                "name": "authority1",
-                "description": "this is authority1",
-                "pluginName": null,
-                "chain": "-----Begin ...",
-                "body": "-----Begin ...",
+                "name": "TestAuthority",
+                "roles": [{
+                    "id": 123,
+                    "name": "secure@example.com"
+                }, {
+                    "id": 564,
+                    "name": "TestAuthority_admin"
+                }, {
+                    "id": 565,
+                    "name": "TestAuthority_operator"
+                }],
+                "options": null,
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
-                "notAfter": "2015-06-10T17:09:39"
-                "options": null
+                "authorityCertificate": {
+                    "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
+                    "status": true,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-02T00:00:15+00:00",
+                    "notAfter": "2023-06-02T23:59:15+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2235,
+                    "name": "TestAuthority"
+                },
+                "owner": "secure@example.com",
+                "id": 43,
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
-           :arg caName: authority's name
-           :arg caDescription: a sensible description about what the CA with be used for
-           :arg ownerEmail: the team or person who 'owns' this authority
+
+           :arg name: authority's name
+           :arg description: a sensible description about what the CA with be used for
+           :arg owner: the team or person who 'owns' this authority
            :arg validityStart: when this authority should start issuing certificates
            :arg validityEnd: when this authority should stop issuing certificates
+           :arg validityYears: starting from `now` how many years into the future the authority should be valid
            :arg extensions: certificate extensions
-           :arg pluginName: name of the plugin to create the authority
-           :arg caType: the type of authority (root/subca)
-           :arg caParent: the parent authority if this is to be a subca
-           :arg caSigningAlgo: algorithm used to sign the authority
+           :arg plugin: name of the plugin to create the authority
+           :arg type: the type of authority (root/subca)
+           :arg parent: the parent authority if this is to be a subca
+           :arg signingAlgorithm: algorithm used to sign the authority
            :arg keyType: key type
-           :arg caSensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored
+           :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored
            in an HSM
-           :arg caKeyName: name of the key to store in the HSM (CloudCA)
-           :arg caSerialNumber: serial number of the authority
-           :arg caFirstSerial: specifies the starting serial number for certificates issued off of this authority
+           :arg keyName: name of the key to store in the HSM (CloudCA)
+           :arg serialNumber: serial number of the authority
+           :arg firstSerial: specifies the starting serial number for certificates issued off of this authority
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 403: unauthenticated
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('caName', type=str, location='json', required=True)
-        self.reqparse.add_argument('caDescription', type=str, location='json', required=False)
-        self.reqparse.add_argument('ownerEmail', type=str, location='json', required=True)
-        self.reqparse.add_argument('caDN', type=dict, location='json', required=False)
-        self.reqparse.add_argument('validityStart', type=str, location='json', required=False)  # TODO validate
-        self.reqparse.add_argument('validityEnd', type=str, location='json', required=False)  # TODO validate
-        self.reqparse.add_argument('extensions', type=dict, location='json', required=False)
-        self.reqparse.add_argument('pluginName', type=str, location='json', required=True)
-        self.reqparse.add_argument('caType', type=str, location='json', required=False)
-        self.reqparse.add_argument('caParent', type=str, location='json', required=False)
-        self.reqparse.add_argument('caSigningAlgo', type=str, location='json', required=False)
-        self.reqparse.add_argument('keyType', type=str, location='json', required=False)
-        self.reqparse.add_argument('caSensitivity', type=str, location='json', required=False)
-        self.reqparse.add_argument('caKeyName', type=str, location='json', required=False)
-        self.reqparse.add_argument('caSerialNumber', type=int, location='json', required=False)
-        self.reqparse.add_argument('caFirstSerial', type=int, location='json', required=False)
-
-        args = self.reqparse.parse_args()
-        return service.create(args)
+        return service.create(**data)
 
 
 class Authorities(AuthenticatedResource):
@@ -206,7 +226,7 @@ class Authorities(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Authorities, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, authority_output_schema)
     def get(self, authority_id):
         """
         .. http:get:: /authorities/1
@@ -230,26 +250,36 @@ class Authorities(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
-                "name": "authority1",
-                "description": "this is authority1",
-                "pluginName": null,
-                "chain": "-----Begin ...",
-                "body": "-----Begin ...",
+                "roles": [{
+                    "id": 123,
+                    "name": "secure@example.com"
+                }, {
+                    "id": 564,
+                    "name": "TestAuthority_admin"
+                }, {
+                    "id": 565,
+                    "name": "TestAuthority_operator"
+                }],
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
-                "notAfter": "2015-06-10T17:09:39"
-                "options": null
+                "owner": "secure@example.com",
+                "id": 43,
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
+           :arg description: a sensible description about what the CA with be used for
+           :arg owner: the team or person who 'owns' this authority
+           :arg active: set whether this authoritity is currently in use
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 403: unauthenticated
+           :statuscode 200: no error
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
         return service.get(authority_id)
 
-    @marshal_items(FIELDS)
-    def put(self, authority_id):
+    @validate_schema(authority_update_schema, authority_output_schema)
+    def put(self, authority_id, data=None):
         """
         .. http:put:: /authorities/1
 
@@ -264,11 +294,42 @@ class Authorities(AuthenticatedResource):
               Accept: application/json, text/javascript
 
               {
-                 "roles": [],
-                 "active": false,
-                 "owner": "bob@example.com",
-                 "description": "this is authority1"
-              }
+                "name": "TestAuthority5",
+                "roles": [{
+                    "id": 566,
+                    "name": "TestAuthority5_admin"
+                }, {
+                    "id": 567,
+                    "name": "TestAuthority5_operator"
+                }, {
+                    "id": 123,
+                    "name": "secure@example.com"
+                }],
+                "active": true,
+                "authorityCertificate": {
+                    "body": "-----BEGIN CERTIFICATE-----",
+                    "status": null,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority5 certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-03T00:00:51+00:00",
+                    "notAfter": "2036-06-03T23:59:51+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2280,
+                    "name": "TestAuthority5"
+                },
+                "owner": "secure@example.com",
+                "id": 44,
+                "description": "This is the ROOT certificate for the TestAuthority5 certificate authority."
+               }
 
            **Example response**:
 
@@ -279,64 +340,74 @@ class Authorities(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
-                "name": "authority1",
-                "description": "this is authority1",
-                "pluginName": null,
-                "chain": "-----begin ...",
-                "body": "-----begin ...",
-                "active": false,
-                "notBefore": "2015-06-05t17:09:39",
-                "notAfter": "2015-06-10t17:09:39"
-                "options": null
+                "name": "TestAuthority",
+                "roles": [{
+                    "id": 123,
+                    "name": "secure@example.com"
+                }, {
+                    "id": 564,
+                    "name": "TestAuthority_admin"
+                }, {
+                    "id": 565,
+                    "name": "TestAuthority_operator"
+                }],
+                "options": null,
+                "active": true,
+                "authorityCertificate": {
+                    "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
+                    "status": true,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-02T00:00:15+00:00",
+                    "notAfter": "2023-06-02T23:59:15+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2235,
+                    "name": "TestAuthority"
+                },
+                "owner": "secure@example.com",
+                "id": 43,
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
-        self.reqparse.add_argument('roles', type=list, default=[], location='json')
-        self.reqparse.add_argument('active', type=str, location='json', required=True)
-        self.reqparse.add_argument('owner', type=str, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json', required=True)
-        args = self.reqparse.parse_args()
-
         authority = service.get(authority_id)
-        role = role_service.get_by_name(authority.owner)
+
+        if not authority:
+            return dict(message='Not Found'), 404
 
         # all the authority role members should be allowed
         roles = [x.name for x in authority.roles]
-
-        # allow "owner" roles by team DL
-        roles.append(role)
         permission = AuthorityPermission(authority_id, roles)
 
-        # we want to make sure that we cannot add roles that we are not members of
-        if not g.current_user.is_admin:
-            role_ids = set([r['id'] for r in args['roles']])
-            user_role_ids = set([r.id for r in g.current_user.roles])
-
-            if not role_ids.issubset(user_role_ids):
-                return dict(message="You are not allowed to associate a role which you are not a member of"), 400
-
         if permission.can():
             return service.update(
                 authority_id,
-                owner=args['owner'],
-                description=args['description'],
-                active=args['active'],
-                roles=args['roles']
+                owner=data['owner'],
+                description=data['description'],
+                active=data['active'],
+                roles=data['roles']
             )
 
-        return dict(message="You are not authorized to update this authority"), 403
+        return dict(message="You are not authorized to update this authority."), 403
 
 
 class CertificateAuthority(AuthenticatedResource):
     def __init__(self):
-        self.reqparse = reqparse.RequestParser()
         super(CertificateAuthority, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, authority_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/authority
@@ -360,16 +431,42 @@ class CertificateAuthority(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
-                "name": "authority1",
-                "description": "this is authority1",
-                "pluginName": null,
-                "chain": "-----Begin ...",
-                "body": "-----Begin ...",
+                "name": "TestAuthority",
+                "roles": [{
+                    "id": 123,
+                    "name": "secure@example.com"
+                }, {
+                    "id": 564,
+                    "name": "TestAuthority_admin"
+                }, {
+                    "id": 565,
+                    "name": "TestAuthority_operator"
+                }],
+                "options": null,
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
-                "notAfter": "2015-06-10T17:09:39"
-                "options": null
+                "authorityCertificate": {
+                    "body": "-----BEGIN CERTIFICATE-----IyMzU5MTVaMHk...",
+                    "status": true,
+                    "cn": "AcommonName",
+                    "description": "This is the ROOT certificate for the TestAuthority certificate authority.",
+                    "chain": "",
+                    "notBefore": "2016-06-02T00:00:15+00:00",
+                    "notAfter": "2023-06-02T23:59:15+00:00",
+                    "owner": "secure@example.com",
+                    "user": {
+                        "username": "joe@example.com",
+                        "active": true,
+                        "email": "joe@example.com",
+                        "id": 3
+                    },
+                    "active": true,
+                    "bits": 2048,
+                    "id": 2235,
+                    "name": "TestAuthority"
+                },
+                "owner": "secure@example.com",
+                "id": 43,
+                "description": "This is the ROOT certificate for the TestAuthority certificate authority."
               }
 
            :reqheader Authorization: OAuth token to authenticate
@@ -378,10 +475,35 @@ class CertificateAuthority(AuthenticatedResource):
         """
         cert = certificate_service.get(certificate_id)
         if not cert:
-            return dict(message="Certificate not found"), 404
+            return dict(message="Certificate not found."), 404
 
         return cert.authority
 
+
+class AuthorityVisualizations(AuthenticatedResource):
+    def get(self, authority_id):
+        """
+        {"name": "flare",
+        "children": [
+            {
+                "name": "analytics",
+                "children": [
+                    {
+                        "name": "cluster",
+                        "children": [
+                            {"name": "AgglomerativeCluster", "size": 3938},
+                            {"name": "CommunityStructure", "size": 3812},
+                            {"name": "HierarchicalCluster", "size": 6714},
+                            {"name": "MergeEdge", "size": 743}
+                        ]
+                    }
+            }
+        ]}
+        """
+        authority = service.get(authority_id)
+        return dict(name=authority.name, children=[{"name": c.name} for c in authority.certificates])
+
 api.add_resource(AuthoritiesList, '/authorities', endpoint='authorities')
 api.add_resource(Authorities, '/authorities/<int:authority_id>', endpoint='authority')
+api.add_resource(AuthorityVisualizations, '/authorities/<int:authority_id>/visualize', endpoint='authority_visualizations')
 api.add_resource(CertificateAuthority, '/certificates/<int:certificate_id>/authority', endpoint='certificateAuthority')

lemur/certificates/models.py

@@ -6,269 +6,147 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import datetime
+
+import lemur.common.utils
 from flask import current_app
 
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
 from sqlalchemy.orm import relationship
+from sqlalchemy.sql.expression import case
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy import event, Integer, ForeignKey, String, DateTime, PassiveDefault, func, Column, Text, Boolean
 
-from lemur.utils import Vault
 from lemur.database import db
+from lemur.models import certificate_associations, certificate_source_associations, \
+    certificate_destination_associations, certificate_notification_associations, \
+    certificate_replacement_associations, roles_certificates
 from lemur.plugins.base import plugins
+from lemur.utils import Vault
 
+from lemur.common import defaults
 from lemur.domains.models import Domain
 
-from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
 
-from lemur.models import certificate_associations, certificate_source_associations, \
-    certificate_destination_associations, certificate_notification_associations
+def get_or_increase_name(name):
+    name = '-'.join(name.strip().split(' '))
+    count = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).count()
 
+    if count >= 1:
+        return name + '-' + str(count)
 
-def create_name(issuer, not_before, not_after, subject, san):
-    """
-    Create a name for our certificate. A naming standard
-    is based on a series of templates. The name includes
-    useful information such as Common Name, Validation dates,
-    and Issuer.
-
-    :rtype : str
-    :return:
-    """
-    if san:
-        t = SAN_NAMING_TEMPLATE
-    else:
-        t = DEFAULT_NAMING_TEMPLATE
-
-    temp = t.format(
-        subject=subject,
-        issuer=issuer,
-        not_before=not_before.strftime('%Y%m%d'),
-        not_after=not_after.strftime('%Y%m%d')
-    )
-
-    # NOTE we may want to give more control over naming
-    # aws doesn't allow special chars except '-'
-    disallowed_chars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
-    disallowed_chars = disallowed_chars.replace("-", "")
-    disallowed_chars = disallowed_chars.replace(".", "")
-    temp = temp.replace('*', "WILDCARD")
-
-    for c in disallowed_chars:
-        temp = temp.replace(c, "")
-
-    # white space is silly too
-    return temp.replace(" ", "-")
-
-
-def get_signing_algorithm(cert):
-    return cert.signature_hash_algorithm.name
-
-
-def get_cn(cert):
-    """
-    Attempts to get a sane common name from a given certificate.
-
-    :param cert:
-    :return: Common name or None
-    """
-    return cert.subject.get_attributes_for_oid(
-        x509.OID_COMMON_NAME
-    )[0].value.strip()
-
-
-def get_domains(cert):
-    """
-    Attempts to get an domains listed in a certificate.
-    If 'subjectAltName' extension is not available we simply
-    return the common name.
-
-    :param cert:
-    :return: List of domains
-    """
-    domains = []
-    try:
-        ext = cert.extensions.get_extension_for_oid(x509.OID_SUBJECT_ALTERNATIVE_NAME)
-        entries = ext.value.get_values_for_type(x509.DNSName)
-        for entry in entries:
-            domains.append(entry)
-    except Exception as e:
-        current_app.logger.warning("Failed to get SubjectAltName: {0}".format(e))
-
-    return domains
-
-
-def get_serial(cert):
-    """
-    Fetch the serial number from the certificate.
-
-    :param cert:
-    :return: serial number
-    """
-    return cert.serial
-
-
-def is_san(cert):
-    """
-    Determines if a given certificate is a SAN certificate.
-    SAN certificates are simply certificates that cover multiple domains.
-
-    :param cert:
-    :return: Bool
-    """
-    if len(get_domains(cert)) > 1:
-        return True
-
-
-def is_wildcard(cert):
-    """
-    Determines if certificate is a wildcard certificate.
-
-    :param cert:
-    :return: Bool
-    """
-    domains = get_domains(cert)
-    if len(domains) == 1 and domains[0][0:1] == "*":
-        return True
-
-    if cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[0].value[0:1] == "*":
-        return True
-
-
-def get_bitstrength(cert):
-    """
-    Calculates a certificates public key bit length.
-
-    :param cert:
-    :return: Integer
-    """
-    return cert.public_key().key_size
-
-
-def get_issuer(cert):
-    """
-    Gets a sane issuer from a given certificate.
-
-    :param cert:
-    :return: Issuer
-    """
-    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
-    try:
-        issuer = str(cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[0].value)
-        for c in delchars:
-            issuer = issuer.replace(c, "")
-        return issuer
-    except Exception as e:
-        current_app.logger.error("Unable to get issuer! {0}".format(e))
-
-
-def get_not_before(cert):
-    """
-    Gets the naive datetime of the certificates 'not_before' field.
-    This field denotes the first date in time which the given certificate
-    is valid.
-
-    :param cert:
-    :return: Datetime
-    """
-    return cert.not_valid_before
-
-
-def get_not_after(cert):
-    """
-    Gets the naive datetime of the certificates 'not_after' field.
-    This field denotes the last date in time which the given certificate
-    is valid.
-
-    :param cert:
-    :return: Datetime
-    """
-    return cert.not_valid_after
-
-
-def get_name_from_arn(arn):
-    """
-    Extract the certificate name from an arn.
-
-    :param arn: IAM SSL arn
-    :return: name of the certificate as uploaded to AWS
-    """
-    return arn.split("/", 1)[1]
-
-
-def get_account_number(arn):
-    """
-    Extract the account number from an arn.
-
-    :param arn: IAM SSL arn
-    :return: account number associated with ARN
-    """
-    return arn.split(":")[4]
+    return name
 
 
 class Certificate(db.Model):
     __tablename__ = 'certificates'
     id = Column(Integer, primary_key=True)
-    owner = Column(String(128))
-    body = Column(Text())
-    private_key = Column(Vault)
-    status = Column(String(128))
-    deleted = Column(Boolean, index=True)
-    name = Column(String(128))
+    owner = Column(String(128), nullable=False)
+    name = Column(String(128), unique=True)
+    description = Column(String(1024))
+    notify = Column(Boolean, default=True)
+
+    body = Column(Text(), nullable=False)
     chain = Column(Text())
-    bits = Column(Integer())
+    private_key = Column(Vault)
+
     issuer = Column(String(128))
     serial = Column(String(128))
     cn = Column(String(128))
-    description = Column(String(1024))
-    active = Column(Boolean, default=True)
-    san = Column(String(1024))
+    deleted = Column(Boolean, index=True)
+
     not_before = Column(DateTime)
     not_after = Column(DateTime)
     date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
+
     signing_algorithm = Column(String(128))
+    status = Column(String(128))
+    bits = Column(Integer())
+    san = Column(String(1024))  # TODO this should be migrated to boolean
+
     user_id = Column(Integer, ForeignKey('users.id'))
-    authority_id = Column(Integer, ForeignKey('authorities.id'))
+    authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
+    root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
+
     notifications = relationship("Notification", secondary=certificate_notification_associations, backref='certificate')
     destinations = relationship("Destination", secondary=certificate_destination_associations, backref='certificate')
     sources = relationship("Source", secondary=certificate_source_associations, backref='certificate')
     domains = relationship("Domain", secondary=certificate_associations, backref="certificate")
+    roles = relationship("Role", secondary=roles_certificates, backref="certificate")
+    replaces = relationship("Certificate",
+                            secondary=certificate_replacement_associations,
+                            primaryjoin=id == certificate_replacement_associations.c.certificate_id,  # noqa
+                            secondaryjoin=id == certificate_replacement_associations.c.replaced_certificate_id,  # noqa
+                            backref='replaced')
 
-    def __init__(self, body, private_key=None, chain=None):
-        self.body = body
-        # We encrypt the private_key on creation
-        self.private_key = private_key
-        self.chain = chain
-        cert = x509.load_pem_x509_certificate(str(self.body), default_backend())
-        self.signing_algorithm = get_signing_algorithm(cert)
-        self.bits = get_bitstrength(cert)
-        self.issuer = get_issuer(cert)
-        self.serial = get_serial(cert)
-        self.cn = get_cn(cert)
-        self.san = is_san(cert)
-        self.not_before = get_not_before(cert)
-        self.not_after = get_not_after(cert)
-        self.name = create_name(self.issuer, self.not_before, self.not_after, self.cn, self.san)
+    endpoints = relationship("Endpoint", backref='certificate')
 
-        for domain in get_domains(cert):
+    def __init__(self, **kwargs):
+        cert = lemur.common.utils.parse_certificate(kwargs['body'])
+
+        self.issuer = defaults.issuer(cert)
+        self.cn = defaults.common_name(cert)
+        self.san = defaults.san(cert)
+        self.not_before = defaults.not_before(cert)
+        self.not_after = defaults.not_after(cert)
+
+        # when destinations are appended they require a valid name.
+        if kwargs.get('name'):
+            self.name = get_or_increase_name(kwargs['name'])
+        else:
+            self.name = get_or_increase_name(defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san))
+
+        self.owner = kwargs['owner']
+        self.body = kwargs['body'].strip()
+
+        if kwargs.get('private_key'):
+            self.private_key = kwargs['private_key'].strip()
+
+        if kwargs.get('chain'):
+            self.chain = kwargs['chain'].strip()
+
+        self.notify = kwargs.get('notify', True)
+        self.destinations = kwargs.get('destinations', [])
+        self.notifications = kwargs.get('notifications', [])
+        self.description = kwargs.get('description')
+        self.roles = list(set(kwargs.get('roles', [])))
+        self.replaces = kwargs.get('replacements', [])
+        self.signing_algorithm = defaults.signing_algorithm(cert)
+        self.bits = defaults.bitstrength(cert)
+        self.serial = defaults.serial(cert)
+
+        for domain in defaults.domains(cert):
             self.domains.append(Domain(name=domain))
 
     @property
-    def is_expired(self):
-        if self.not_after < datetime.datetime.now():
+    def active(self):
+        return self.notify
+
+    @hybrid_property
+    def expired(self):
+        if self.not_after <= datetime.datetime.now():
             return True
 
-    @property
-    def is_unused(self):
-        if self.elb_listeners.count() == 0:
+    @expired.expression
+    def expired(cls):
+        return case(
+            [
+                (cls.now_after <= datetime.datetime.now(), True)
+            ],
+            else_=False
+        )
+
+    @hybrid_property
+    def revoked(self):
+        if 'revoked' == self.status:
             return True
 
-    @property
-    def is_revoked(self):
-        # we might not yet know the condition of the cert
-        if self.status:
-            if 'revoked' in self.status:
-                return True
+    @revoked.expression
+    def revoked(cls):
+        return case(
+            [
+                (cls.status == 'revoked', True)
+            ],
+            else_=False
+        )
 
     def get_arn(self, account_number):
         """
@@ -280,11 +158,53 @@ class Certificate(db.Model):
         """
         return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)
 
-    def as_dict(self):
-        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
+    def __repr__(self):
+        return "Certificate(name={name})".format(name=self.name)
 
 
 @event.listens_for(Certificate.destinations, 'append')
 def update_destinations(target, value, initiator):
+    """
+    Attempt to upload the new certificate to the new destination
+
+    :param target:
+    :param value:
+    :param initiator:
+    :return:
+    """
     destination_plugin = plugins.get(value.plugin_name)
-    destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
+
+    try:
+        destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
+    except Exception as e:
+        current_app.logger.exception(e)
+
+
+@event.listens_for(Certificate.replaces, 'append')
+def update_replacement(target, value, initiator):
+    """
+    When a certificate is marked as 'replaced' we should not notify.
+
+    :param target:
+    :param value:
+    :param initiator:
+    :return:
+    """
+    value.notify = False
+
+
+# @event.listens_for(Certificate, 'before_update')
+# def protect_active(mapper, connection, target):
+#    """
+#     When a certificate has a replacement do not allow it to be marked as 'active'
+#
+#     :param connection:
+#     :param mapper:
+#     :param target:
+#     :return:
+#     """
+#     if target.active:
+#         if not target.notify:
+#             raise Exception(
+#                 "Cannot silence notification for a certificate Lemur has been found to be currently deployed onto endpoints"
+#             )

lemur/certificates/schemas.py

@@ -0,0 +1,186 @@
+"""
+.. module: lemur.certificates.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import current_app
+from marshmallow import fields, validates_schema, post_load, pre_load
+from marshmallow.exceptions import ValidationError
+
+from lemur.schemas import AssociatedAuthoritySchema, AssociatedDestinationSchema, AssociatedCertificateSchema, \
+    AssociatedNotificationSchema, PluginInputSchema, ExtensionSchema, AssociatedRoleSchema, EndpointNestedOutputSchema
+
+from lemur.authorities.schemas import AuthorityNestedOutputSchema
+from lemur.destinations.schemas import DestinationNestedOutputSchema
+from lemur.notifications.schemas import NotificationNestedOutputSchema
+from lemur.roles.schemas import RoleNestedOutputSchema
+from lemur.domains.schemas import DomainNestedOutputSchema
+from lemur.users.schemas import UserNestedOutputSchema
+
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.common import validators, missing
+from lemur.notifications import service as notification_service
+
+from lemur.common.fields import ArrowDateTime
+
+
+class CertificateSchema(LemurInputSchema):
+    owner = fields.Email(required=True)
+    description = fields.String()
+
+
+class CertificateCreationSchema(CertificateSchema):
+    @post_load
+    def default_notification(self, data):
+        if not data['notifications']:
+            notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
+            data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']])
+
+        notification_name = 'DEFAULT_SECURITY'
+        data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
+        return data
+
+
+class CertificateInputSchema(CertificateCreationSchema):
+    name = fields.String()
+    common_name = fields.String(required=True, validate=validators.sensitive_domain)
+    authority = fields.Nested(AssociatedAuthoritySchema, required=True)
+
+    validity_start = ArrowDateTime()
+    validity_end = ArrowDateTime()
+    validity_years = fields.Integer()
+
+    destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
+    notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
+    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
+    roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
+
+    csr = fields.String(validate=validators.csr)
+
+    notify = fields.Boolean(default=True)
+
+    # certificate body fields
+    organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
+    organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
+    location = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_LOCATION'))
+    country = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_COUNTRY'))
+    state = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_STATE'))
+
+    extensions = fields.Nested(ExtensionSchema)
+
+    @validates_schema
+    def validate_dates(self, data):
+        validators.dates(data)
+
+    @pre_load
+    def ensure_dates(self, data):
+        return missing.convert_validity_years(data)
+
+
+class CertificateEditInputSchema(CertificateSchema):
+    notify = fields.Boolean()
+    owner = fields.String()
+    destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
+    notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
+    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
+    roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
+
+    @post_load
+    def enforce_notifications(self, data):
+        """
+        Ensures that when an owner changes, default notifications are added for the new owner.
+        Old owner notifications are retained unless explicitly removed.
+        :param data:
+        :return:
+        """
+        if data['owner']:
+            notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
+            data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']])
+        return data
+
+
+class CertificateNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    active = fields.Boolean()
+    bits = fields.Integer()
+    body = fields.String()
+    chain = fields.String()
+    description = fields.String()
+    name = fields.String()
+    cn = fields.String()
+    not_after = fields.DateTime()
+    not_before = fields.DateTime()
+    owner = fields.Email()
+    status = fields.Boolean()
+    creator = fields.Nested(UserNestedOutputSchema)
+    issuer = fields.Nested(AuthorityNestedOutputSchema)
+
+
+class CertificateCloneSchema(LemurOutputSchema):
+    __envelope__ = False
+    description = fields.String()
+    common_name = fields.String()
+
+
+class CertificateOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    active = fields.Boolean()
+    notify = fields.Boolean()
+    bits = fields.Integer()
+    body = fields.String()
+    chain = fields.String()
+    deleted = fields.Boolean(default=False)
+    description = fields.String()
+    issuer = fields.String()
+    name = fields.String()
+    cn = fields.String()
+    not_after = fields.DateTime()
+    not_before = fields.DateTime()
+    owner = fields.Email()
+    san = fields.Boolean()
+    serial = fields.String()
+    signing_algorithm = fields.String()
+    status = fields.Boolean()
+    user = fields.Nested(UserNestedOutputSchema)
+    domains = fields.Nested(DomainNestedOutputSchema, many=True)
+    destinations = fields.Nested(DestinationNestedOutputSchema, many=True)
+    notifications = fields.Nested(NotificationNestedOutputSchema, many=True)
+    replaces = fields.Nested(CertificateNestedOutputSchema, many=True)
+    authority = fields.Nested(AuthorityNestedOutputSchema)
+    roles = fields.Nested(RoleNestedOutputSchema, many=True)
+    endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
+
+
+class CertificateUploadInputSchema(CertificateCreationSchema):
+    name = fields.String()
+    notify = fields.Boolean(missing=True)
+
+    private_key = fields.String(validate=validators.private_key)
+    body = fields.String(required=True, validate=validators.public_certificate)
+    chain = fields.String(validate=validators.public_certificate)  # TODO this could be multiple certificates
+
+    destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
+    notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
+    replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
+    roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
+
+    @validates_schema
+    def keys(self, data):
+        if data.get('destinations'):
+            if not data.get('private_key'):
+                raise ValidationError('Destinations require private key.')
+
+
+class CertificateExportInputSchema(LemurInputSchema):
+    plugin = fields.Nested(PluginInputSchema)
+
+
+certificate_input_schema = CertificateInputSchema()
+certificate_output_schema = CertificateOutputSchema()
+certificates_output_schema = CertificateOutputSchema(many=True)
+certificate_upload_input_schema = CertificateUploadInputSchema()
+certificate_export_input_schema = CertificateExportInputSchema()
+certificate_edit_input_schema = CertificateEditInputSchema()

lemur/certificates/service.py

@@ -11,14 +11,17 @@ from sqlalchemy import func, or_
 from flask import g, current_app
 
 from lemur import database
+from lemur.extensions import metrics
 from lemur.plugins.base import plugins
 from lemur.certificates.models import Certificate
 
 from lemur.destinations.models import Destination
 from lemur.notifications.models import Notification
 from lemur.authorities.models import Authority
+from lemur.domains.models import Domain
 
 from lemur.roles.models import Role
+from lemur.roles import service as role_service
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -64,74 +67,96 @@ def get_all_certs():
     return Certificate.query.all()
 
 
-def find_duplicates(cert_body):
+def get_by_source(source_label):
+    """
+    Retrieves all certificates from a given source.
+
+    :param source_label:
+    :return:
+    """
+    return Certificate.query.filter(Certificate.sources.any(label=source_label))
+
+
+def find_duplicates(cert):
     """
     Finds certificates that already exist within Lemur. We do this by looking for
     certificate bodies that are the same. This is the most reliable way to determine
     if a certificate is already being tracked by Lemur.
 
-    :param cert_body:
+    :param cert:
     :return:
     """
-    return Certificate.query.filter_by(body=cert_body).all()
+    return Certificate.query.filter_by(body=cert['body'].strip(), chain=cert['chain'].strip()).all()
 
 
-def update(cert_id, owner, description, active, destinations, notifications):
+def export(cert, export_plugin):
     """
-    Updates a certificate.
+    Exports a certificate to the requested format. This format
+    may be a binary format.
 
+    :param export_plugin:
+    :param cert:
+    :return:
+    """
+    plugin = plugins.get(export_plugin['slug'])
+    return plugin.export(cert.body, cert.chain, cert.private_key, export_plugin['pluginOptions'])
+
+
+def update(cert_id, owner, description, notify, destinations, notifications, replaces, roles):
+    """
+    Updates a certificate
     :param cert_id:
     :param owner:
-    :param active:
+    :param description:
+    :param notify:
+    :param destinations:
+    :param notifications:
+    :param replaces:
     :return:
     """
-    from lemur.notifications import service as notification_service
     cert = get(cert_id)
-    cert.active = active
+    cert.notify = notify
     cert.description = description
-
-    # we might have to create new notifications if the owner changes
-    new_notifications = []
-    # get existing names to remove
-    notification_name = "DEFAULT_{0}".format(cert.owner.split('@')[0].upper())
-    for n in notifications:
-        if notification_name not in n.label:
-            new_notifications.append(n)
-
-    notification_name = "DEFAULT_{0}".format(owner.split('@')[0].upper())
-    new_notifications += notification_service.create_default_expiration_notifications(notification_name, owner)
-
-    cert.notifications = new_notifications
-
-    database.update_list(cert, 'destinations', Destination, destinations)
-
+    cert.destinations = destinations
+    cert.notifications = notifications
+    cert.roles = roles
+    cert.replaces = replaces
     cert.owner = owner
 
     return database.update(cert)
 
 
-def mint(issuer_options):
+def create_certificate_roles(**kwargs):
+    # create an role for the owner and assign it
+    owner_role = role_service.get_by_name(kwargs['owner'])
+    if not owner_role:
+        owner_role = role_service.create(
+            kwargs['owner'],
+            description="Auto generated role based on owner: {0}".format(kwargs['owner'])
+        )
+
+    return [owner_role]
+
+
+def mint(**kwargs):
     """
     Minting is slightly different for each authority.
     Support for multiple authorities is handled by individual plugins.
 
-    :param issuer_options:
     """
-    authority = issuer_options['authority']
+    authority = kwargs['authority']
 
     issuer = plugins.get(authority.plugin_name)
 
-    csr, private_key = create_csr(issuer_options)
+    # allow the CSR to be specified by the user
+    if not kwargs.get('csr'):
+        csr, private_key = create_csr(**kwargs)
+    else:
+        csr = str(kwargs.get('csr'))
+        private_key = None
 
-    issuer_options['creator'] = g.user.email
-    cert_body, cert_chain = issuer.create_certificate(csr, issuer_options)
-
-    cert = Certificate(cert_body, private_key, cert_chain)
-
-    cert.user = g.user
-    cert.authority = authority
-    database.update(cert)
-    return cert, private_key, cert_chain,
+    cert_body, cert_chain = issuer.create_certificate(csr, kwargs)
+    return cert_body, private_key, cert_chain,
 
 
 def import_certificate(**kwargs):
@@ -147,102 +172,64 @@ def import_certificate(**kwargs):
 
     :param kwargs:
     """
-    from lemur.users import service as user_service
-    from lemur.notifications import service as notification_service
-    cert = Certificate(kwargs['public_certificate'], chain=kwargs['intermediate_certificate'])
+    if not kwargs.get('owner'):
+        kwargs['owner'] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0]
 
-    # TODO future source plugins might have a better understanding of who the 'owner' is we should support this
-    cert.owner = kwargs.get('owner', current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0])
-    cert.creator = kwargs.get('creator', user_service.get_by_email('lemur@nobody'))
-
-    # NOTE existing certs may not follow our naming standard we will
-    # overwrite the generated name with the actual cert name
-    if kwargs.get('name'):
-        cert.name = kwargs.get('name')
-
-    if kwargs.get('user'):
-        cert.user = kwargs.get('user')
-
-    notification_name = 'DEFAULT_SECURITY'
-    notifications = notification_service.create_default_expiration_notifications(notification_name, current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
-    cert.notifications = notifications
-
-    cert = database.create(cert)
-    return cert
+    return upload(**kwargs)
 
 
 def upload(**kwargs):
     """
     Allows for pre-made certificates to be imported into Lemur.
     """
-    from lemur.notifications import service as notification_service
-    cert = Certificate(
-        kwargs.get('public_cert'),
-        kwargs.get('private_key'),
-        kwargs.get('intermediate_cert'),
-    )
+    roles = create_certificate_roles(**kwargs)
 
-    # we override the generated name if one is provided
-    if kwargs.get('name'):
-        cert.name = kwargs['name']
+    if kwargs.get('roles'):
+        kwargs['roles'] += roles
+    else:
+        kwargs['roles'] = roles
 
-    cert.description = kwargs.get('description')
+    if kwargs.get('private_key'):
+        private_key = kwargs['private_key']
+        if not isinstance(private_key, bytes):
+            kwargs['private_key'] = private_key.encode('utf-8')
+
+    cert = Certificate(**kwargs)
 
-    cert.owner = kwargs['owner']
     cert = database.create(cert)
 
-    g.user.certificates.append(cert)
+    try:
+        g.user.certificates.append(cert)
+    except AttributeError:
+        current_app.logger.debug("No user to associate uploaded certificate to.")
 
-    database.update_list(cert, 'destinations', Destination, kwargs.get('destinations'))
-
-    database.update_list(cert, 'notifications', Notification, kwargs.get('notifications'))
-
-    # create default notifications for this certificate if none are provided
-    notifications = []
-    if not kwargs.get('notifications'):
-        notification_name = "DEFAULT_{0}".format(cert.owner.split('@')[0].upper())
-        notifications += notification_service.create_default_expiration_notifications(notification_name, [cert.owner])
-
-    notification_name = 'DEFAULT_SECURITY'
-    notifications += notification_service.create_default_expiration_notifications(notification_name, current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
-    cert.notifications = notifications
-
-    database.update(cert)
-    return cert
+    return database.update(cert)
 
 
 def create(**kwargs):
     """
     Creates a new certificate.
     """
-    from lemur.notifications import service as notification_service
-    cert, private_key, cert_chain = mint(kwargs)
+    kwargs['creator'] = g.user.email
+    cert_body, private_key, cert_chain = mint(**kwargs)
+    kwargs['body'] = cert_body
+    kwargs['private_key'] = private_key
+    kwargs['chain'] = cert_chain
 
-    cert.owner = kwargs['owner']
+    roles = create_certificate_roles(**kwargs)
+
+    if kwargs.get('roles'):
+        kwargs['roles'] += roles
+    else:
+        kwargs['roles'] = roles
+
+    cert = Certificate(**kwargs)
 
-    database.create(cert)
-    cert.description = kwargs['description']
     g.user.certificates.append(cert)
-    database.update(g.user)
+    cert.authority = kwargs['authority']
+    database.commit()
 
-    # do this after the certificate has already been created because if it fails to upload to the third party
-    # we do not want to lose the certificate information.
-    database.update_list(cert, 'destinations', Destination, kwargs.get('destinations'))
-
-    database.update_list(cert, 'notifications', Notification, kwargs.get('notifications'))
-
-    # create default notifications for this certificate if none are provided
-    notifications = cert.notifications
-    if not kwargs.get('notifications'):
-        notification_name = "DEFAULT_{0}".format(cert.owner.split('@')[0].upper())
-        notifications += notification_service.create_default_expiration_notifications(notification_name, [cert.owner])
-
-    notification_name = 'DEFAULT_SECURITY'
-    notifications += notification_service.create_default_expiration_notifications(notification_name,
-                                                                                  current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'))
-    cert.notifications = notifications
-
-    database.update(cert)
+    metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
     return cert
 
 
@@ -266,6 +253,7 @@ def render(args):
 
     if filt:
         terms = filt.split(';')
+
         if 'issuer' in terms:
             # we can't rely on issuer being correct in the cert directly so we combine queries
             sub_query = database.session_query(Authority.id)\
@@ -280,10 +268,17 @@ def render(args):
             )
             return database.sort_and_page(query, Certificate, args)
 
-        if 'destination' in terms:
+        elif 'destination' in terms:
             query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
-        elif 'active' in filt:  # this is really weird but strcmp seems to not work here??
+        elif 'active' in filt:
             query = query.filter(Certificate.active == terms[1])
+        elif 'cn' in terms:
+            query = query.filter(
+                or_(
+                    Certificate.cn.ilike('%{0}%'.format(terms[1])),
+                    Certificate.domains.any(Domain.name.ilike('%{0}%'.format(terms[1])))
+                )
+            )
         else:
             query = database.filter(query, Certificate, terms)
 
@@ -310,7 +305,7 @@ def render(args):
     return database.sort_and_page(query, Certificate, args)
 
 
-def create_csr(csr_config):
+def create_csr(**csr_config):
     """
     Given a list of domains create the appropriate csr
     for those domains
@@ -326,12 +321,13 @@ def create_csr(csr_config):
     # TODO When we figure out a better way to validate these options they should be parsed as str
     builder = x509.CertificateSigningRequestBuilder()
     builder = builder.subject_name(x509.Name([
-        x509.NameAttribute(x509.OID_COMMON_NAME, csr_config['commonName']),
+        x509.NameAttribute(x509.OID_COMMON_NAME, csr_config['common_name']),
         x509.NameAttribute(x509.OID_ORGANIZATION_NAME, csr_config['organization']),
-        x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config['organizationalUnit']),
+        x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, csr_config['organizational_unit']),
         x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config['country']),
         x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config['state']),
         x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config['location']),
+        x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config['owner'])
     ]))
 
     builder = builder.add_extension(
@@ -340,11 +336,11 @@ def create_csr(csr_config):
 
     if csr_config.get('extensions'):
         for k, v in csr_config.get('extensions', {}).items():
-            if k == 'subAltNames':
+            if k == 'sub_alt_names':
                 # map types to their x509 objects
                 general_names = []
                 for name in v['names']:
-                    if name['nameType'] == 'DNSName':
+                    if name['name_type'] == 'DNSName':
                         general_names.append(x509.DNSName(name['value']))
 
                 builder = builder.add_extension(
@@ -399,17 +395,20 @@ def create_csr(csr_config):
     )
 
     # serialize our private key and CSR
-    pem = private_key.private_bytes(
+    private_key = private_key.private_bytes(
         encoding=serialization.Encoding.PEM,
         format=serialization.PrivateFormat.TraditionalOpenSSL,  # would like to use PKCS8 but AWS ELBs don't like it
         encryption_algorithm=serialization.NoEncryption()
     )
 
+    if isinstance(private_key, bytes):
+        private_key = private_key.decode('utf-8')
+
     csr = request.public_bytes(
         encoding=serialization.Encoding.PEM
     )
 
-    return csr, pem
+    return csr, private_key
 
 
 def stats(**kwargs):
@@ -440,3 +439,69 @@ def stats(**kwargs):
         values.append(count)
 
     return {'labels': keys, 'values': values}
+
+
+def get_account_number(arn):
+    """
+    Extract the account number from an arn.
+
+    :param arn: IAM SSL arn
+    :return: account number associated with ARN
+    """
+    return arn.split(":")[4]
+
+
+def get_name_from_arn(arn):
+    """
+    Extract the certificate name from an arn.
+
+    :param arn: IAM SSL arn
+    :return: name of the certificate as uploaded to AWS
+    """
+    return arn.split("/", 1)[1]
+
+
+def calculate_reissue_range(start, end):
+    """
+    Determine what the new validity_start and validity_end dates should be.
+    :param start:
+    :param end:
+    :return:
+    """
+    span = end - start
+
+    new_start = arrow.utcnow().date()
+    new_end = new_start + span
+
+    return new_start, new_end
+
+
+# TODO pull the OU, O, CN, etc + other extensions.
+def get_certificate_primitives(certificate):
+    """
+    Retrieve key primitive from a certificate such that the certificate
+    could be recreated with new expiration or be used to build upon.
+    :param certificate:
+    :return: dict of certificate primitives, should be enough to effectively re-issue
+    certificate via `create`.
+    """
+    start, end = calculate_reissue_range(certificate.not_before, certificate.not_after)
+    names = [{'name_type': 'DNSName', 'value': x.name} for x in certificate.domains]
+
+    extensions = {
+        'sub_alt_names': {
+            'names': names
+        }
+    }
+
+    return dict(
+        authority=certificate.authority,
+        common_name=certificate.cn,
+        description=certificate.description,
+        validity_start=start,
+        validity_end=end,
+        destinations=certificate.destinations,
+        roles=certificate.roles,
+        extensions=extensions,
+        owner=certificate.owner
+    )

lemur/certificates/verify.py

@@ -5,7 +5,6 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import os
 import requests
 import subprocess
 from OpenSSL import crypto
@@ -13,20 +12,7 @@ from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 
 from flask import current_app
-
-from contextlib import contextmanager
-from tempfile import NamedTemporaryFile
-
-
-@contextmanager
-def mktempfile():
-    with NamedTemporaryFile(delete=False) as f:
-        name = f.name
-
-    try:
-        yield name
-    finally:
-        os.unlink(name)
+from lemur.utils import mktempfile
 
 
 def ocsp_verify(cert_path, issuer_chain_path):

lemur/certificates/views.py

@@ -5,109 +5,37 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import base64
 from builtins import str
 
 from flask import Blueprint, make_response, jsonify
-from flask.ext.restful import reqparse, Api, fields
+from flask.ext.restful import reqparse, Api
 
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
-
-from lemur.certificates import service
-from lemur.authorities.models import Authority
+from lemur.common.schema import validate_schema
+from lemur.common.utils import paginated_parser
 
 from lemur.auth.service import AuthenticatedResource
-from lemur.auth.permissions import ViewKeyPermission, AuthorityPermission, UpdateCertificatePermission
+from lemur.auth.permissions import AuthorityPermission, CertificatePermission
+
+from lemur.certificates import service
+from lemur.certificates.schemas import certificate_input_schema, certificate_output_schema, \
+    certificate_upload_input_schema, certificates_output_schema, certificate_export_input_schema, certificate_edit_input_schema
 
 from lemur.roles import service as role_service
 
-from lemur.common.utils import marshal_items, paginated_parser
-
-from lemur.notifications.views import notification_list
-
 
 mod = Blueprint('certificates', __name__)
 api = Api(mod)
 
 
-FIELDS = {
-    'name': fields.String,
-    'id': fields.Integer,
-    'bits': fields.Integer,
-    'deleted': fields.String,
-    'issuer': fields.String,
-    'serial': fields.String,
-    'owner': fields.String,
-    'chain': fields.String,
-    'san': fields.String,
-    'active': fields.Boolean,
-    'description': fields.String,
-    'notBefore': fields.DateTime(dt_format='iso8601', attribute='not_before'),
-    'notAfter': fields.DateTime(dt_format='iso8601', attribute='not_after'),
-    'cn': fields.String,
-    'signingAlgorithm': fields.String(attribute='signing_algorithm'),
-    'status': fields.String,
-    'body': fields.String
-}
-
-
-def valid_authority(authority_options):
-    """
-    Defends against invalid authorities
-
-    :param authority_options:
-    :return: :raise ValueError:
-    """
-    name = authority_options['name']
-    authority = Authority.query.filter(Authority.name == name).one()
-
-    if not authority:
-        raise ValueError("Unable to find authority specified")
-
-    if not authority.active:
-        raise ValueError("Selected authority [{0}] is not currently active".format(name))
-
-    return authority
-
-
-def pem_str(value, name):
-    """
-    Used to validate that the given string is a PEM formatted string
-
-    :param value:
-    :param name:
-    :return: :raise ValueError:
-    """
-    try:
-        x509.load_pem_x509_certificate(bytes(value), default_backend())
-    except Exception:
-        raise ValueError("The parameter '{0}' needs to be a valid PEM string".format(name))
-    return value
-
-
-def private_key_str(value, name):
-    """
-    User to validate that a given string is a RSA private key
-
-    :param value:
-    :param name:
-    :return: :raise ValueError:
-    """
-    try:
-        serialization.load_pem_private_key(bytes(value), None, backend=default_backend())
-    except Exception:
-        raise ValueError("The parameter '{0}' needs to be a valid RSA private key".format(name))
-    return value
-
-
 class CertificatesList(AuthenticatedResource):
     """ Defines the 'certificates' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(CertificatesList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, certificates_output_schema)
     def get(self):
         """
         .. http:get:: /certificates
@@ -131,37 +59,65 @@ class CertificatesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
-                    {
-                      "id": 1,
-                      "name": "cert1",
-                      "description": "this is cert1",
-                      "bits": 2048,
-                      "deleted": false,
-                      "issuer": "ExampeInc.",
-                      "serial": "123450",
-                      "chain": "-----Begin ...",
-                      "body": "-----Begin ...",
-                      "san": true,
-                      "owner": 'bob@example.com",
-                      "active": true,
-                      "notBefore": "2015-06-05T17:09:39",
-                      "notAfter": "2015-06-10T17:09:39",
-                      "cn": "example.com",
-                      "status": "unknown"
-                    }
-                  ]
+                "items": [{
+                    "status": null,
+                    "cn": "*.test.example.net",
+                    "chain": "",
+                    "authority": {
+                        "active": true,
+                        "owner": "secure@example.com",
+                        "id": 1,
+                        "description": "verisign test authority",
+                        "name": "verisign"
+                    },
+                    "owner": "joe@example.com",
+                    "serial": "82311058732025924142789179368889309156",
+                    "id": 2288,
+                    "issuer": "SymantecCorporation",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }]
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
                 "total": 1
               }
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
            :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query filter: key value pair format is k;v
+           :query count: count number. default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
         parser = paginated_parser.copy()
         parser.add_argument('timeRange', type=int, dest='time_range', location='args')
@@ -175,8 +131,8 @@ class CertificatesList(AuthenticatedResource):
         args = parser.parse_args()
         return service.render(args)
 
-    @marshal_items(FIELDS)
-    def post(self):
+    @validate_schema(certificate_input_schema, certificate_output_schema)
+    def post(self, data=None):
         """
         .. http:post:: /certificates
 
@@ -191,86 +147,38 @@ class CertificatesList(AuthenticatedResource):
               Accept: application/json, text/javascript
 
               {
-                "country": "US",
-                "state": "CA",
-                "location": "A Place",
-                "organization": "ExampleInc.",
-                "organizationalUnit": "Operations",
-                "owner": "bob@example.com",
-                "description": "test",
-                "selectedAuthority": "timetest2",
-                "authority": {
-                    "body": "-----BEGIN...",
-                    "name": "timetest2",
-                    "chain": "",
-                    "notBefore": "2015-06-05T15:20:59",
-                    "active": true,
-                    "id": 50,
-                    "notAfter": "2015-06-17T15:21:08",
-                    "description": "dsfdsf"
-                },
-                "notifications": [
-                    {
-                      "description": "Default 30 day expiration notification",
-                      "notificationOptions": [
-                        {
-                          "name": "interval",
-                          "required": true,
-                          "value": 30,
-                          "helpMessage": "Number of days to be alert before expiration.",
-                          "validation": "^\\d+$",
-                          "type": "int"
-                        },
-                        {
-                          "available": [
-                            "days",
-                            "weeks",
-                            "months"
-                          ],
-                          "name": "unit",
-                          "required": true,
-                          "value": "days",
-                          "helpMessage": "Interval unit",
-                          "validation": "",
-                          "type": "select"
-                        },
-                        {
-                          "name": "recipients",
-                          "required": true,
-                          "value": "bob@example.com",
-                          "helpMessage": "Comma delimited list of email addresses",
-                          "validation": "^([\\w+-.%]+@[\\w-.]+\\.[A-Za-z]{2,4},?)+$",
-                            "type": "str"
-                          }
-                        ],
-                        "label": "DEFAULT_KGLISSON_30_DAY",
-                        "pluginName": "email-notification",
-                        "active": true,
-                        "id": 7
-                    }
-                ],
-                "extensions": {
-                    "basicConstraints": {},
-                    "keyUsage": {
-                        "isCritical": true,
-                        "useKeyEncipherment": true,
-                        "useDigitalSignature": true
-                    },
-                    "extendedKeyUsage": {
-                        "isCritical": true,
-                        "useServerAuthentication": true
-                    },
-                    "subjectKeyIdentifier": {
-                        "includeSKI": true
-                    },
+                  "owner": "secure@example.net",
+                  "commonName": "test.example.net",
+                  "country": "US",
+                  "extensions": {
                     "subAltNames": {
-                        "names": []
+                      "names": [
+                        {
+                          "nameType": "DNSName",
+                          "value": "*.test.example.net"
+                        },
+                        {
+                          "nameType": "DNSName",
+                          "value": "www.test.example.net"
+                        }
+                      ]
                     }
-                },
-                "commonName": "test",
-                "validityStart": "2015-06-05T07:00:00.000Z",
-                "validityEnd": "2015-06-16T07:00:00.000Z"
-             }
+                  },
+                  "replacements": [{
+                    "id": 1
+                  },
+                  "notify": true,
+                  "validityEnd": "2026-01-01T08:00:00.000Z",
+                  "authority": {
+                    "name": "verisign"
+                  },
+                  "organization": "Netflix, Inc.",
+                  "location": "Los Gatos",
+                  "state": "California",
+                  "validityStart": "2016-11-11T04:19:48.000Z",
+                  "organizationalUnit": "Operations"
+              }
+
 
            **Example response**:
 
@@ -281,24 +189,56 @@ class CertificatesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
-                "name": "cert1",
-                "description": "this is cert1",
+                "status": null,
+                "cn": "*.test.example.net",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
                 "bits": 2048,
-                "deleted": false,
-                "issuer": "ExampeInc.",
-                "serial": "123450",
-                "chain": "-----Begin ...",
-                "body": "-----Begin ...",
-                "san": true,
-                "owner": "jimbob@example.com",
-                "active": false,
-                "notBefore": "2015-06-05T17:09:39",
-                "notAfter": "2015-06-10T17:09:39",
-                "cn": "example.com",
-                "status": "unknown"
+                "body": "-----BEGIN CERTIFICATE-----...",
+                "description": null,
+                "deleted": null,
+                "notifications": [{
+                    "id": 1
+                }]
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [{
+                    "id": 1
+                }],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "san": null
               }
 
+
            :arg extensions: extensions to be used in the certificate
            :arg description: description for new certificate
            :arg owner: owner email
@@ -309,52 +249,36 @@ class CertificatesList(AuthenticatedResource):
            :arg state: state for the CSR
            :arg location: location for the CSR
            :arg organization: organization for CSR
-           :arg commonName: certiifcate common name
+           :arg commonName: certificate common name
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
-        self.reqparse.add_argument('extensions', type=dict, location='json')
-        self.reqparse.add_argument('destinations', type=list, default=[], location='json')
-        self.reqparse.add_argument('notifications', type=list, default=[], location='json')
-        self.reqparse.add_argument('validityStart', type=str, location='json')  # TODO validate
-        self.reqparse.add_argument('validityEnd', type=str, location='json')  # TODO validate
-        self.reqparse.add_argument('authority', type=valid_authority, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-        self.reqparse.add_argument('country', type=str, location='json', required=True)
-        self.reqparse.add_argument('state', type=str, location='json', required=True)
-        self.reqparse.add_argument('location', type=str, location='json', required=True)
-        self.reqparse.add_argument('organization', type=str, location='json', required=True)
-        self.reqparse.add_argument('organizationalUnit', type=str, location='json', required=True)
-        self.reqparse.add_argument('owner', type=str, location='json', required=True)
-        self.reqparse.add_argument('commonName', type=str, location='json', required=True)
-
-        args = self.reqparse.parse_args()
-
-        authority = args['authority']
-        role = role_service.get_by_name(authority.owner)
+        role = role_service.get_by_name(data['authority'].owner)
 
         # all the authority role members should be allowed
-        roles = [x.name for x in authority.roles]
+        roles = [x.name for x in data['authority'].roles]
 
         # allow "owner" roles by team DL
         roles.append(role)
-        permission = AuthorityPermission(authority.id, roles)
+        authority_permission = AuthorityPermission(data['authority'].id, roles)
 
-        if permission.can():
-            return service.create(**args)
+        if authority_permission.can():
+            return service.create(**data)
 
-        return dict(message="You are not authorized to use {0}".format(args['authority'].name)), 403
+        return dict(message="You are not authorized to use {0}".format(data['authority'].name)), 403
 
 
 class CertificatesUpload(AuthenticatedResource):
     """ Defines the 'certificates' upload endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(CertificatesUpload, self).__init__()
 
-    @marshal_items(FIELDS)
-    def post(self):
+    @validate_schema(certificate_upload_input_schema, certificate_output_schema)
+    def post(self, data=None):
         """
         .. http:post:: /certificates/upload
 
@@ -375,6 +299,7 @@ class CertificatesUpload(AuthenticatedResource):
                  "privateKey": "---Begin Private..."
                  "destinations": [],
                  "notifications": [],
+                 "replacements": [],
                  "name": "cert1"
               }
 
@@ -387,23 +312,51 @@ class CertificatesUpload(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                 "id": 1,
-                 "name": "cert1",
-                 "description": "this is cert1",
-                 "bits": 2048,
-                 "deleted": false,
-                 "issuer": "ExampeInc.",
-                 "serial": "123450",
-                 "chain": "-----Begin ...",
-                 "body": "-----Begin ...",
-                 "san": true,
-                 "owner": "joe@example.com",
-                 "active": true,
-                 "notBefore": "2015-06-05T17:09:39",
-                 "notAfter": "2015-06-10T17:09:39",
-                 "signingAlgorithm": "sha2"
-                 "cn": "example.com",
-                 "status": "unknown"
+                "status": null,
+                "cn": "*.test.example.net",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
+                "bits": 2048,
+                "body": "-----BEGIN CERTIFICATE-----...",
+                "description": null,
+                "deleted": null,
+                "notifications": [{
+                    "id": 1
+                }]
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "san": null
               }
 
            :arg owner: owner email for certificate
@@ -414,27 +367,19 @@ class CertificatesUpload(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 403: unauthenticated
            :statuscode 200: no error
-        """
-        self.reqparse.add_argument('description', type=str, location='json')
-        self.reqparse.add_argument('owner', type=str, required=True, location='json')
-        self.reqparse.add_argument('name', type=str, location='json')
-        self.reqparse.add_argument('publicCert', type=pem_str, required=True, dest='public_cert', location='json')
-        self.reqparse.add_argument('destinations', type=list, default=[], dest='destinations', location='json')
-        self.reqparse.add_argument('notifications', type=list, default=[], dest='notifications', location='json')
-        self.reqparse.add_argument('intermediateCert', type=pem_str, dest='intermediate_cert', location='json')
-        self.reqparse.add_argument('privateKey', type=private_key_str, dest='private_key', location='json')
 
-        args = self.reqparse.parse_args()
-        if args.get('destinations'):
-            if args.get('private_key'):
-                return service.upload(**args)
+        """
+        if data.get('destinations'):
+            if data.get('private_key'):
+                return service.upload(**data)
             else:
                 raise Exception("Private key must be provided in order to upload certificate to AWS")
-        return service.upload(**args)
+        return service.upload(**data)
 
 
 class CertificatesStats(AuthenticatedResource):
     """ Defines the 'certificates' stats endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(CertificatesStats, self).__init__()
@@ -489,9 +434,8 @@ class CertificatePrivateKey(AuthenticatedResource):
         if not cert:
             return dict(message="Cannot find specified certificate"), 404
 
-        role = role_service.get_by_name(cert.owner)
-
-        permission = ViewKeyPermission(certificate_id, getattr(role, 'name', None))
+        owner_role = role_service.get_by_name(cert.owner)
+        permission = CertificatePermission(cert.id, owner_role, [x.name for x in cert.roles])
 
         if permission.can():
             response = make_response(jsonify(key=cert.private_key), 200)
@@ -507,7 +451,7 @@ class Certificates(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Certificates, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, certificate_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1
@@ -531,33 +475,62 @@ class Certificates(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
-                "name": "cert1",
-                "description": "this is cert1",
+                "status": null,
+                "cn": "*.test.example.net",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
                 "bits": 2048,
-                "deleted": false,
-                "issuer": "ExampeInc.",
-                "serial": "123450",
-                "chain": "-----Begin ...",
-                "body": "-----Begin ...",
-                "san": true,
-                "owner": "bob@example.com",
+                "body": "-----BEGIN CERTIFICATE-----...",
+                "description": null,
+                "deleted": null,
+                "notifications": [{
+                    "id": 1
+                }]
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
                 "active": true,
-                "notBefore": "2015-06-05T17:09:39",
-                "notAfter": "2015-06-10T17:09:39",
-                "signingAlgorithm": "sha2",
-                "cn": "example.com",
-                "status": "unknown"
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "san": null
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
         return service.get(certificate_id)
 
-    @marshal_items(FIELDS)
-    def put(self, certificate_id):
+    @validate_schema(certificate_edit_input_schema, certificate_output_schema)
+    def put(self, certificate_id, data=None):
         """
         .. http:put:: /certificates/1
 
@@ -575,7 +548,8 @@ class Certificates(AuthenticatedResource):
                  "owner": "jimbob@example.com",
                  "active": false
                  "notifications": [],
-                 "destinations": []
+                 "destinations": [],
+                 "replacements": []
               }
 
            **Example response**:
@@ -587,48 +561,78 @@ class Certificates(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "id": 1,
-                "name": "cert1",
-                "description": "this is cert1",
+                "status": null,
+                "cn": "*.test.example.net",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
                 "bits": 2048,
-                "deleted": false,
-                "issuer": "ExampeInc.",
-                "serial": "123450",
-                "chain": "-----Begin ...",
-                "body": "-----Begin ...",
-                "san": true,
-                "owner": "jimbob@example.com",
-                "active": false,
-                "notBefore": "2015-06-05T17:09:39",
-                "notAfter": "2015-06-10T17:09:39",
-                "cn": "example.com",
-                "status": "unknown",
+                "body": "-----BEGIN CERTIFICATE-----...",
+                "description": null,
+                "deleted": null,
+                "notifications": [{
+                    "id": 1
+                }]
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "san": null
               }
 
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
-        self.reqparse.add_argument('active', type=bool, location='json')
-        self.reqparse.add_argument('owner', type=str, location='json')
-        self.reqparse.add_argument('description', type=str, location='json')
-        self.reqparse.add_argument('destinations', type=list, default=[], location='json')
-        self.reqparse.add_argument('notifications', type=notification_list, default=[], location='json')
-        args = self.reqparse.parse_args()
-
         cert = service.get(certificate_id)
-        role = role_service.get_by_name(cert.owner)
 
-        permission = UpdateCertificatePermission(certificate_id, getattr(role, 'name', None))
+        owner_role = role_service.get_by_name(cert.owner)
+        permission = CertificatePermission(cert.id, owner_role, [x.name for x in cert.roles])
 
         if permission.can():
+            for destination in data['destinations']:
+                if destination.plugin.requires_key:
+                    if not cert.private_key:
+                        return dict('Unable to add destination: {0}. Certificate does not have required private key.'.format(destination.label))
+
             return service.update(
                 certificate_id,
-                args['owner'],
-                args['description'],
-                args['active'],
-                args['destinations'],
-                args['notifications']
+                data['owner'],
+                data['description'],
+                data['notify'],
+                data['destinations'],
+                data['notifications'],
+                data['replacements'],
+                data['roles']
             )
 
         return dict(message='You are not authorized to update this certificate'), 403
@@ -636,11 +640,12 @@ class Certificates(AuthenticatedResource):
 
 class NotificationCertificatesList(AuthenticatedResource):
     """ Defines the 'certificates' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(NotificationCertificatesList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, certificates_output_schema)
     def get(self, notification_id):
         """
         .. http:get:: /notifications/1/certificates
@@ -664,38 +669,65 @@ class NotificationCertificatesList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
-                    {
-                      "id": 1,
-                      "name": "cert1",
-                      "description": "this is cert1",
-                      "bits": 2048,
-                      "deleted": false,
-                      "issuer": "ExampeInc.",
-                      "serial": "123450",
-                      "chain": "-----Begin ...",
-                      "body": "-----Begin ...",
-                      "san": true,
-                      "owner": 'bob@example.com",
-                      "active": true,
-                      "notBefore": "2015-06-05T17:09:39",
-                      "notAfter": "2015-06-10T17:09:39",
-                      "signingAlgorithm": "sha2",
-                      "cn": "example.com",
-                      "status": "unknown"
-                    }
-                  ]
+                "items": [{
+                    "status": null,
+                    "cn": "*.test.example.net",
+                    "chain": "",
+                    "authority": {
+                        "active": true,
+                        "owner": "secure@example.com",
+                        "id": 1,
+                        "description": "verisign test authority",
+                        "name": "verisign"
+                    },
+                    "owner": "joe@example.com",
+                    "serial": "82311058732025924142789179368889309156",
+                    "id": 2288,
+                    "issuer": "SymantecCorporation",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }]
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
                 "total": 1
               }
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
-           :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
+
         """
         parser = paginated_parser.copy()
         parser.add_argument('timeRange', type=int, dest='time_range', location='args')
@@ -711,9 +743,207 @@ class NotificationCertificatesList(AuthenticatedResource):
         return service.render(args)
 
 
+class CertificatesReplacementsList(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificatesReplacementsList, self).__init__()
+
+    @validate_schema(None, certificates_output_schema)
+    def get(self, certificate_id):
+        """
+        .. http:get:: /certificates/1/replacements
+
+           One certificate
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /certificates/1/replacements HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "items": [{
+                    "status": null,
+                    "cn": "*.test.example.net",
+                    "chain": "",
+                    "authority": {
+                        "active": true,
+                        "owner": "secure@example.com",
+                        "id": 1,
+                        "description": "verisign test authority",
+                        "name": "verisign"
+                    },
+                    "owner": "joe@example.com",
+                    "serial": "82311058732025924142789179368889309156",
+                    "id": 2288,
+                    "issuer": "SymantecCorporation",
+                    "notBefore": "2016-06-03T00:00:00+00:00",
+                    "notAfter": "2018-01-12T23:59:59+00:00",
+                    "destinations": [],
+                    "bits": 2048,
+                    "body": "-----BEGIN CERTIFICATE-----...",
+                    "description": null,
+                    "deleted": null,
+                    "notifications": [{
+                        "id": 1
+                    }]
+                    "signingAlgorithm": "sha256",
+                    "user": {
+                        "username": "jane",
+                        "active": true,
+                        "email": "jane@example.com",
+                        "id": 2
+                    },
+                    "active": true,
+                    "domains": [{
+                        "sensitive": false,
+                        "id": 1090,
+                        "name": "*.test.example.net"
+                    }],
+                    "replaces": [],
+                    "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                    "roles": [{
+                        "id": 464,
+                        "description": "This is a google group based role created by Lemur",
+                        "name": "joe@example.com"
+                    }],
+                    "san": null
+                }],
+                "total": 1
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        return service.get(certificate_id).replaces
+
+
+class CertificateExport(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificateExport, self).__init__()
+
+    @validate_schema(certificate_export_input_schema, None)
+    def post(self, certificate_id, data=None):
+        """
+        .. http:post:: /certificates/1/export
+
+           Export a certificate
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              PUT /certificates/1/export HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                "export": {
+                    "plugin": {
+                        "pluginOptions": [{
+                            "available": ["Java Key Store (JKS)"],
+                            "required": true,
+                            "type": "select",
+                            "name": "type",
+                            "helpMessage": "Choose the format you wish to export",
+                            "value": "Java Key Store (JKS)"
+                        }, {
+                            "required": false,
+                            "type": "str",
+                            "name": "passphrase",
+                            "validation": "^(?=.*[A-Za-z])(?=.*\\d)(?=.*[$@$!%*#?&])[A-Za-z\\d$@$!%*#?&]{8,}$",
+                            "helpMessage": "If no passphrase is given one will be generated for you, we highly recommend this. Minimum length is 8."
+                        }, {
+                            "required": false,
+                            "type": "str",
+                            "name": "alias",
+                            "helpMessage": "Enter the alias you wish to use for the keystore."
+                        }],
+                        "version": "unknown",
+                        "description": "Attempts to generate a JKS keystore or truststore",
+                        "title": "Java",
+                        "author": "Kevin Glisson",
+                        "type": "export",
+                        "slug": "java-export"
+                    }
+                }
+              }
+
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "data": "base64encodedstring",
+                "passphrase": "UAWOHW#&@_%!tnwmxh832025",
+                "extension": "jks"
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        cert = service.get(certificate_id)
+
+        owner_role = role_service.get_by_name(cert.owner)
+        permission = CertificatePermission(cert.id, owner_role, [x.name for x in cert.roles])
+
+        options = data['plugin']['plugin_options']
+        plugin = data['plugin']['plugin_object']
+
+        if plugin.requires_key:
+            if cert.private_key:
+                if permission.can():
+                    extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
+                else:
+                    return dict(message='You are not authorized to export this certificate.'), 403
+            else:
+                return dict(message='Unable to export certificate, plugin: {0} requires a private key but no key was found.'.format(plugin.slug))
+        else:
+            extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
+
+        # we take a hit in message size when b64 encoding
+        return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data).decode('utf-8'))
+
+
+class CertificateClone(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificateExport, self).__init__()
+
+    @validate_schema(None, certificate_output_schema)
+    def get(self, certificate_id):
+
+        pass
+
+
 api.add_resource(CertificatesList, '/certificates', endpoint='certificates')
 api.add_resource(Certificates, '/certificates/<int:certificate_id>', endpoint='certificate')
 api.add_resource(CertificatesStats, '/certificates/stats', endpoint='certificateStats')
 api.add_resource(CertificatesUpload, '/certificates/upload', endpoint='certificateUpload')
 api.add_resource(CertificatePrivateKey, '/certificates/<int:certificate_id>/key', endpoint='privateKeyCertificates')
-api.add_resource(NotificationCertificatesList, '/notifications/<int:notification_id>/certificates', endpoint='notificationCertificates')
+api.add_resource(CertificateExport, '/certificates/<int:certificate_id>/export', endpoint='exportCertificate')
+api.add_resource(CertificateClone, '/certificates/<int:certificate_id>/clone', endpoint='cloneCertificate')
+api.add_resource(NotificationCertificatesList, '/notifications/<int:notification_id>/certificates',
+                 endpoint='notificationCertificates')
+api.add_resource(CertificatesReplacementsList, '/certificates/<int:certificate_id>/replacements',
+                 endpoint='replacements')

lemur/common/defaults.py

@@ -0,0 +1,169 @@
+from cryptography import x509
+from flask import current_app
+from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
+
+
+def certificate_name(common_name, issuer, not_before, not_after, san):
+    """
+    Create a name for our certificate. A naming standard
+    is based on a series of templates. The name includes
+    useful information such as Common Name, Validation dates,
+    and Issuer.
+
+    :param san:
+    :param common_name:
+    :param not_after:
+    :param issuer:
+    :param not_before:
+    :rtype: str
+    :return:
+    """
+    if san:
+        t = SAN_NAMING_TEMPLATE
+    else:
+        t = DEFAULT_NAMING_TEMPLATE
+
+    temp = t.format(
+        subject=common_name,
+        issuer=issuer,
+        not_before=not_before.strftime('%Y%m%d'),
+        not_after=not_after.strftime('%Y%m%d')
+    )
+
+    disallowed_chars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
+    disallowed_chars = disallowed_chars.replace("-", "")
+    disallowed_chars = disallowed_chars.replace(".", "")
+    temp = temp.replace('*', "WILDCARD")
+
+    for c in disallowed_chars:
+        temp = temp.replace(c, "")
+
+    # white space is silly too
+    return temp.replace(" ", "-")
+
+
+def signing_algorithm(cert):
+    return cert.signature_hash_algorithm.name
+
+
+def common_name(cert):
+    """
+    Attempts to get a sane common name from a given certificate.
+
+    :param cert:
+    :return: Common name or None
+    """
+    return cert.subject.get_attributes_for_oid(
+        x509.OID_COMMON_NAME
+    )[0].value.strip()
+
+
+def domains(cert):
+    """
+    Attempts to get an domains listed in a certificate.
+    If 'subjectAltName' extension is not available we simply
+    return the common name.
+
+    :param cert:
+    :return: List of domains
+    """
+    domains = []
+    try:
+        ext = cert.extensions.get_extension_for_oid(x509.OID_SUBJECT_ALTERNATIVE_NAME)
+        entries = ext.value.get_values_for_type(x509.DNSName)
+        for entry in entries:
+            domains.append(entry)
+    except Exception as e:
+        current_app.logger.warning("Failed to get SubjectAltName: {0}".format(e))
+
+    return domains
+
+
+def serial(cert):
+    """
+    Fetch the serial number from the certificate.
+
+    :param cert:
+    :return: serial number
+    """
+    return cert.serial
+
+
+def san(cert):
+    """
+    Determines if a given certificate is a SAN certificate.
+    SAN certificates are simply certificates that cover multiple domains.
+
+    :param cert:
+    :return: Bool
+    """
+    if len(domains(cert)) > 1:
+        return True
+
+
+def is_wildcard(cert):
+    """
+    Determines if certificate is a wildcard certificate.
+
+    :param cert:
+    :return: Bool
+    """
+    d = domains(cert)
+    if len(d) == 1 and d[0][0:1] == "*":
+        return True
+
+    if cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[0].value[0:1] == "*":
+        return True
+
+
+def bitstrength(cert):
+    """
+    Calculates a certificates public key bit length.
+
+    :param cert:
+    :return: Integer
+    """
+    try:
+        return cert.public_key().key_size
+    except AttributeError:
+        current_app.logger.debug('Unable to get bitstrength.')
+
+
+def issuer(cert):
+    """
+    Gets a sane issuer from a given certificate.
+
+    :param cert:
+    :return: Issuer
+    """
+    delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
+    try:
+        issuer = str(cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[0].value)
+        for c in delchars:
+            issuer = issuer.replace(c, "")
+        return issuer
+    except Exception as e:
+        current_app.logger.error("Unable to get issuer! {0}".format(e))
+
+
+def not_before(cert):
+    """
+    Gets the naive datetime of the certificates 'not_before' field.
+    This field denotes the first date in time which the given certificate
+    is valid.
+
+    :param cert:
+    :return: Datetime
+    """
+    return cert.not_valid_before
+
+
+def not_after(cert):
+    """
+    Gets the naive datetime of the certificates 'not_after' field.
+    This field denotes the last date in time which the given certificate
+    is valid.
+
+    :return: Datetime
+    """
+    return cert.not_valid_after

lemur/common/fields.py

@@ -0,0 +1,93 @@
+import arrow
+import warnings
+from datetime import datetime as dt
+from marshmallow.fields import Field
+from marshmallow import utils
+
+
+class ArrowDateTime(Field):
+    """A formatted datetime string in UTC.
+
+    Example: ``'2014-12-22T03:12:58.019077+00:00'``
+
+    Timezone-naive `datetime` objects are converted to
+    UTC (+00:00) by :meth:`Schema.dump <marshmallow.Schema.dump>`.
+    :meth:`Schema.load <marshmallow.Schema.load>` returns `datetime`
+    objects that are timezone-aware.
+
+    :param str format: Either ``"rfc"`` (for RFC822), ``"iso"`` (for ISO8601),
+        or a date format string. If `None`, defaults to "iso".
+    :param kwargs: The same keyword arguments that :class:`Field` receives.
+
+    """
+
+    DATEFORMAT_SERIALIZATION_FUNCS = {
+        'iso': utils.isoformat,
+        'iso8601': utils.isoformat,
+        'rfc': utils.rfcformat,
+        'rfc822': utils.rfcformat,
+    }
+
+    DATEFORMAT_DESERIALIZATION_FUNCS = {
+        'iso': utils.from_iso,
+        'iso8601': utils.from_iso,
+        'rfc': utils.from_rfc,
+        'rfc822': utils.from_rfc,
+    }
+
+    DEFAULT_FORMAT = 'iso'
+
+    localtime = False
+    default_error_messages = {
+        'invalid': 'Not a valid datetime.',
+        'format': '"{input}" cannot be formatted as a datetime.',
+    }
+
+    def __init__(self, format=None, **kwargs):
+        super(ArrowDateTime, self).__init__(**kwargs)
+        # Allow this to be None. It may be set later in the ``_serialize``
+        # or ``_desrialize`` methods This allows a Schema to dynamically set the
+        # dateformat, e.g. from a Meta option
+        self.dateformat = format
+
+    def _add_to_schema(self, field_name, schema):
+        super(ArrowDateTime, self)._add_to_schema(field_name, schema)
+        self.dateformat = self.dateformat or schema.opts.dateformat
+
+    def _serialize(self, value, attr, obj):
+        if value is None:
+            return None
+        self.dateformat = self.dateformat or self.DEFAULT_FORMAT
+        format_func = self.DATEFORMAT_SERIALIZATION_FUNCS.get(self.dateformat, None)
+        if format_func:
+            try:
+                return format_func(value, localtime=self.localtime)
+            except (AttributeError, ValueError) as err:
+                self.fail('format', input=value)
+        else:
+            return value.strftime(self.dateformat)
+
+    def _deserialize(self, value, attr, data):
+        if not value:  # Falsy values, e.g. '', None, [] are not valid
+            raise self.fail('invalid')
+        self.dateformat = self.dateformat or self.DEFAULT_FORMAT
+        func = self.DATEFORMAT_DESERIALIZATION_FUNCS.get(self.dateformat)
+        if func:
+            try:
+                return arrow.get(func(value))
+            except (TypeError, AttributeError, ValueError):
+                raise self.fail('invalid')
+        elif self.dateformat:
+            try:
+                return dt.datetime.strptime(value, self.dateformat)
+            except (TypeError, AttributeError, ValueError):
+                raise self.fail('invalid')
+        elif utils.dateutil_available:
+            try:
+                return arrow.get(utils.from_datestring(value))
+            except TypeError:
+                raise self.fail('invalid')
+        else:
+            warnings.warn('It is recommended that you install python-dateutil '
+                          'for improved datetime deserialization.')
+            raise self.fail('invalid')

lemur/common/managers.py

@@ -58,8 +58,8 @@ class InstanceManager(object):
                     results.append(cls())
                 else:
                     results.append(cls)
-            except Exception:
-                current_app.logger.exception('Unable to import %s', cls_path)
+            except Exception as e:
+                current_app.logger.exception('Unable to import %s. Reason: %s', cls_path, e)
                 continue
         self.cache = results
 

lemur/common/missing.py

@@ -0,0 +1,24 @@
+import arrow
+from flask import current_app
+
+from lemur.common.utils import is_weekend
+
+
+def convert_validity_years(data):
+    """
+    Convert validity years to validity_start and validity_end
+
+    :param data:
+    :return:
+    """
+    if data.get('validity_years'):
+        now = arrow.utcnow()
+        data['validity_start'] = now.isoformat()
+
+        end = now.replace(years=+int(data['validity_years']))
+        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+            if is_weekend(end):
+                end = end.replace(days=-2)
+
+        data['validity_end'] = end.isoformat()
+    return data

lemur/common/schema.py

@@ -0,0 +1,153 @@
+"""
+.. module: lemur.common.schema
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+
+"""
+from functools import wraps
+from flask import request, current_app
+
+from sqlalchemy.orm.collections import InstrumentedList
+
+from inflection import camelize, underscore
+from marshmallow import Schema, post_dump, pre_load, pre_dump
+
+
+class LemurSchema(Schema):
+    """
+    Base schema from which all grouper schema's inherit
+    """
+    __envelope__ = True
+
+    def under(self, data, many=None):
+        items = []
+        if many:
+            for i in data:
+                items.append(
+                    {underscore(key): value for key, value in i.items()}
+                )
+            return items
+        return {
+            underscore(key): value
+            for key, value in data.items()
+        }
+
+    def camel(self, data, many=None):
+        items = []
+        if many:
+            for i in data:
+                items.append(
+                    {camelize(key, uppercase_first_letter=False): value for key, value in i.items()}
+                )
+            return items
+        return {
+            camelize(key, uppercase_first_letter=False): value
+            for key, value in data.items()
+        }
+
+    def wrap_with_envelope(self, data, many):
+        if many:
+            if 'total' in self.context.keys():
+                return dict(total=self.context['total'], items=data)
+        return data
+
+
+class LemurInputSchema(LemurSchema):
+    @pre_load(pass_many=True)
+    def preprocess(self, data, many):
+        return self.under(data, many=many)
+
+
+class LemurOutputSchema(LemurSchema):
+    @pre_load(pass_many=True)
+    def preprocess(self, data, many):
+        if many:
+            data = self.unwrap_envelope(data, many)
+        return self.under(data, many=many)
+
+    @pre_dump(pass_many=True)
+    def unwrap_envelope(self, data, many):
+        if many:
+            if data:
+                if isinstance(data, InstrumentedList) or isinstance(data, list):
+                    self.context['total'] = len(data)
+                    return data
+                else:
+                    self.context['total'] = data['total']
+            else:
+                self.context['total'] = 0
+                data = {'items': []}
+
+            return data['items']
+
+        return data
+
+    @post_dump(pass_many=True)
+    def post_process(self, data, many):
+        if data:
+            data = self.camel(data, many=many)
+        if self.__envelope__:
+            return self.wrap_with_envelope(data, many=many)
+        else:
+            return data
+
+
+def format_errors(messages):
+    errors = {}
+    for k, v in messages.items():
+        key = camelize(k, uppercase_first_letter=False)
+        if isinstance(v, dict):
+            errors[key] = format_errors(v)
+        elif isinstance(v, list):
+            errors[key] = v[0]
+    return errors
+
+
+def wrap_errors(messages):
+    errors = dict(message='Validation Error.')
+    if messages.get('_schema'):
+        errors['reasons'] = {'Schema': {'rule': messages['_schema']}}
+    else:
+        errors['reasons'] = format_errors(messages)
+    return errors
+
+
+def validate_schema(input_schema, output_schema):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            if input_schema:
+                if request.get_json():
+                    request_data = request.get_json()
+                else:
+                    request_data = request.args
+
+                data, errors = input_schema.load(request_data)
+
+                if errors:
+                    return wrap_errors(errors), 400
+
+                kwargs['data'] = data
+
+            try:
+                resp = f(*args, **kwargs)
+            except Exception as e:
+                current_app.logger.exception(e)
+                return dict(message=str(e)), 500
+
+            if isinstance(resp, tuple):
+                return resp[0], resp[1]
+
+            if not resp:
+                return dict(message="No data found"), 404
+
+            if output_schema:
+                data = output_schema.dump(resp)
+                return data.data, 200
+            return resp, 200
+
+        return decorated_function
+    return decorator

lemur/common/utils.py

@@ -6,15 +6,23 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import six
+import sys
 import string
 import random
-from functools import wraps
 
-from flask import current_app
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
 
-from flask.ext.restful import marshal
 from flask.ext.restful.reqparse import RequestParser
-from flask.ext.sqlalchemy import Pagination
+
+paginated_parser = RequestParser()
+
+paginated_parser.add_argument('count', type=int, default=10, location='args')
+paginated_parser.add_argument('page', type=int, default=1, location='args')
+paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
+paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
+paginated_parser.add_argument('filter', type=str, location='args')
 
 
 def get_psuedo_random_string():
@@ -28,51 +36,22 @@ def get_psuedo_random_string():
     return challenge
 
 
-class marshal_items(object):
-    def __init__(self, fields, envelope=None):
-        self.fields = fields
-        self.envelop = envelope
+def parse_certificate(body):
+    if sys.version_info[0] <= 2:
+        return x509.load_pem_x509_certificate(bytes(body), default_backend())
 
-    def __call__(self, f):
-        def _filter_items(items):
-            filtered_items = []
-            for item in items:
-                filtered_items.append(marshal(item, self.fields))
-            return filtered_items
+    if isinstance(body, six.string_types):
+        body = body.encode('utf-8')
 
-        @wraps(f)
-        def wrapper(*args, **kwargs):
-            try:
-                resp = f(*args, **kwargs)
-
-                # this is a bit weird way to handle non standard error codes returned from the marshaled function
-                if isinstance(resp, tuple):
-                    return resp[0], resp[1]
-
-                if isinstance(resp, Pagination):
-                    return {'items': _filter_items(resp.items), 'total': resp.total}
-
-                if isinstance(resp, list):
-                    return {'items': _filter_items(resp), 'total': len(resp)}
-
-                return marshal(resp, self.fields)
-            except Exception as e:
-                current_app.logger.exception(e)
-                # this is a little weird hack to respect flask restful parsing errors on marshaled functions
-                if hasattr(e, 'code'):
-                    if hasattr(e, 'data'):
-                        return {'message': e.data['message']}, 400
-                    else:
-                        return {'message': 'unknown'}, 400
-                else:
-                    return {'message': str(e)}, 400
-        return wrapper
+    return x509.load_pem_x509_certificate(body, default_backend())
 
 
-paginated_parser = RequestParser()
+def is_weekend(date):
+    """
+    Determines if a given date is on a weekend.
 
-paginated_parser.add_argument('count', type=int, default=10, location='args')
-paginated_parser.add_argument('page', type=int, default=1, location='args')
-paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
-paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
-paginated_parser.add_argument('filter', type=str, location='args')
+    :param date:
+    :return:
+    """
+    if date.weekday() > 5:
+        return True

lemur/common/validators.py

@@ -0,0 +1,118 @@
+import re
+
+from flask import current_app
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from marshmallow.exceptions import ValidationError
+
+from lemur.auth.permissions import SensitiveDomainPermission
+from lemur.common.utils import parse_certificate, is_weekend
+from lemur.domains import service as domain_service
+
+
+def public_certificate(body):
+    """
+    Determines if specified string is valid public certificate.
+
+    :param body:
+    :return:
+    """
+    try:
+        parse_certificate(body)
+    except Exception:
+        raise ValidationError('Public certificate presented is not valid.')
+
+
+def private_key(key):
+    """
+    User to validate that a given string is a RSA private key
+
+    :param key:
+    :return: :raise ValueError:
+    """
+    try:
+        if isinstance(key, bytes):
+            serialization.load_pem_private_key(key, None, backend=default_backend())
+        else:
+            serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
+    except Exception:
+        raise ValidationError('Private key presented is not valid.')
+
+
+def sensitive_domain(domain):
+    """
+    Determines if domain has been marked as sensitive.
+    :param domain:
+    :return:
+    """
+    restricted_domains = current_app.config.get('LEMUR_RESTRICTED_DOMAINS', [])
+    if restricted_domains:
+        domains = domain_service.get_by_name(domain)
+        for domain in domains:
+            # we only care about non-admins
+            if not SensitiveDomainPermission().can():
+                if domain.sensitive or any([re.match(pattern, domain.name) for pattern in restricted_domains]):
+                    raise ValidationError(
+                        'Domain {0} has been marked as sensitive, contact and administrator \
+                        to issue the certificate.'.format(domain))
+
+
+def encoding(oid_encoding):
+    """
+    Determines if the specified oid type is valid.
+    :param oid_encoding:
+    :return:
+    """
+    valid_types = ['b64asn1', 'string', 'ia5string']
+    if oid_encoding.lower() not in [o_type.lower() for o_type in valid_types]:
+        raise ValidationError('Invalid Oid Encoding: {0} choose from {1}'.format(oid_encoding, ",".join(valid_types)))
+
+
+def sub_alt_type(alt_type):
+    """
+    Determines if the specified subject alternate type is valid.
+    :param alt_type:
+    :return:
+    """
+    valid_types = ['DNSName', 'IPAddress', 'uniFormResourceIdentifier', 'directoryName', 'rfc822Name', 'registrationID',
+                   'otherName', 'x400Address', 'EDIPartyName']
+    if alt_type.lower() not in [a_type.lower() for a_type in valid_types]:
+        raise ValidationError('Invalid SubAltName Type: {0} choose from {1}'.format(type, ",".join(valid_types)))
+
+
+def csr(data):
+    """
+    Determines if the CSR is valid.
+    :param data:
+    :return:
+    """
+    try:
+        x509.load_pem_x509_csr(bytes(data), default_backend())
+    except Exception:
+        raise ValidationError('CSR presented is not valid.')
+
+
+def dates(data):
+    if not data.get('validity_start') and data.get('validity_end'):
+        raise ValidationError('If validity start is specified so must validity end.')
+
+    if not data.get('validity_end') and data.get('validity_start'):
+        raise ValidationError('If validity end is specified so must validity start.')
+
+    if data.get('validity_start') and data.get('validity_end'):
+        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+            if is_weekend(data.get('validity_end')):
+                raise ValidationError('Validity end must not land on a weekend.')
+
+        if not data['validity_start'] < data['validity_end']:
+            raise ValidationError('Validity start must be before validity end.')
+
+        if data.get('authority'):
+            if data.get('validity_start').date() < data['authority'].authority_certificate.not_before.date():
+                raise ValidationError('Validity start must not be before {0}'.format(data['authority'].authority_certificate.not_before))
+
+            if data.get('validity_end').date() > data['authority'].authority_certificate.not_after.date():
+                raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
+
+    return data

lemur/database.py

@@ -11,7 +11,7 @@
 """
 from sqlalchemy import exc
 from sqlalchemy.sql import and_, or_
-from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.orm import make_transient
 
 from lemur.extensions import db
 from lemur.exceptions import AttrNotFound, DuplicateError
@@ -123,10 +123,7 @@ def get(model, value, field="id"):
     :return:
     """
     query = session_query(model)
-    try:
-        return query.filter(getattr(model, field) == value).one()
-    except NoResultFound as e:
-        return
+    return query.filter(getattr(model, field) == value).scalar()
 
 
 def get_all(model, value, field="id"):
@@ -237,9 +234,6 @@ def update_list(model, model_attr, item_model, items):
     """
     ids = []
 
-    for i in items:
-        ids.append(i['id'])
-
     for i in getattr(model, model_attr):
         if i.id not in ids:
             getattr(model, model_attr).remove(i)
@@ -254,6 +248,18 @@ def update_list(model, model_attr, item_model, items):
     return model
 
 
+def clone(model):
+    """
+    Clones the given model and removes it's primary key
+    :param model:
+    :return:
+    """
+    db.session.expunge(model)
+    make_transient(model)
+    model.id = None
+    return model
+
+
 def sort_and_page(query, model, args):
     """
     Helper that allows us to combine sorting and paging
@@ -273,4 +279,9 @@ def sort_and_page(query, model, args):
     if sort_by and sort_dir:
         query = sort(query, model, sort_by, sort_dir)
 
-    return paginate(query, page, count)
+    total = query.count()
+
+    # offset calculated at zero
+    page -= 1
+    items = query.offset(count * page).limit(count).all()
+    return dict(items=items, total=total)

lemur/default.conf.py

@@ -3,8 +3,6 @@
 import os
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
-ADMINS = frozenset([''])
-
 THREADS_PER_PAGE = 8
 
 # General

lemur/defaults/views.py

@@ -57,7 +57,8 @@ class LemurDefaults(AuthenticatedResource):
             state=current_app.config.get('LEMUR_DEFAULT_STATE'),
             location=current_app.config.get('LEMUR_DEFAULT_LOCATION'),
             organization=current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'),
-            organizationalUnit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT')
+            organizationalUnit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'),
+            issuerPlugin=current_app.config.get('LEMUR_DEFAULT_ISSUER_PLUGIN')
         )
 
 api.add_resource(LemurDefaults, '/defaults', endpoint='default')

lemur/destinations/models.py

@@ -5,7 +5,6 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import copy
 from sqlalchemy import Column, Integer, String, Text
 from sqlalchemy_utils import JSONType
 from lemur.database import db
@@ -23,7 +22,7 @@ class Destination(db.Model):
 
     @property
     def plugin(self):
-        p = plugins.get(self.plugin_name)
-        c = copy.deepcopy(p)
-        c.options = self.options
-        return c
+        return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Destination(label={label})".format(label=self.label)

lemur/destinations/schemas.py

@@ -0,0 +1,42 @@
+"""
+.. module: lemur.destinations.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+
+from marshmallow import fields, post_dump
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.schemas import PluginInputSchema, PluginOutputSchema
+
+
+class DestinationInputSchema(LemurInputSchema):
+    id = fields.Integer()
+    label = fields.String(required=True)
+    description = fields.String(required=True)
+    active = fields.Boolean()
+    plugin = fields.Nested(PluginInputSchema, required=True)
+
+
+class DestinationOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    label = fields.String()
+    description = fields.String()
+    active = fields.Boolean()
+    plugin = fields.Nested(PluginOutputSchema)
+    options = fields.List(fields.Dict())
+
+    @post_dump
+    def fill_object(self, data):
+        data['plugin']['pluginOptions'] = data['options']
+        return data
+
+
+class DestinationNestedOutputSchema(DestinationOutputSchema):
+    __envelope__ = False
+
+
+destination_input_schema = DestinationInputSchema()
+destinations_output_schema = DestinationOutputSchema(many=True)
+destination_output_schema = DestinationOutputSchema()

lemur/destinations/service.py

@@ -86,10 +86,6 @@ def get_all():
 
 
 def render(args):
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
     certificate_id = args.pop('certificate_id', None)
 
@@ -103,12 +99,7 @@ def render(args):
         terms = filt.split(';')
         query = database.filter(query, Destination, terms)
 
-    query = database.find_all(query, Destination, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Destination, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)
+    return database.sort_and_page(query, Destination, args)
 
 
 def stats(**kwargs):

lemur/destinations/views.py

@@ -7,34 +7,28 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import Blueprint
-from flask.ext.restful import Api, reqparse, fields
+from flask.ext.restful import Api, reqparse
 from lemur.destinations import service
 
 from lemur.auth.service import AuthenticatedResource
 from lemur.auth.permissions import admin_permission
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.common.utils import paginated_parser
+
+from lemur.common.schema import validate_schema
+from lemur.destinations.schemas import destinations_output_schema, destination_input_schema, destination_output_schema
 
 
 mod = Blueprint('destinations', __name__)
 api = Api(mod)
 
 
-FIELDS = {
-    'description': fields.String,
-    'destinationOptions': fields.Raw(attribute='options'),
-    'pluginName': fields.String(attribute='plugin_name'),
-    'label': fields.String,
-    'id': fields.Integer,
-}
-
-
 class DestinationsList(AuthenticatedResource):
     """ Defines the 'destinations' endpoint """
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(DestinationsList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, destinations_output_schema)
     def get(self):
         """
         .. http:get:: /destinations
@@ -58,32 +52,40 @@ class DestinationsList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
-                    {
-                        "destinationOptions": [
-                            {
-                                "name": "accountNumber",
-                                "required": true,
-                                "value": 111111111112,
-                                "helpMessage": "Must be a valid AWS account number!",
-                                "validation": "/^[0-9]{12,12}$/",
-                                "type": "int"
-                            }
-                        ],
-                        "pluginName": "aws-destination",
-                        "id": 3,
-                        "description": "test",
-                        "label": "test"
-                    }
-                ],
+                "items": [{
+                    "description": "test",
+                    "options": [{
+                        "name": "accountNumber",
+                        "required": true,
+                        "value": "111111111111111",
+                        "helpMessage": "Must be a valid AWS account number!",
+                        "validation": "/^[0-9]{12,12}$/",
+                        "type": "str"
+                    }],
+                    "id": 4,
+                    "plugin": {
+                        "pluginOptions": [{
+                            "name": "accountNumber",
+                            "required": true,
+                            "value": "111111111111111",
+                            "helpMessage": "Must be a valid AWS account number!",
+                            "validation": "/^[0-9]{12,12}$/",
+                            "type": "str"
+                        }],
+                        "description": "Allow the uploading of certificates to AWS IAM",
+                        "slug": "aws-destination",
+                        "title": "AWS"
+                    },
+                    "label": "test546"
+                }
                 "total": 1
               }
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
            :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
@@ -92,8 +94,8 @@ class DestinationsList(AuthenticatedResource):
         return service.render(args)
 
     @admin_permission.require(http_exception=403)
-    @marshal_items(FIELDS)
-    def post(self):
+    @validate_schema(destination_input_schema, destination_output_schema)
+    def post(self, data=None):
         """
         .. http:post:: /destinations
 
@@ -108,20 +110,30 @@ class DestinationsList(AuthenticatedResource):
               Accept: application/json, text/javascript
 
               {
-                "destinationOptions": [
-                    {
+                "description": "test33",
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "34324324",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "34324324",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
-                    }
-                ],
-                "pluginName": "aws-destination",
-                "id": 3,
-                "description": "test",
-                "label": "test"
+                        "type": "str"
+                    }],
+                    "description": "Allow the uploading of certificates to AWS IAM",
+                    "slug": "aws-destination",
+                    "title": "AWS"
+                },
+                "label": "test546"
               }
 
            **Example response**:
@@ -133,20 +145,30 @@ class DestinationsList(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "destinationOptions": [
-                    {
+                "description": "test33",
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "34324324",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "111111111111111",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
-                    }
-                ],
-                "pluginName": "aws-destination",
-                "id": 3,
-                "description": "test",
-                "label": "test"
+                        "type": "str"
+                    }],
+                    "description": "Allow the uploading of certificates to AWS IAM",
+                    "slug": "aws-destination",
+                    "title": "AWS"
+                },
+                "label": "test546"
               }
 
            :arg label: human readable account label
@@ -154,12 +176,7 @@ class DestinationsList(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-
-        args = self.reqparse.parse_args()
-        return service.create(args['label'], args['plugin']['slug'], args['plugin']['pluginOptions'], args['description'])
+        return service.create(data['label'], data['plugin']['slug'], data['plugin']['plugin_options'], data['description'])
 
 
 class Destinations(AuthenticatedResource):
@@ -167,7 +184,7 @@ class Destinations(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Destinations, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, destination_output_schema)
     def get(self, destination_id):
         """
         .. http:get:: /destinations/1
@@ -191,20 +208,30 @@ class Destinations(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "destinationOptions": [
-                    {
+                "description": "test",
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "111111111111111",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "111111111111111",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
-                    }
-                ],
-                "pluginName": "aws-destination",
-                "id": 3,
-                "description": "test",
-                "label": "test"
+                        "type": "str"
+                    }],
+                    "description": "Allow the uploading of certificates to AWS IAM",
+                    "slug": "aws-destination",
+                    "title": "AWS"
+                },
+                "label": "test546"
               }
 
            :reqheader Authorization: OAuth token to authenticate
@@ -213,8 +240,8 @@ class Destinations(AuthenticatedResource):
         return service.get(destination_id)
 
     @admin_permission.require(http_exception=403)
-    @marshal_items(FIELDS)
-    def put(self, destination_id):
+    @validate_schema(destination_input_schema, destination_output_schema)
+    def put(self, destination_id, data=None):
         """
         .. http:put:: /destinations/1
 
@@ -228,23 +255,35 @@ class Destinations(AuthenticatedResource):
               Host: example.com
               Accept: application/json, text/javascript
 
+
               {
-                "destinationOptions": [
-                    {
+                "description": "test33",
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "34324324",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "34324324",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
-                    }
-                ],
-                "pluginName": "aws-destination",
-                "id": 3,
-                "description": "test",
-                "label": "test"
+                        "type": "str"
+                    }],
+                    "description": "Allow the uploading of certificates to AWS IAM",
+                    "slug": "aws-destination",
+                    "title": "AWS"
+                },
+                "label": "test546"
               }
 
+
            **Example response**:
 
            .. sourcecode:: http
@@ -254,20 +293,30 @@ class Destinations(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "destinationOptions": [
-                    {
+                "description": "test",
+                "options": [{
+                    "name": "accountNumber",
+                    "required": true,
+                    "value": "111111111111111",
+                    "helpMessage": "Must be a valid AWS account number!",
+                    "validation": "/^[0-9]{12,12}$/",
+                    "type": "str"
+                }],
+                "id": 4,
+                "plugin": {
+                    "pluginOptions": [{
                         "name": "accountNumber",
                         "required": true,
-                        "value": 111111111112,
+                        "value": "111111111111111",
                         "helpMessage": "Must be a valid AWS account number!",
                         "validation": "/^[0-9]{12,12}$/",
-                        "type": "int"
-                    }
-                ],
-                "pluginName": "aws-destination",
-                "id": 3,
-                "description": "test",
-                "label": "test"
+                        "type": "str"
+                    }],
+                    "description": "Allow the uploading of certificates to AWS IAM",
+                    "slug": "aws-destination",
+                    "title": "AWS"
+                },
+                "label": "test546"
               }
 
            :arg accountNumber: aws account number
@@ -276,12 +325,7 @@ class Destinations(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-
-        args = self.reqparse.parse_args()
-        return service.update(destination_id, args['label'], args['plugin']['pluginOptions'], args['description'])
+        return service.update(destination_id, data['label'], data['plugin']['plugin_options'], data['description'])
 
     @admin_permission.require(http_exception=403)
     def delete(self, destination_id):
@@ -294,7 +338,7 @@ class CertificateDestinations(AuthenticatedResource):
     def __init__(self):
         super(CertificateDestinations, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, destination_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/destinations
@@ -318,32 +362,40 @@ class CertificateDestinations(AuthenticatedResource):
               Content-Type: text/javascript
 
               {
-                "items": [
-                    {
-                        "destinationOptions": [
-                            {
-                                "name": "accountNumber",
-                                "required": true,
-                                "value": 111111111112,
-                                "helpMessage": "Must be a valid AWS account number!",
-                                "validation": "/^[0-9]{12,12}$/",
-                                "type": "int"
-                            }
-                        ],
-                        "pluginName": "aws-destination",
-                        "id": 3,
-                        "description": "test",
-                        "label": "test"
-                    }
-                ],
+                "items": [{
+                    "description": "test",
+                    "options": [{
+                        "name": "accountNumber",
+                        "required": true,
+                        "value": "111111111111111",
+                        "helpMessage": "Must be a valid AWS account number!",
+                        "validation": "/^[0-9]{12,12}$/",
+                        "type": "str"
+                    }],
+                    "id": 4,
+                    "plugin": {
+                        "pluginOptions": [{
+                            "name": "accountNumber",
+                            "required": true,
+                            "value": "111111111111111",
+                            "helpMessage": "Must be a valid AWS account number!",
+                            "validation": "/^[0-9]{12,12}$/",
+                            "type": "str"
+                        }],
+                        "description": "Allow the uploading of certificates to AWS IAM",
+                        "slug": "aws-destination",
+                        "title": "AWS"
+                    },
+                    "label": "test546"
+                }
                 "total": 1
               }
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
-           :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """

lemur/domains/models.py

@@ -7,7 +7,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, Integer, String, Boolean
 
 from lemur.database import db
 
@@ -16,11 +16,7 @@ class Domain(db.Model):
     __tablename__ = 'domains'
     id = Column(Integer, primary_key=True)
     name = Column(String(256))
+    sensitive = Column(Boolean, default=False)
 
-    def as_dict(self):
-        return {c.name: getattr(self, c.name) for c in self.__table__.columns}
-
-    def serialize(self):
-        blob = self.as_dict()
-        blob['certificates'] = [x.id for x in self.certificate]
-        return blob
+    def __repr__(self):
+        return "Domain(name={name})".format(name=self.name)

lemur/domains/schemas.py

@@ -0,0 +1,35 @@
+"""
+.. module: lemur.domains.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.schemas import AssociatedCertificateSchema
+
+# from lemur.certificates.schemas import CertificateNestedOutputSchema
+
+
+class DomainInputSchema(LemurInputSchema):
+    id = fields.Integer()
+    name = fields.String(required=True)
+    sensitive = fields.Boolean()
+    certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[])
+
+
+class DomainOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    name = fields.String()
+    sensitive = fields.Boolean()
+    # certificates = fields.Nested(CertificateNestedOutputSchema, many=True, missing=[])
+
+
+class DomainNestedOutputSchema(DomainOutputSchema):
+    __envelope__ = False
+
+
+domain_input_schema = DomainInputSchema()
+domain_output_schema = DomainOutputSchema()
+domains_output_schema = DomainOutputSchema(many=True)

lemur/domains/service.py

@@ -32,6 +32,43 @@ def get_all():
     return database.find_all(query, Domain, {}).all()
 
 
+def get_by_name(name):
+    """
+    Fetches domain by it's name
+
+    :param name:
+    :return:
+    """
+    return database.get_all(Domain, name, field="name").all()
+
+
+def create(name, sensitive):
+    """
+    Create a new domain
+
+    :param name:
+    :param sensitive:
+    :return:
+    """
+    domain = Domain(name=name, sensitive=sensitive)
+    return database.create(domain)
+
+
+def update(domain_id, name, sensitive):
+    """
+    Update an existing domain
+
+    :param domain_id:
+    :param name:
+    :param sensitive:
+    :return:
+    """
+    domain = get(domain_id)
+    domain.name = name
+    domain.sensitive = sensitive
+    database.update(domain)
+
+
 def render(args):
     """
     Helper to parse REST Api requests
@@ -40,11 +77,6 @@ def render(args):
     :return:
     """
     query = database.session_query(Domain).join(Certificate, Domain.certificate)
-
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
     certificate_id = args.pop('certificate_id', None)
 
@@ -55,9 +87,4 @@ def render(args):
     if certificate_id:
         query = query.filter(Certificate.id == certificate_id)
 
-    query = database.find_all(query, Domain, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Domain, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)
+    return database.sort_and_page(query, Domain, args)

lemur/domains/views.py

@@ -8,17 +8,16 @@
 
 """
 from flask import Blueprint
-from flask.ext.restful import reqparse, Api, fields
+from flask.ext.restful import reqparse, Api
 
 from lemur.domains import service
 from lemur.auth.service import AuthenticatedResource
+from lemur.auth.permissions import SensitiveDomainPermission
 
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.common.schema import validate_schema
+from lemur.common.utils import paginated_parser
 
-FIELDS = {
-    'id': fields.Integer,
-    'name': fields.String
-}
+from lemur.domains.schemas import domain_input_schema, domain_output_schema, domains_output_schema
 
 mod = Blueprint('domains', __name__)
 api = Api(mod)
@@ -29,7 +28,7 @@ class DomainsList(AuthenticatedResource):
     def __init__(self):
         super(DomainsList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, domains_output_schema)
     def get(self):
         """
         .. http:get:: /domains
@@ -57,10 +56,12 @@ class DomainsList(AuthenticatedResource):
                     {
                       "id": 1,
                       "name": "www.example.com",
+                      "sensitive": false
                     },
                     {
                       "id": 2,
                       "name": "www.example2.com",
+                      "sensitive": false
                     }
                   ]
                 "total": 2
@@ -68,9 +69,9 @@ class DomainsList(AuthenticatedResource):
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
-           :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number. default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
@@ -79,13 +80,58 @@ class DomainsList(AuthenticatedResource):
         args = parser.parse_args()
         return service.render(args)
 
+    @validate_schema(domain_input_schema, domain_output_schema)
+    def post(self, data=None):
+        """
+        .. http:post:: /domains
+
+           The current domain list
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /domains HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                "name": "www.example.com",
+                "sensitive": false
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "id": 1,
+                "name": "www.example.com",
+                "sensitive": false
+              }
+
+           :query sortBy: field to sort on
+           :query sortDir: acs or desc
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        return service.create(data['name'], data['sensitive'])
+
 
 class Domains(AuthenticatedResource):
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(Domains, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, domain_output_schema)
     def get(self, domain_id):
         """
         .. http:get:: /domains/1
@@ -111,6 +157,7 @@ class Domains(AuthenticatedResource):
               {
                   "id": 1,
                   "name": "www.example.com",
+                  "sensitive": false
               }
 
            :reqheader Authorization: OAuth token to authenticate
@@ -119,13 +166,56 @@ class Domains(AuthenticatedResource):
         """
         return service.get(domain_id)
 
+    @validate_schema(domain_input_schema, domain_output_schema)
+    def put(self, domain_id, data=None):
+        """
+        .. http:get:: /domains/1
+
+           update one domain
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /domains HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                  "name": "www.example.com",
+                  "sensitive": false
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                  "id": 1,
+                  "name": "www.example.com",
+                  "sensitive": false
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        if SensitiveDomainPermission().can():
+            return service.update(domain_id, data['name'], data['sensitive'])
+
+        return dict(message='You are not authorized to modify this domain'), 403
+
 
 class CertificateDomains(AuthenticatedResource):
     """ Defines the 'domains' endpoint """
     def __init__(self):
         super(CertificateDomains, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, domains_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/domains
@@ -153,10 +243,12 @@ class CertificateDomains(AuthenticatedResource):
                     {
                       "id": 1,
                       "name": "www.example.com",
+                      "sensitive": false
                     },
                     {
                       "id": 2,
                       "name": "www.example2.com",
+                      "sensitive": false
                     }
                   ]
                 "total": 2
@@ -164,9 +256,9 @@ class CertificateDomains(AuthenticatedResource):
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
-           :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated

lemur/endpoints/models.py

@@ -0,0 +1,86 @@
+"""
+.. module: lemur.endpoints.models
+    :platform: unix
+    :synopsis: This module contains all of the models need to create a authority within Lemur.
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from sqlalchemy.orm import relationship
+from sqlalchemy import Column, Integer, String, func, DateTime, PassiveDefault, Boolean, ForeignKey
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy.sql.expression import case
+
+from lemur.database import db
+
+from lemur.models import policies_ciphers
+
+
+BAD_CIPHERS = [
+    'Protocol-SSLv3',
+    'Protocol-SSLv2',
+    'Protocol-TLSv1'
+]
+
+
+class Cipher(db.Model):
+    __tablename__ = 'ciphers'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(128), nullable=False)
+
+    @hybrid_property
+    def deprecated(self):
+        return self.name in BAD_CIPHERS
+
+    @deprecated.expression
+    def deprecated(cls):
+        return case(
+            [
+                (cls.name in BAD_CIPHERS, True)
+            ],
+            else_=False
+        )
+
+
+class Policy(db.Model):
+    ___tablename__ = 'policies'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(128), nullable=True)
+    ciphers = relationship('Cipher', secondary=policies_ciphers, backref='policy')
+
+
+class Endpoint(db.Model):
+    __tablename__ = 'endpoints'
+    id = Column(Integer, primary_key=True)
+    owner = Column(String(128))
+    name = Column(String(128))
+    dnsname = Column(String(256))
+    type = Column(String(128))
+    active = Column(Boolean, default=True)
+    port = Column(Integer)
+    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
+    policy_id = Column(Integer, ForeignKey('policy.id'))
+    policy = relationship('Policy', backref='endpoint')
+    certificate_id = Column(Integer, ForeignKey('certificates.id'))
+    source_id = Column(Integer, ForeignKey('sources.id'))
+    sensitive = Column(Boolean, default=False)
+    source = relationship('Source', back_populates='endpoints')
+
+    @property
+    def issues(self):
+        issues = []
+
+        for cipher in self.policy.ciphers:
+            if cipher.deprecated:
+                issues.append({'name': 'deprecated cipher', 'value': '{0} has been deprecated consider removing it.'.format(cipher.name)})
+
+        if self.certificate.expired:
+            issues.append({'name': 'expired certificate', 'value': 'There is an expired certificate attached to this endpoint consider replacing it.'})
+
+        if self.certificate.revoked:
+            issues.append({'name': 'revoked', 'value': 'There is a revoked certificate attached to this endpoint consider replacing it.'})
+
+        return issues
+
+    def __repr__(self):
+        return "Endpoint(name={name})".format(name=self.name)

lemur/endpoints/schemas.py

@@ -0,0 +1,43 @@
+"""
+.. module: lemur.endpoints.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+
+from lemur.common.schema import LemurOutputSchema
+from lemur.certificates.schemas import CertificateNestedOutputSchema
+
+
+class CipherNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    deprecated = fields.Boolean()
+    name = fields.String()
+
+
+class PolicyNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    name = fields.String()
+    ciphers = fields.Nested(CipherNestedOutputSchema, many=True)
+
+
+class EndpointOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    dnsname = fields.String()
+    owner = fields.Email()
+    type = fields.String()
+    port = fields.Integer()
+    active = fields.Boolean()
+    certificate = fields.Nested(CertificateNestedOutputSchema)
+    policy = fields.Nested(PolicyNestedOutputSchema)
+
+    issues = fields.List(fields.Dict())
+
+endpoint_output_schema = EndpointOutputSchema()
+endpoints_output_schema = EndpointOutputSchema(many=True)

lemur/endpoints/service.py

@@ -0,0 +1,143 @@
+"""
+.. module: lemur.endpoints.service
+    :platform: Unix
+    :synopsis: This module contains all of the services level functions used to
+    administer endpoints in Lemur
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+
+"""
+from lemur import database
+from lemur.extensions import metrics
+from lemur.endpoints.models import Endpoint, Policy, Cipher
+
+from sqlalchemy import func
+
+
+def get_all():
+    """
+    Get all endpoints that are currently in Lemur.
+
+    :rtype : List
+    :return:
+    """
+    query = database.session_query(Endpoint)
+    return database.find_all(query, Endpoint, {}).all()
+
+
+def get(endpoint_id):
+    """
+    Retrieves an endpoint given it's ID
+
+    :param endpoint_id:
+    :return:
+    """
+    return database.get(Endpoint, endpoint_id)
+
+
+def get_by_dnsname(endpoint_dnsname):
+    """
+    Retrieves an endpoint given it's name.
+
+    :param endpoint_dnsname:
+    :return:
+    """
+    return database.get(Endpoint, endpoint_dnsname, field='dnsname')
+
+
+def get_by_source(source_label):
+    """
+    Retrieves all endpoints for a given source.
+    :param source_label:
+    :return:
+    """
+    return Endpoint.query.filter(Endpoint.source.label == source_label).all()  # noqa
+
+
+def create(**kwargs):
+    """
+    Creates a new endpoint.
+    :param kwargs:
+    :return:
+    """
+    endpoint = Endpoint(**kwargs)
+    database.create(endpoint)
+    metrics.send('endpoint_added', 'counter', 1)
+    return endpoint
+
+
+def get_or_create_policy(**kwargs):
+    policy = database.get(Policy, kwargs['name'], field='name')
+
+    if not policy:
+        policy = Policy(**kwargs)
+        database.create(policy)
+
+    return policy
+
+
+def get_or_create_cipher(**kwargs):
+    cipher = database.get(Cipher, kwargs['name'], field='name')
+
+    if not cipher:
+        cipher = Cipher(**kwargs)
+        database.create(cipher)
+
+    return cipher
+
+
+def update(endpoint_id, **kwargs):
+    endpoint = database.get(Endpoint, endpoint_id)
+
+    endpoint.policy = kwargs['policy']
+    endpoint.certificate = kwargs['certificate']
+    database.update(endpoint)
+    return endpoint
+
+
+def render(args):
+    """
+    Helper that helps us render the REST Api responses.
+    :param args:
+    :return:
+    """
+    query = database.session_query(Endpoint)
+    filt = args.pop('filter')
+
+    if filt:
+        terms = filt.split(';')
+        if 'active' in filt:  # this is really weird but strcmp seems to not work here??
+            query = query.filter(Endpoint.active == terms[1])
+        elif 'port' in filt:
+            if terms[1] != 'null':  # ng-table adds 'null' if a number is removed
+                query = query.filter(Endpoint.port == terms[1])
+        elif 'ciphers' in filt:
+            query = query.filter(
+                Cipher.name == terms[1]
+            )
+        else:
+            query = database.filter(query, Endpoint, terms)
+
+    return database.sort_and_page(query, Endpoint, args)
+
+
+def stats(**kwargs):
+    """
+    Helper that defines some useful statistics about endpoints.
+
+    :param kwargs:
+    :return:
+    """
+    attr = getattr(Endpoint, kwargs.get('metric'))
+    query = database.db.session.query(attr, func.count(attr))
+
+    items = query.group_by(attr).all()
+
+    keys = []
+    values = []
+    for key, count in items:
+        keys.append(key)
+        values.append(count)
+
+    return {'labels': keys, 'values': values}

lemur/endpoints/views.py

@@ -0,0 +1,106 @@
+"""
+.. module: lemur.endpoints.views
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import Blueprint
+from flask.ext.restful import reqparse, Api
+
+from lemur.common.utils import paginated_parser
+from lemur.common.schema import validate_schema
+from lemur.auth.service import AuthenticatedResource
+
+from lemur.endpoints import service
+from lemur.endpoints.schemas import endpoint_output_schema, endpoints_output_schema
+
+
+mod = Blueprint('endpoints', __name__)
+api = Api(mod)
+
+
+class EndpointsList(AuthenticatedResource):
+    """ Defines the 'endpoints' endpoint """
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(EndpointsList, self).__init__()
+
+    @validate_schema(None, endpoints_output_schema)
+    def get(self):
+        """
+        .. http:get:: /endpoints
+
+           The current list of endpoints
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /endpoints HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+
+           :query sortBy: field to sort on
+           :query sortDir: acs or desc
+           :query page: int default is 1
+           :query filter: key value pair. format is k;v
+           :query limit: limit number default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+           :note: this will only show certificates that the current user is authorized to use
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        return service.render(args)
+
+
+class Endpoints(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(Endpoints, self).__init__()
+
+    @validate_schema(None, endpoint_output_schema)
+    def get(self, endpoint_id):
+        """
+        .. http:get:: /endpoints/1
+
+           One endpoint
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /endpoints/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        return service.get(endpoint_id)
+
+
+api.add_resource(EndpointsList, '/endpoints', endpoint='endpoints')
+api.add_resource(Endpoints, '/endpoints/<int:endpoint_id>', endpoint='endpoint')

lemur/exceptions.py

@@ -36,6 +36,14 @@ class IntegrityError(LemurException):
         return repr(self.message)
 
 
+class AssociatedObjectNotFound(LemurException):
+    def __init__(self, message):
+        self.message = message
+
+    def __str__(self):
+        return repr(self.message)
+
+
 class InvalidListener(LemurException):
     def __str__(self):
         return repr("Invalid listener, ensure you select a certificate if you are using a secure protocol")

lemur/extensions.py

@@ -3,17 +3,20 @@
     :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 """
-from flask.ext.sqlalchemy import SQLAlchemy
+from flask_sqlalchemy import SQLAlchemy
 db = SQLAlchemy()
 
-from flask.ext.migrate import Migrate
+from flask_migrate import Migrate
 migrate = Migrate()
 
-from flask.ext.bcrypt import Bcrypt
+from flask_bcrypt import Bcrypt
 bcrypt = Bcrypt()
 
-from flask.ext.principal import Principal
+from flask_principal import Principal
 principal = Principal()
 
 from flask_mail import Mail
 smtp_mail = Mail()
+
+from lemur.metrics import Metrics
+metrics = Metrics()

lemur/factory.py

@@ -14,12 +14,12 @@ import imp
 import errno
 import pkg_resources
 
-from logging import Formatter
+from logging import Formatter, StreamHandler
 from logging.handlers import RotatingFileHandler
 
 from flask import Flask
 from lemur.common.health import mod as health
-from lemur.extensions import db, migrate, principal, smtp_mail
+from lemur.extensions import db, migrate, principal, smtp_mail, metrics
 
 
 DEFAULT_BLUEPRINTS = (
@@ -90,12 +90,15 @@ def configure_app(app, config=None):
     :param config:
     :return:
     """
+    # respect the config first
+    if config and config != 'None':
+        app.config.from_object(from_file(config))
+
     try:
         app.config.from_envvar("LEMUR_CONF")
     except RuntimeError:
-        if config and config != 'None':
-            app.config.from_object(from_file(config))
-        elif os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
+        # look in default paths
+        if os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
             app.config.from_object(from_file(os.path.expanduser("~/.lemur/lemur.conf.py")))
         else:
             app.config.from_object(from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'default.conf.py')))
@@ -112,6 +115,7 @@ def configure_extensions(app):
     migrate.init_app(app, db)
     principal.init_app(app)
     smtp_mail.init_app(app)
+    metrics.init_app(app)
 
 
 def configure_blueprints(app, blueprints):
@@ -143,14 +147,19 @@ def configure_logging(app):
     app.logger.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
     app.logger.addHandler(handler)
 
+    stream_handler = StreamHandler()
+    stream_handler.setLevel(app.config.get('LOG_LEVEL'))
+    app.logger.addHandler(stream_handler)
+
 
 def install_plugins(app):
     """
     Installs new issuers that are not currently bundled with Lemur.
 
-    :param settings:
+    :param app:
     :return:
     """
+    from lemur.plugins import plugins
     from lemur.plugins.base import register
     # entry_points={
     #    'lemur.plugins': [
@@ -165,3 +174,11 @@ def install_plugins(app):
             app.logger.error("Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc()))
         else:
             register(plugin)
+
+    # ensure that we have some way to notify
+    with app.app_context():
+        try:
+            slug = app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
+            plugins.get(slug)
+        except KeyError:
+            raise Exception("Unable to location notification plugin: {slug}. Ensure that LEMUR_DEFAULT_NOTIFICATION_PLUGIN is set to a valid and installed notification plugin.".format(slug=slug))

lemur/manage.py

@@ -1,31 +1,39 @@
 from __future__ import unicode_literals    # at top of module
 
+from datetime import datetime, timedelta
+from collections import Counter
+
 import os
 import sys
 import base64
 import time
 import requests
 import json
+
+from tabulate import tabulate
 from gunicorn.config import make_settings
 
 from cryptography.fernet import Fernet
 
-from lockfile import LockFile, LockTimeout
-
 from flask import current_app
 from flask.ext.script import Manager, Command, Option, prompt_pass
 from flask.ext.migrate import Migrate, MigrateCommand, stamp
 from flask_script.commands import ShowUrls, Clean, Server
 
 from lemur import database
+from lemur.extensions import metrics
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
 from lemur.certificates import service as cert_service
-from lemur.sources import service as source_service
+from lemur.authorities import service as authority_service
 from lemur.notifications import service as notification_service
 
+from lemur.certificates.service import get_name_from_arn
 from lemur.certificates.verify import verify_string
-from lemur.sources.service import sync
+
+from lemur.plugins.lemur_aws import elb
+
+from lemur.sources import service as source_service
 
 from lemur import create_app
 
@@ -57,8 +65,6 @@ CONFIG_TEMPLATE = """
 import os
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
-ADMINS = frozenset([''])
-
 THREADS_PER_PAGE = 8
 
 # General
@@ -90,6 +96,8 @@ LEMUR_DEFAULT_LOCATION = ''
 LEMUR_DEFAULT_ORGANIZATION = ''
 LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = ''
 
+# Authentication Providers
+ACTIVE_PROVIDERS = []
 
 # Logging
 
@@ -181,58 +189,6 @@ def generate_settings():
     return output
 
 
-@manager.option('-s', '--sources', dest='labels')
-def sync_sources(labels):
-    """
-    Attempts to run several methods Certificate discovery. This is
-    run on a periodic basis and updates the Lemur datastore with the
-    information it discovers.
-    """
-    if not labels:
-        sys.stdout.write("Active\tLabel\tDescription\n")
-        for source in source_service.get_all():
-            sys.stdout.write(
-                "{active}\t{label}\t{description}!\n".format(
-                    label=source.label,
-                    description=source.description,
-                    active=source.active
-                )
-            )
-    else:
-        start_time = time.time()
-        lock_file = "/tmp/.lemur_lock"
-        sync_lock = LockFile(lock_file)
-
-        while not sync_lock.i_am_locking():
-            try:
-                sync_lock.acquire(timeout=10)    # wait up to 10 seconds
-
-                sys.stdout.write("[+] Staring to sync sources: {labels}!\n".format(labels=labels))
-                labels = labels.split(",")
-
-                if labels[0] == 'all':
-                    sync()
-                else:
-                    sync(labels=labels)
-
-                sys.stdout.write(
-                    "[+] Finished syncing sources. Run Time: {time}\n".format(
-                        time=(time.time() - start_time)
-                    )
-                )
-            except LockTimeout:
-                sys.stderr.write(
-                    "[!] Unable to acquire file lock on {file}, is there another sync running?\n".format(
-                        file=lock_file
-                    )
-                )
-                sync_lock.break_lock()
-                sync_lock.acquire()
-                sync_lock.release()
-
-        sync_lock.release()
-
-
 @manager.command
 def notify():
     """
@@ -310,7 +266,7 @@ class InitializeApp(Command):
 
 class CreateUser(Command):
     """
-    This command allows for the creation of a new user within Lemur
+    This command allows for the creation of a new user within Lemur.
     """
     option_list = (
         Option('-u', '--username', dest='username', required=True),
@@ -326,18 +282,46 @@ class CreateUser(Command):
             if role_obj:
                 role_objs.append(role_obj)
             else:
-                sys.stderr.write("[!] Cannot find role {0}".format(r))
+                sys.stderr.write("[!] Cannot find role {0}\n".format(r))
                 sys.exit(1)
 
         password1 = prompt_pass("Password")
         password2 = prompt_pass("Confirm Password")
 
         if password1 != password2:
-            sys.stderr.write("[!] Passwords do not match")
+            sys.stderr.write("[!] Passwords do not match!\n")
             sys.exit(1)
 
         user_service.create(username, password1, email, active, None, role_objs)
-        sys.stdout.write("[+] Created new user: {0}".format(username))
+        sys.stdout.write("[+] Created new user: {0}\n".format(username))
+
+
+class ResetPassword(Command):
+    """
+    This command allows you to reset a user's password.
+    """
+    option_list = (
+        Option('-u', '--username', dest='username', required=True),
+    )
+
+    def run(self, username):
+        user = user_service.get_by_username(username)
+
+        if not user:
+            sys.stderr.write("[!] No user found for username: {0}\n".format(username))
+            sys.exit(1)
+
+        sys.stderr.write("[+] Resetting password for {0}\n".format(username))
+        password1 = prompt_pass("Password")
+        password2 = prompt_pass("Confirm Password")
+
+        if password1 != password2:
+            sys.stderr.write("[!] Passwords do not match\n")
+            sys.exit(1)
+
+        user.password = password1
+        user.hash_password()
+        database.commit()
 
 
 class CreateRole(Command):
@@ -384,7 +368,7 @@ class LemurServer(Command):
         settings = make_settings()
         options = (
             Option(*klass.cli, action=klass.action)
-            for setting, klass in settings.iteritems() if klass.cli
+            for setting, klass in settings.items() if klass.cli
         )
 
         return options
@@ -503,11 +487,77 @@ def unicode_(data):
     return data
 
 
+class RotateELBs(Command):
+    """
+    Rotates existing certificates to a new one on an ELB
+    """
+    option_list = (
+        Option('-e', '--elb-list', dest='elb_list', required=True),
+        Option('-p', '--chain-path', dest='chain_path'),
+        Option('-c', '--cert-name', dest='cert_name'),
+        Option('-a', '--cert-prefix', dest='cert_prefix'),
+        Option('-d', '--description', dest='description')
+    )
+
+    def run(self, elb_list, chain_path, cert_name, cert_prefix, description):
+
+        for e in open(elb_list, 'r').readlines():
+            elb_name, account_id, region, from_port, to_port, protocol = e.strip().split(',')
+
+            if cert_name:
+                arn = "arn:aws:iam::{0}:server-certificate/{1}".format(account_id, cert_name)
+
+            else:
+                # if no cert name is provided we need to discover it
+                listeners = elb.get_listeners(account_id, region, elb_name)
+
+                # get the listener we care about
+                for listener in listeners:
+                    if listener[0] == int(from_port) and listener[1] == int(to_port):
+                        arn = listener[4]
+                        name = get_name_from_arn(arn)
+                        certificate = cert_service.get_by_name(name)
+                        break
+                else:
+                    sys.stdout.write("[-] Could not find ELB {0}".format(elb_name))
+                    continue
+
+                if not certificate:
+                    sys.stdout.write("[-] Could not find certificate {0} in Lemur".format(name))
+                    continue
+
+                dests = []
+                for d in certificate.destinations:
+                    dests.append({'id': d.id})
+
+                nots = []
+                for n in certificate.notifications:
+                    nots.append({'id': n.id})
+
+                new_certificate = database.clone(certificate)
+
+                if cert_prefix:
+                    new_certificate.name = "{0}-{1}".format(cert_prefix, new_certificate.name)
+
+                new_certificate.chain = open(chain_path, 'r').read()
+                new_certificate.description = "{0} - {1}".format(new_certificate.description, description)
+
+                new_certificate = database.create(new_certificate)
+                database.update_list(new_certificate, 'destinations', Destination, dests)
+                database.update_list(new_certificate, 'notifications', Notification, nots)
+                database.update(new_certificate)
+
+                arn = new_certificate.get_arn(account_id)
+
+            elb.update_listeners(account_id, region, elb_name, [(from_port, to_port, protocol, arn)], [from_port])
+
+            sys.stdout.write("[+] Updated {0} to use {1}\n".format(elb_name, new_certificate.name))
+
+
 class ProvisionELB(Command):
     """
     Creates and provisions a certificate on an ELB based on command line arguments
     """
-
     option_list = (
         Option('-d', '--dns', dest='dns', action='append', required=True, type=unicode_),
         Option('-e', '--elb', dest='elb_name', required=True, type=unicode_),
@@ -719,33 +769,179 @@ def publish_verisign_units():
 
 
 @manager.command
-def backfill_signing_algo():
+def publish_unapproved_verisign_certificates():
     """
-    Will attempt to backfill the signing_algorithm column
-
+    Query the Verisign for any certificates that need to be approved.
     :return:
     """
-    from cryptography import x509
-    from cryptography.hazmat.backends import default_backend
-    from lemur.certificates.models import get_signing_algorithm
-    for c in cert_service.get_all_certs():
-        cert = x509.load_pem_x509_certificate(str(c.body), default_backend())
-        c.signing_algorithm = get_signing_algorithm(cert)
-        c.signing_algorithm
-        database.update(c)
-        print(c.signing_algorithm)
+    from lemur.plugins import plugins
+    from lemur.extensions import metrics
+    v = plugins.get('verisign-issuer')
+    certs = v.get_pending_certificates()
+    metrics.send('pending_certificates', 'gauge', certs)
+
+
+class Report(Command):
+    """
+    Defines a set of reports to be run periodically against Lemur.
+    """
+    option_list = (
+        Option('-n', '--name', dest='name', default=None, help='Name of the report to run.'),
+        Option('-d', '--duration', dest='duration', default=356, help='Number of days to run the report'),
+    )
+
+    def run(self, name, duration):
+
+        end = datetime.utcnow()
+        start = end - timedelta(days=duration)
+        self.certificates_issued(name, start, end)
+
+    @staticmethod
+    def certificates_issued(name=None, start=None, end=None):
+        """
+        Generates simple report of number of certificates issued by the authority, if no authority
+        is specified report on total number of certificates.
+
+        :param name:
+        :param start:
+        :param end:
+        :return:
+        """
+
+        def _calculate_row(authority):
+            day_cnt = Counter()
+            month_cnt = Counter()
+            year_cnt = Counter()
+
+            for cert in authority.certificates:
+                date = cert.date_created.date()
+                day_cnt[date.day] += 1
+                month_cnt[date.month] += 1
+                year_cnt[date.year] += 1
+
+            try:
+                day_avg = int(sum(day_cnt.values()) / len(day_cnt.keys()))
+            except ZeroDivisionError:
+                day_avg = 0
+
+            try:
+                month_avg = int(sum(month_cnt.values()) / len(month_cnt.keys()))
+            except ZeroDivisionError:
+                month_avg = 0
+
+            try:
+                year_avg = int(sum(year_cnt.values()) / len(year_cnt.keys()))
+            except ZeroDivisionError:
+                year_avg = 0
+
+            return [authority.name, authority.description, day_avg, month_avg, year_avg]
+
+        rows = []
+        if not name:
+            for authority in authority_service.get_all():
+                rows.append(_calculate_row(authority))
+
+        else:
+            authority = authority_service.get_by_name(name)
+
+            if not authority:
+                sys.stderr.write('[!] Authority {0} was not found.'.format(name))
+                sys.exit(1)
+
+            rows.append(_calculate_row(authority))
+
+        sys.stdout.write(tabulate(rows, headers=["Authority Name", "Description", "Daily Average", "Monthy Average", "Yearly Average"]) + "\n")
+
+
+class Sources(Command):
+    """
+    Defines a set of actions to take against Lemur's sources.
+    """
+    option_list = (
+        Option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.'),
+        Option('-a', '--action', choices=['sync', 'clean'], dest='action', help='Action to take on source.')
+    )
+
+    def run(self, source_strings, action):
+        sources = []
+        if not source_strings:
+            table = []
+            for source in source_service.get_all():
+                table.append([source.label, source.active, source.description])
+
+            sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description']))
+            sys.exit(1)
+
+        elif 'all' in source_strings:
+            sources = source_service.get_all()
+
+        else:
+            for source_str in source_strings:
+                source = source_service.get_by_label(source_str)
+
+                if not source:
+                    sys.stderr.write("Unable to find specified source with label: {0}".format(source_str))
+
+                sources.append(source)
+
+        for source in sources:
+            if action == 'sync':
+                self.sync(source)
+
+            if action == 'clean':
+                self.clean(source)
+
+    @staticmethod
+    def sync(source):
+        start_time = time.time()
+        sys.stdout.write("[+] Staring to sync source: {label}!\n".format(label=source.label))
+
+        user = user_service.get_by_username('lemur')
+
+        try:
+            source_service.sync(source, user)
+            sys.stdout.write(
+                "[+] Finished syncing source: {label}. Run Time: {time}\n".format(
+                    label=source.label,
+                    time=(time.time() - start_time)
+                )
+            )
+        except Exception as e:
+            current_app.logger.exception(e)
+
+            sys.stdout.write(
+                "[X] Failed syncing source {label}!\n".format(label=source.label)
+            )
+
+            metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label})
+
+    @staticmethod
+    def clean(source):
+        start_time = time.time()
+        sys.stdout.write("[+] Staring to clean source: {label}!\n".format(label=source.label))
+        source_service.clean(source)
+        sys.stdout.write(
+            "[+] Finished cleaning source: {label}. Run Time: {time}\n".format(
+                label=source.label,
+                time=(time.time() - start_time)
+            )
+        )
 
 
 def main():
     manager.add_command("start", LemurServer())
-    manager.add_command("runserver", Server(host='127.0.0.1'))
+    manager.add_command("runserver", Server(host='127.0.0.1', threaded=True))
     manager.add_command("clean", Clean())
     manager.add_command("show_urls", ShowUrls())
     manager.add_command("db", MigrateCommand)
     manager.add_command("init", InitializeApp())
     manager.add_command("create_user", CreateUser())
+    manager.add_command("reset_password", ResetPassword())
     manager.add_command("create_role", CreateRole())
     manager.add_command("provision_elb", ProvisionELB())
+    manager.add_command("rotate_elbs", RotateELBs())
+    manager.add_command("sources", Sources())
+    manager.add_command("report", Report())
     manager.run()
 
 if __name__ == "__main__":

lemur/metrics.py

@@ -0,0 +1,32 @@
+"""
+.. module: lemur.metrics
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+"""
+from flask import current_app
+from lemur.plugins.base import plugins
+
+
+class Metrics(object):
+    """
+    :param app: The Flask application object. Defaults to None.
+    """
+    _providers = []
+
+    def __init__(self, app=None):
+        if app is not None:
+            self.init_app(app)
+
+    def init_app(self, app):
+        """Initializes the application with the extension.
+
+        :param app: The Flask application object.
+        """
+        self._providers = app.config.get('METRIC_PROVIDERS', [])
+
+    def send(self, metric_name, metric_type, metric_value, *args, **kwargs):
+        for provider in self._providers:
+            current_app.logger.debug(
+                "Sending metric '{metric}' to the {provider} provider.".format(metric=metric_name, provider=provider))
+            p = plugins.get(provider)
+            p.submit(metric_name, metric_type, metric_value, *args, **kwargs)

lemur/migrations/versions/1ff763f5b80b_.py

@@ -1,42 +0,0 @@
-"""Adding in models for certificate sources
-
-Revision ID: 1ff763f5b80b
-Revises: 4dc5ddd111b8
-Create Date: 2015-08-01 15:24:20.412725
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '1ff763f5b80b'
-down_revision = '4dc5ddd111b8'
-
-from alembic import op
-import sqlalchemy as sa
-
-import sqlalchemy_utils
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('sources',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('label', sa.String(length=32), nullable=True),
-    sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True),
-    sa.Column('description', sa.Text(), nullable=True),
-    sa.Column('plugin_name', sa.String(length=32), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('certificate_source_associations',
-    sa.Column('source_id', sa.Integer(), nullable=True),
-    sa.Column('certificate_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['source_id'], ['destinations.id'], ondelete='cascade')
-    )
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('certificate_source_associations')
-    op.drop_table('sources')
-    ### end Alembic commands ###

lemur/migrations/versions/29d8c8455c86_.py

@@ -0,0 +1,62 @@
+"""Adding endpoint tables
+
+Revision ID: 29d8c8455c86
+Revises: 3307381f3b88
+Create Date: 2016-06-28 16:05:25.720213
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '29d8c8455c86'
+down_revision = '3307381f3b88'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('ciphers',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=128), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('policy',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=128), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('policies_ciphers',
+    sa.Column('cipher_id', sa.Integer(), nullable=True),
+    sa.Column('policy_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['cipher_id'], ['ciphers.id'], ),
+    sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], )
+    )
+    op.create_index('policies_ciphers_ix', 'policies_ciphers', ['cipher_id', 'policy_id'], unique=False)
+    op.create_table('endpoints',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('owner', sa.String(length=128), nullable=True),
+    sa.Column('name', sa.String(length=128), nullable=True),
+    sa.Column('dnsname', sa.String(length=256), nullable=True),
+    sa.Column('type', sa.String(length=128), nullable=True),
+    sa.Column('active', sa.Boolean(), nullable=True),
+    sa.Column('port', sa.Integer(), nullable=True),
+    sa.Column('date_created', sa.DateTime(), server_default=sa.text(u'now()'), nullable=False),
+    sa.Column('policy_id', sa.Integer(), nullable=True),
+    sa.Column('certificate_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ),
+    sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('endpoints')
+    op.drop_index('policies_ciphers_ix', table_name='policies_ciphers')
+    op.drop_table('policies_ciphers')
+    op.drop_table('policy')
+    op.drop_table('ciphers')
+    ### end Alembic commands ###

lemur/migrations/versions/3307381f3b88_.py

@@ -0,0 +1,131 @@
+"""
+Refactor authority columns and associates an authorities root certificate with a certificate stored in the
+certificate tables.
+
+Migrates existing authority owners to associated roles.
+Migrates existing certificate owners to associated role.
+
+Revision ID: 3307381f3b88
+Revises: 412b22cb656a
+Create Date: 2016-05-20 17:33:04.360687
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3307381f3b88'
+down_revision = '412b22cb656a'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import text
+from sqlalchemy.dialects import postgresql
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('authorities', 'owner',
+               existing_type=sa.VARCHAR(length=128),
+               nullable=True)
+    op.drop_column('authorities', 'not_after')
+    op.drop_column('authorities', 'bits')
+    op.drop_column('authorities', 'cn')
+    op.drop_column('authorities', 'not_before')
+    op.add_column('certificates', sa.Column('root_authority_id', sa.Integer(), nullable=True))
+    op.alter_column('certificates', 'body',
+               existing_type=sa.TEXT(),
+               nullable=False)
+    op.alter_column('certificates', 'owner',
+               existing_type=sa.VARCHAR(length=128),
+               nullable=True)
+    op.drop_constraint(u'certificates_authority_id_fkey', 'certificates', type_='foreignkey')
+    op.create_foreign_key(None, 'certificates', 'authorities', ['authority_id'], ['id'], ondelete='CASCADE')
+    op.create_foreign_key(None, 'certificates', 'authorities', ['root_authority_id'], ['id'], ondelete='CASCADE')
+    ### end Alembic commands ###
+
+    # link existing certificate to their authority certificates
+    conn = op.get_bind()
+    for id, body, owner in conn.execute(text('select id, body, owner from authorities')):
+        if not owner:
+            owner = "lemur@nobody"
+
+        # look up certificate by body, if duplications are found, pick one
+        stmt = text('select id from certificates where body=:body')
+        stmt = stmt.bindparams(body=body)
+        root_certificate = conn.execute(stmt).fetchone()
+        if root_certificate:
+            stmt = text('update certificates set root_authority_id=:root_authority_id where id=:id')
+            stmt = stmt.bindparams(root_authority_id=id, id=root_certificate[0])
+            op.execute(stmt)
+
+        # link owner roles to their authorities
+        stmt = text('select id from roles where name=:name')
+        stmt = stmt.bindparams(name=owner)
+        owner_role = conn.execute(stmt).fetchone()
+
+        if not owner_role:
+            stmt = text('insert into roles (name, description) values (:name, :description)')
+            stmt = stmt.bindparams(name=owner, description='Lemur generated role or existing owner.')
+            op.execute(stmt)
+
+        stmt = text('select id from roles where name=:name')
+        stmt = stmt.bindparams(name=owner)
+        owner_role = conn.execute(stmt).fetchone()
+
+        stmt = text('select * from roles_authorities where role_id=:role_id and authority_id=:authority_id')
+        stmt = stmt.bindparams(role_id=owner_role[0], authority_id=id)
+        exists = conn.execute(stmt).fetchone()
+
+        if not exists:
+            stmt = text('insert into roles_authorities (role_id, authority_id) values (:role_id, :authority_id)')
+            stmt = stmt.bindparams(role_id=owner_role[0], authority_id=id)
+            op.execute(stmt)
+
+    # link owner roles to their certificates
+    for id, owner in conn.execute(text('select id, owner from certificates')):
+        if not owner:
+            owner = "lemur@nobody"
+
+        stmt = text('select id from roles where name=:name')
+        stmt = stmt.bindparams(name=owner)
+        owner_role = conn.execute(stmt).fetchone()
+
+        if not owner_role:
+            stmt = text('insert into roles (name, description) values (:name, :description)')
+            stmt = stmt.bindparams(name=owner, description='Lemur generated role or existing owner.')
+            op.execute(stmt)
+
+        # link owner roles to their authorities
+        stmt = text('select id from roles where name=:name')
+        stmt = stmt.bindparams(name=owner)
+        owner_role = conn.execute(stmt).fetchone()
+
+        stmt = text('select * from roles_certificates where role_id=:role_id and certificate_id=:certificate_id')
+        stmt = stmt.bindparams(role_id=owner_role[0], certificate_id=id)
+        exists = conn.execute(stmt).fetchone()
+
+        if not exists:
+            stmt = text('insert into roles_certificates (role_id, certificate_id) values (:role_id, :certificate_id)')
+            stmt = stmt.bindparams(role_id=owner_role[0], certificate_id=id)
+            op.execute(stmt)
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'certificates', type_='foreignkey')
+    op.drop_constraint(None, 'certificates', type_='foreignkey')
+    op.create_foreign_key(u'certificates_authority_id_fkey', 'certificates', 'authorities', ['authority_id'], ['id'])
+    op.alter_column('certificates', 'owner',
+               existing_type=sa.VARCHAR(length=128),
+               nullable=True)
+    op.alter_column('certificates', 'body',
+               existing_type=sa.TEXT(),
+               nullable=True)
+    op.drop_column('certificates', 'root_authority_id')
+    op.add_column('authorities', sa.Column('not_before', postgresql.TIMESTAMP(), autoincrement=False, nullable=True))
+    op.add_column('authorities', sa.Column('cn', sa.VARCHAR(length=128), autoincrement=False, nullable=True))
+    op.add_column('authorities', sa.Column('bits', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.add_column('authorities', sa.Column('not_after', postgresql.TIMESTAMP(), autoincrement=False, nullable=True))
+    op.alter_column('authorities', 'owner',
+               existing_type=sa.VARCHAR(length=128),
+               nullable=True)
+    ### end Alembic commands ###

lemur/migrations/versions/33de094da890_.py

@@ -0,0 +1,31 @@
+"""Adding the ability to specify certificate replacements
+
+Revision ID: 33de094da890
+Revises: ed422fc58ba
+Create Date: 2015-11-30 15:40:19.827272
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '33de094da890'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('certificate_replacement_associations',
+    sa.Column('replaced_certificate_id', sa.Integer(), nullable=True),
+    sa.Column('certificate_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
+    sa.ForeignKeyConstraint(['replaced_certificate_id'], ['certificates.id'], ondelete='cascade')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('certificate_replacement_associations')
+    ### end Alembic commands ###

lemur/migrations/versions/3b718f59b8ce_.py

@@ -1,49 +0,0 @@
-"""Refactors Accounts to Destinations
-
-Revision ID: 3b718f59b8ce
-Revises: None
-Create Date: 2015-07-09 17:44:55.626221
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '3b718f59b8ce'
-down_revision = None
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('certificate_account_associations')
-    op.drop_table('accounts')
-    op.add_column('destinations', sa.Column('plugin_name', sa.String(length=32), nullable=True))
-    op.drop_index('ix_elbs_account_id', table_name='elbs')
-    op.drop_constraint(u'elbs_account_id_fkey', 'elbs', type_='foreignkey')
-    op.drop_column('elbs', 'account_id')
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('elbs', sa.Column('account_id', sa.BIGINT(), autoincrement=False, nullable=True))
-    op.create_foreign_key(u'elbs_account_id_fkey', 'elbs', 'accounts', ['account_id'], ['id'])
-    op.create_index('ix_elbs_account_id', 'elbs', ['account_id'], unique=False)
-    op.drop_column('destinations', 'plugin_name')
-    op.create_table('accounts',
-    sa.Column('id', sa.INTEGER(), server_default=sa.text(u"nextval('accounts_id_seq'::regclass)"), nullable=False),
-    sa.Column('account_number', sa.VARCHAR(length=32), autoincrement=False, nullable=True),
-    sa.Column('label', sa.VARCHAR(length=32), autoincrement=False, nullable=True),
-    sa.Column('notes', sa.TEXT(), autoincrement=False, nullable=True),
-    sa.PrimaryKeyConstraint('id', name=u'accounts_pkey'),
-    sa.UniqueConstraint('account_number', name=u'accounts_account_number_key'),
-        postgresql_ignore_search_path=False
-    )
-    op.create_table('certificate_account_associations',
-    sa.Column('account_id', sa.INTEGER(), autoincrement=False, nullable=True),
-    sa.Column('certificate_id', sa.INTEGER(), autoincrement=False, nullable=True),
-    sa.ForeignKeyConstraint(['account_id'], [u'accounts.id'], name=u'certificate_account_associations_account_id_fkey', ondelete=u'CASCADE'),
-    sa.ForeignKeyConstraint(['certificate_id'], [u'certificates.id'], name=u'certificate_account_associations_certificate_id_fkey', ondelete=u'CASCADE')
-    )
-    ### end Alembic commands ###

lemur/migrations/versions/412b22cb656a_.py

@@ -0,0 +1,63 @@
+"""
+
+Revision ID: 412b22cb656a
+Revises: 4c50b903d1ae
+Create Date: 2016-05-17 17:37:41.210232
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '412b22cb656a'
+down_revision = '4c50b903d1ae'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import text
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('roles_authorities',
+    sa.Column('authority_id', sa.Integer(), nullable=True),
+    sa.Column('role_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['authority_id'], ['authorities.id'], ),
+    sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
+    )
+    op.create_index('roles_authorities_ix', 'roles_authorities', ['authority_id', 'role_id'], unique=True)
+    op.create_table('roles_certificates',
+    sa.Column('certificate_id', sa.Integer(), nullable=True),
+    sa.Column('role_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ),
+    sa.ForeignKeyConstraint(['role_id'], ['roles.id'], )
+    )
+    op.create_index('roles_certificates_ix', 'roles_certificates', ['certificate_id', 'role_id'], unique=True)
+    op.create_index('certificate_associations_ix', 'certificate_associations', ['domain_id', 'certificate_id'], unique=True)
+    op.create_index('certificate_destination_associations_ix', 'certificate_destination_associations', ['destination_id', 'certificate_id'], unique=True)
+    op.create_index('certificate_notification_associations_ix', 'certificate_notification_associations', ['notification_id', 'certificate_id'], unique=True)
+    op.create_index('certificate_replacement_associations_ix', 'certificate_replacement_associations', ['certificate_id', 'certificate_id'], unique=True)
+    op.create_index('certificate_source_associations_ix', 'certificate_source_associations', ['source_id', 'certificate_id'], unique=True)
+    op.create_index('roles_users_ix', 'roles_users', ['user_id', 'role_id'], unique=True)
+
+    ### end Alembic commands ###
+
+    # migrate existing authority_id relationship to many_to_many
+    conn = op.get_bind()
+    for id, authority_id in conn.execute(text('select id, authority_id from roles where authority_id is not null')):
+        stmt = text('insert into roles_authoritties (role_id, authority_id) values (:role_id, :authority_id)')
+        stmt = stmt.bindparams(role_id=id, authority_id=authority_id)
+        op.execute(stmt)
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('roles_users_ix', table_name='roles_users')
+    op.drop_index('certificate_source_associations_ix', table_name='certificate_source_associations')
+    op.drop_index('certificate_replacement_associations_ix', table_name='certificate_replacement_associations')
+    op.drop_index('certificate_notification_associations_ix', table_name='certificate_notification_associations')
+    op.drop_index('certificate_destination_associations_ix', table_name='certificate_destination_associations')
+    op.drop_index('certificate_associations_ix', table_name='certificate_associations')
+    op.drop_index('roles_certificates_ix', table_name='roles_certificates')
+    op.drop_table('roles_certificates')
+    op.drop_index('roles_authorities_ix', table_name='roles_authorities')
+    op.drop_table('roles_authorities')
+    ### end Alembic commands ###

lemur/migrations/versions/4bcfa2c36623_.py

@@ -1,26 +0,0 @@
-"""Adding certificate signing algorithm
-
-Revision ID: 4bcfa2c36623
-Revises: 1ff763f5b80b
-Create Date: 2015-10-06 10:03:47.993204
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4bcfa2c36623'
-down_revision = '1ff763f5b80b'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('certificates', sa.Column('signing_algorithm', sa.String(length=128), nullable=True))
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('certificates', 'signing_algorithm')
-    ### end Alembic commands ###

lemur/migrations/versions/4c50b903d1ae_.py

@@ -0,0 +1,26 @@
+"""Adding ability to mark domains as 'sensitive'
+
+Revision ID: 4c50b903d1ae
+Revises: 33de094da890
+Create Date: 2015-12-30 10:19:30.057791
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4c50b903d1ae'
+down_revision = '33de094da890'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('domains', sa.Column('sensitive', sa.Boolean(), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('domains', 'sensitive')
+    ### end Alembic commands ###

lemur/migrations/versions/4c8915e461b3_.py

@@ -1,41 +0,0 @@
-"""Adding notifications
-
-Revision ID: 4c8915e461b3
-Revises: 3b718f59b8ce
-Create Date: 2015-07-24 14:34:57.316273
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4c8915e461b3'
-down_revision = '3b718f59b8ce'
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-import sqlalchemy_utils
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('notifications',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('label', sa.String(length=128), nullable=True),
-    sa.Column('description', sa.Text(), nullable=True),
-    sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True),
-    sa.Column('active', sa.Boolean(), nullable=True),
-    sa.Column('plugin_name', sa.String(length=32), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.drop_column(u'certificates', 'challenge')
-    op.drop_column(u'certificates', 'csr_config')
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column(u'certificates', sa.Column('csr_config', sa.TEXT(), autoincrement=False, nullable=True))
-    op.add_column(u'certificates', sa.Column('challenge', postgresql.BYTEA(), autoincrement=False, nullable=True))
-    op.drop_table('notifications')
-    ### end Alembic commands ###

lemur/migrations/versions/4dc5ddd111b8_.py

@@ -1,31 +0,0 @@
-"""Creating a one-to-many relationship for notifications
-
-Revision ID: 4dc5ddd111b8
-Revises: 4c8915e461b3
-Create Date: 2015-07-24 15:02:04.398262
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4dc5ddd111b8'
-down_revision = '4c8915e461b3'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('certificate_notification_associations',
-    sa.Column('notification_id', sa.Integer(), nullable=True),
-    sa.Column('certificate_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['notification_id'], ['notifications.id'], ondelete='cascade')
-    )
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('certificate_notification_associations')
-    ### end Alembic commands ###

lemur/migrations/versions/7f71c0cea31a_.py

@@ -0,0 +1,35 @@
+"""Ensures that certificate name is unique.
+If duplicates are found, we follow the standard naming convention of appending '-X'
+with x being the number of duplicates starting at 1.
+
+Revision ID: 7f71c0cea31a
+Revises: 29d8c8455c86
+Create Date: 2016-07-28 09:39:12.736506
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '7f71c0cea31a'
+down_revision = '29d8c8455c86'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import text
+
+
+def upgrade():
+    conn = op.get_bind()
+    for name in conn.execute(text('select name from certificates group by name having count(*) > 1')):
+        for idx, id in enumerate(conn.execute(text("select id from certificates where certificates.name like :name order by id ASC").bindparams(name=name[0]))):
+            if not idx:
+                continue
+            new_name = name[0] + '-' + str(idx)
+            stmt = text('update certificates set name=:name where id=:id')
+            stmt = stmt.bindparams(name=new_name, id=id[0])
+            op.execute(stmt)
+
+    op.create_unique_constraint(None, 'certificates', ['name'])
+
+
+def downgrade():
+    op.drop_constraint(None, 'certificates', type_='unique')

lemur/migrations/versions/932525b82f1a_.py

@@ -0,0 +1,21 @@
+"""Changing the column name to the more accurately named 'notify'.
+
+Revision ID: 932525b82f1a
+Revises: 7f71c0cea31a
+Create Date: 2016-10-13 20:14:33.928029
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '932525b82f1a'
+down_revision = '7f71c0cea31a'
+
+from alembic import op
+
+
+def upgrade():
+    op.alter_column('certificates', 'active', new_column_name='notify')
+
+
+def downgrade():
+    op.alter_column('certificates', 'notify', new_column_name='active')

lemur/migrations/versions/ed422fc58ba_.py

@@ -1,255 +0,0 @@
-"""Migrates the private key encrypted column from AES to fernet encryption scheme.
-
-Revision ID: ed422fc58ba
-Revises: 4bcfa2c36623
-Create Date: 2015-10-23 09:19:28.654126
-
-"""
-import base64
-
-# revision identifiers, used by Alembic.
-revision = 'ed422fc58ba'
-down_revision = '4bcfa2c36623'
-import six
-
-from StringIO import StringIO
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.sql import text
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
-from cryptography.fernet import Fernet, MultiFernet
-
-from flask import current_app
-from lemur.common.utils import get_psuedo_random_string
-
-conn = op.get_bind()
-
-#op.drop_table('encrypted_keys')
-#op.drop_table('encrypted_passwords')
-
-# helper tables to migrate data
-temp_key_table = op.create_table('encrypted_keys',
-                                 sa.Column('id', sa.Integer(), nullable=False),
-                                 sa.Column('aes', sa.Binary()),
-                                 sa.Column('fernet', sa.Binary()),
-                                 sa.PrimaryKeyConstraint('id')
-                                 )
-
-# helper table to migrate data
-temp_password_table = op.create_table('encrypted_passwords',
-                                      sa.Column('id', sa.Integer(), nullable=False),
-                                      sa.Column('aes', sa.Binary()),
-                                      sa.Column('fernet', sa.Binary()),
-                                      sa.PrimaryKeyConstraint('id')
-                                      )
-
-
-# From http://sqlalchemy-utils.readthedocs.org/en/latest/_modules/sqlalchemy_utils/types/encrypted.html#EncryptedType
-# for migration purposes only
-class EncryptionDecryptionBaseEngine(object):
-    """A base encryption and decryption engine.
-
-    This class must be sub-classed in order to create
-    new engines.
-    """
-
-    def _update_key(self, key):
-        if isinstance(key, six.string_types):
-            key = key.encode()
-        digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
-        digest.update(key)
-        engine_key = digest.finalize()
-
-        self._initialize_engine(engine_key)
-
-    def encrypt(self, value):
-        raise NotImplementedError('Subclasses must implement this!')
-
-    def decrypt(self, value):
-        raise NotImplementedError('Subclasses must implement this!')
-
-
-class AesEngine(EncryptionDecryptionBaseEngine):
-    """Provide AES encryption and decryption methods."""
-
-    BLOCK_SIZE = 16
-    PADDING = six.b('*')
-
-    def _initialize_engine(self, parent_class_key):
-        self.secret_key = parent_class_key
-        self.iv = self.secret_key[:16]
-        self.cipher = Cipher(
-            algorithms.AES(self.secret_key),
-            modes.CBC(self.iv),
-            backend=default_backend()
-        )
-
-    def _pad(self, value):
-        """Pad the message to be encrypted, if needed."""
-        BS = self.BLOCK_SIZE
-        P = self.PADDING
-        padded = (value + (BS - len(value) % BS) * P)
-        return padded
-
-    def encrypt(self, value):
-        if not isinstance(value, six.string_types):
-            value = repr(value)
-        if isinstance(value, six.text_type):
-            value = str(value)
-        value = value.encode()
-        value = self._pad(value)
-        encryptor = self.cipher.encryptor()
-        encrypted = encryptor.update(value) + encryptor.finalize()
-        encrypted = base64.b64encode(encrypted)
-        return encrypted
-
-    def decrypt(self, value):
-        if isinstance(value, six.text_type):
-            value = str(value)
-        decryptor = self.cipher.decryptor()
-        decrypted = base64.b64decode(value)
-        decrypted = decryptor.update(decrypted) + decryptor.finalize()
-        decrypted = decrypted.rstrip(self.PADDING)
-        if not isinstance(decrypted, six.string_types):
-            decrypted = decrypted.decode('utf-8')
-        return decrypted
-
-
-def migrate_to_fernet(aes_encrypted, old_key, new_key):
-    """
-    Will attempt to migrate an aes encrypted to fernet encryption
-    :param aes_encrypted:
-    :return: fernet encrypted value
-    """
-    engine = AesEngine()
-    engine._update_key(old_key)
-
-    if not isinstance(aes_encrypted, six.string_types):
-        return
-
-    aes_decrypted = engine.decrypt(aes_encrypted)
-    fernet_encrypted = MultiFernet([Fernet(k) for k in new_key]).encrypt(bytes(aes_decrypted))
-
-    # sanity check
-    fernet_decrypted = MultiFernet([Fernet(k) for k in new_key]).decrypt(fernet_encrypted)
-    if fernet_decrypted != aes_decrypted:
-        raise Exception("WARNING: Decrypted values do not match!")
-
-    return fernet_encrypted
-
-
-def migrate_from_fernet(fernet_encrypted, old_key, new_key):
-    """
-    Will attempt to migrate from a fernet encryption to aes
-    :param fernet_encrypted:
-    :return:
-    """
-    engine = AesEngine()
-    engine._update_key(new_key)
-
-    fernet_decrypted = MultiFernet([Fernet(k) for k in old_key]).decrypt(fernet_encrypted)
-    aes_encrypted = engine.encrypt(fernet_decrypted)
-
-    # sanity check
-    aes_decrypted = engine.decrypt(aes_encrypted)
-    if fernet_decrypted != aes_decrypted:
-        raise Exception("WARNING: Decrypted values do not match!")
-
-    return aes_encrypted
-
-
-def upgrade():
-    old_key = current_app.config.get('LEMUR_ENCRYPTION_KEY')
-    print "Using: {0} as decryption key".format(old_key)
-    # generate a new fernet token
-
-    if current_app.config.get('LEMUR_ENCRYPTION_KEYS'):
-        new_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS')
-    else:
-        new_key = [Fernet.generate_key()]
-
-    print "Using: {0} as new encryption key, save this and place it in your configuration!".format(new_key)
-
-    # migrate private_keys
-    temp_keys = []
-    for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')):
-        aes_encrypted = StringIO(private_key).read()
-        fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key)
-        temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_key_table, temp_keys)
-
-    for id, fernet in conn.execute(text('select id, fernet from encrypted_keys')):
-        stmt = text("update certificates set private_key=:key where id=:id")
-        stmt = stmt.bindparams(key=fernet, id=id)
-        op.execute(stmt)
-        print "Certificate {0} has been migrated".format(id)
-
-    # migrate role_passwords
-    temp_passwords = []
-    for id, password in conn.execute(text('select id, password from roles where password is not null')):
-        aes_encrypted = StringIO(password).read()
-        fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key)
-        temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_password_table, temp_passwords)
-
-    for id, fernet in conn.execute(text('select id, fernet from encrypted_passwords')):
-        stmt = text("update roles set password=:password where id=:id")
-        stmt = stmt.bindparams(password=fernet, id=id)
-        print stmt
-        op.execute(stmt)
-        print "Password {0} has been migrated".format(id)
-
-    op.drop_table('encrypted_keys')
-    op.drop_table('encrypted_passwords')
-
-
-def downgrade():
-    old_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS')
-    print "Using: {0} as decryption key(s)".format(old_key)
-
-    # generate aes valid key
-    if current_app.config.get('LEMUR_ENCRYPTION_KEY'):
-        new_key = current_app.config.get('LEMUR_ENCRYPTION_KEY')
-    else:
-        new_key = get_psuedo_random_string()
-    print "Using: {0} as the encryption key, save this and place it in your configuration!".format(new_key)
-
-    # migrate keys
-    temp_keys = []
-    for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')):
-        fernet_encrypted = StringIO(private_key).read()
-        aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key)
-        temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_key_table, temp_keys)
-
-    for id, aes in conn.execute(text('select id, aes from encrypted_keys')):
-        stmt = text("update certificates set private_key=:key where id=:id")
-        stmt = stmt.bindparams(key=aes, id=id)
-        print stmt
-        op.execute(stmt)
-        print "Certificate {0} has been migrated".format(id)
-
-    # migrate role_passwords
-    temp_passwords = []
-    for id, password in conn.execute(text('select id, password from roles where password is not null')):
-        fernet_encrypted = StringIO(password).read()
-        aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key)
-        temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_password_table, temp_passwords)
-
-    for id, aes in conn.execute(text('select id, aes from encrypted_passwords')):
-        stmt = text("update roles set password=:password where id=:id")
-        stmt = stmt.bindparams(password=aes, id=id)
-        op.execute(stmt)
-        print "Password {0} has been migrated".format(id)
-
-    op.drop_table('encrypted_keys')
-    op.drop_table('encrypted_passwords')

lemur/models.py

@@ -8,7 +8,8 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from sqlalchemy import Column, Integer, ForeignKey
+from sqlalchemy import Column, Integer, ForeignKey, Index
+
 from lemur.database import db
 
 certificate_associations = db.Table('certificate_associations',
@@ -16,6 +17,8 @@ certificate_associations = db.Table('certificate_associations',
                                     Column('certificate_id', Integer, ForeignKey('certificates.id'))
                                     )
 
+Index('certificate_associations_ix', certificate_associations.c.domain_id, certificate_associations.c.certificate_id)
+
 certificate_destination_associations = db.Table('certificate_destination_associations',
                                                 Column('destination_id', Integer,
                                                        ForeignKey('destinations.id', ondelete='cascade')),
@@ -23,6 +26,8 @@ certificate_destination_associations = db.Table('certificate_destination_associa
                                                        ForeignKey('certificates.id', ondelete='cascade'))
                                                 )
 
+Index('certificate_destination_associations_ix', certificate_destination_associations.c.destination_id, certificate_destination_associations.c.certificate_id)
+
 certificate_source_associations = db.Table('certificate_source_associations',
                                            Column('source_id', Integer,
                                                   ForeignKey('sources.id', ondelete='cascade')),
@@ -30,13 +35,51 @@ certificate_source_associations = db.Table('certificate_source_associations',
                                                   ForeignKey('certificates.id', ondelete='cascade'))
                                            )
 
+Index('certificate_source_associations_ix', certificate_source_associations.c.source_id, certificate_source_associations.c.certificate_id)
+
 certificate_notification_associations = db.Table('certificate_notification_associations',
                                                  Column('notification_id', Integer,
                                                         ForeignKey('notifications.id', ondelete='cascade')),
                                                  Column('certificate_id', Integer,
                                                         ForeignKey('certificates.id', ondelete='cascade'))
                                                  )
+
+Index('certificate_notification_associations_ix', certificate_notification_associations.c.notification_id, certificate_notification_associations.c.certificate_id)
+
+certificate_replacement_associations = db.Table('certificate_replacement_associations',
+                                                Column('replaced_certificate_id', Integer,
+                                                       ForeignKey('certificates.id', ondelete='cascade')),
+                                                Column('certificate_id', Integer,
+                                                       ForeignKey('certificates.id', ondelete='cascade'))
+                                                )
+
+Index('certificate_replacement_associations_ix', certificate_replacement_associations.c.certificate_id, certificate_replacement_associations.c.certificate_id)
+
+roles_authorities = db.Table('roles_authorities',
+                             Column('authority_id', Integer, ForeignKey('authorities.id')),
+                             Column('role_id', Integer, ForeignKey('roles.id'))
+                             )
+
+Index('roles_authorities_ix', roles_authorities.c.authority_id, roles_authorities.c.role_id)
+
+roles_certificates = db.Table('roles_certificates',
+                              Column('certificate_id', Integer, ForeignKey('certificates.id')),
+                              Column('role_id', Integer, ForeignKey('roles.id'))
+                              )
+
+Index('roles_certificates_ix', roles_certificates.c.certificate_id, roles_certificates.c.role_id)
+
+
 roles_users = db.Table('roles_users',
                        Column('user_id', Integer, ForeignKey('users.id')),
                        Column('role_id', Integer, ForeignKey('roles.id'))
                        )
+
+Index('roles_users_ix', roles_users.c.user_id, roles_users.c.role_id)
+
+
+policies_ciphers = db.Table('policies_ciphers',
+                            Column('cipher_id', Integer, ForeignKey('ciphers.id')),
+                            Column('policy_id', Integer, ForeignKey('policy.id')))
+
+Index('policies_ciphers_ix', policies_ciphers.c.cipher_id, policies_ciphers.c.policy_id)

lemur/notifications/models.py

@@ -33,3 +33,6 @@ class Notification(db.Model):
     @property
     def plugin(self):
         return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Notification(label={label})".format(label=self.label)

lemur/notifications/schemas.py

@@ -0,0 +1,49 @@
+"""
+.. module: lemur.notifications.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields, post_dump
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.schemas import PluginInputSchema, PluginOutputSchema, AssociatedCertificateSchema
+
+
+class NotificationInputSchema(LemurInputSchema):
+    id = fields.Integer()
+    label = fields.String(required=True)
+    description = fields.String()
+    active = fields.Boolean()
+    plugin = fields.Nested(PluginInputSchema, required=True)
+    certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[])
+
+
+class NotificationOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    label = fields.String()
+    description = fields.String()
+    active = fields.Boolean()
+    options = fields.List(fields.Dict())
+    plugin = fields.Nested(PluginOutputSchema)
+    certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[])
+
+    @post_dump
+    def fill_object(self, data):
+        data['plugin']['pluginOptions'] = data['options']
+        return data
+
+
+class NotificationNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    label = fields.String()
+    description = fields.String()
+    active = fields.Boolean()
+    options = fields.List(fields.Dict())
+    plugin = fields.Nested(PluginOutputSchema)
+
+
+notification_input_schema = NotificationInputSchema()
+notification_output_schema = NotificationOutputSchema()
+notifications_output_schema = NotificationOutputSchema(many=True)

lemur/notifications/service.py

@@ -36,13 +36,17 @@ def _get_message_data(cert):
     :param cert:
     :return:
     """
-    cert_dict = cert.as_dict()
+    cert_dict = {}
 
     if cert.user:
         cert_dict['creator'] = cert.user.email
 
-    cert_dict['domains'] = [x .name for x in cert.domains]
-    cert_dict['superseded'] = list(set([x.name for x in _find_superseded(cert) if cert.name != x]))
+    cert_dict['not_after'] = cert.not_after
+    cert_dict['owner'] = cert.owner
+    cert_dict['name'] = cert.name
+    cert_dict['body'] = cert.body
+    cert_dict['endpoints'] = [{'name': x.name, 'dnsname': x.dnsname} for x in cert.endpoints]
+
     return cert_dict
 
 
@@ -156,6 +160,9 @@ def _is_eligible_for_notifications(cert):
     :param cert:
     :return:
     """
+    if not cert.notify:
+        return
+
     now = arrow.utcnow()
     days = (cert.not_after - now.naive).days
 
@@ -228,7 +235,7 @@ def create_default_expiration_notifications(name, recipients):
             inter.extend(options)
             n = create(
                 label="{name}_{interval}_DAY".format(name=name, interval=i),
-                plugin_name="email-notification",
+                plugin_name=current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification"),
                 options=list(inter),
                 description="Default {interval} day expiration notification".format(interval=i),
                 certificates=[]
@@ -240,27 +247,31 @@ def create_default_expiration_notifications(name, recipients):
 
 def create(label, plugin_name, options, description, certificates):
     """
-    Creates a new destination, that can then be used as a destination for certificates.
+    Creates a new notification.
 
-    :param label: Notification common name
+    :param label: Notification label
     :param plugin_name:
     :param options:
     :param description:
+    :param certificates:
     :rtype : Notification
     :return:
     """
     notification = Notification(label=label, options=options, plugin_name=plugin_name, description=description)
-    notification = database.update_list(notification, 'certificates', Certificate, certificates)
+    notification.certificates = certificates
     return database.create(notification)
 
 
 def update(notification_id, label, options, description, active, certificates):
     """
-    Updates an existing destination.
+    Updates an existing notification.
 
-    :param label: Notification common name
+    :param notification_id:
+    :param label: Notification label
     :param options:
     :param description:
+    :param active:
+    :param certificates:
     :rtype : Notification
     :return:
     """
@@ -270,7 +281,7 @@ def update(notification_id, label, options, description, active, certificates):
     notification.options = options
     notification.description = description
     notification.active = active
-    notification = database.update_list(notification, 'certificates', Certificate, certificates)
+    notification.certificates = certificates
 
     return database.update(notification)
 
@@ -316,10 +327,6 @@ def get_all():
 
 
 def render(args):
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
     certificate_id = args.pop('certificate_id', None)
 
@@ -338,9 +345,4 @@ def render(args):
         else:
             query = database.filter(query, Notification, terms)
 
-    query = database.find_all(query, Notification, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Notification, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)
+    return database.sort_and_page(query, Notification, args)

lemur/notifications/views.py

@@ -7,64 +7,27 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import Blueprint
-from flask.ext.restful import Api, reqparse, fields
+from flask.ext.restful import Api, reqparse
 from lemur.notifications import service
+from lemur.notifications.schemas import notification_input_schema, notification_output_schema, notifications_output_schema
 
 from lemur.auth.service import AuthenticatedResource
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.common.utils import paginated_parser
+
+from lemur.common.schema import validate_schema
 
 
 mod = Blueprint('notifications', __name__)
 api = Api(mod)
 
 
-FIELDS = {
-    'description': fields.String,
-    'notificationOptions': fields.Raw(attribute='options'),
-    'pluginName': fields.String(attribute='plugin_name'),
-    'label': fields.String,
-    'active': fields.Boolean,
-    'id': fields.Integer,
-}
-
-
-def notification(value, name):
-    """
-    Validates a given notification exits
-    :param value:
-    :param name:
-    :return:
-    """
-    n = service.get(value)
-    if not n:
-        raise ValueError("Unable to find notification specified")
-    return n
-
-
-def notification_list(value, name):
-    """
-    Validates a given notification exists and returns a list
-    :param value:
-    :param name:
-    :return:
-    """
-    notifications = []
-    for v in value:
-        try:
-            notifications.append(notification(v['id'], 'id'))
-        except ValueError:
-            pass
-
-    return notifications
-
-
 class NotificationsList(AuthenticatedResource):
     """ Defines the 'notifications' endpoint """
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(NotificationsList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, notifications_output_schema)
     def get(self):
         """
         .. http:get:: /notifications
@@ -91,7 +54,7 @@ class NotificationsList(AuthenticatedResource):
                 "items": [
                     {
                         "description": "An example",
-                        "notificationOptions": [
+                        "options": [
                             {
                                 "name": "interval",
                                 "required": true,
@@ -133,9 +96,9 @@ class NotificationsList(AuthenticatedResource):
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
-           :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
@@ -144,8 +107,8 @@ class NotificationsList(AuthenticatedResource):
         args = parser.parse_args()
         return service.render(args)
 
-    @marshal_items(FIELDS)
-    def post(self):
+    @validate_schema(notification_input_schema, notification_output_schema)
+    def post(self, data=None):
         """
         .. http:post:: /notifications
 
@@ -161,7 +124,7 @@ class NotificationsList(AuthenticatedResource):
 
               {
                 "description": "a test",
-                "notificationOptions": [
+                "options": [
                     {
                         "name": "interval",
                         "required": true,
@@ -208,7 +171,7 @@ class NotificationsList(AuthenticatedResource):
 
               {
                 "description": "a test",
-                "notificationOptions": [
+                "options": [
                     {
                         "name": "interval",
                         "required": true,
@@ -251,18 +214,12 @@ class NotificationsList(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-        self.reqparse.add_argument('certificates', type=list, default=[], location='json')
-
-        args = self.reqparse.parse_args()
         return service.create(
-            args['label'],
-            args['plugin']['slug'],
-            args['plugin']['pluginOptions'],
-            args['description'],
-            args['certificates']
+            data['label'],
+            data['plugin']['slug'],
+            data['plugin']['plugin_options'],
+            data['description'],
+            data['certificates']
         )
 
 
@@ -271,7 +228,7 @@ class Notifications(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Notifications, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, notification_output_schema)
     def get(self, notification_id):
         """
         .. http:get:: /notifications/1
@@ -296,7 +253,7 @@ class Notifications(AuthenticatedResource):
 
               {
                 "description": "a test",
-                "notificationOptions": [
+                "options": [
                     {
                         "name": "interval",
                         "required": true,
@@ -338,8 +295,8 @@ class Notifications(AuthenticatedResource):
         """
         return service.get(notification_id)
 
-    @marshal_items(FIELDS)
-    def put(self, notification_id):
+    @validate_schema(notification_input_schema, notification_output_schema)
+    def put(self, notification_id, data=None):
         """
         .. http:put:: /notifications/1
 
@@ -375,20 +332,13 @@ class Notifications(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('active', type=bool, location='json')
-        self.reqparse.add_argument('certificates', type=list, default=[], location='json')
-        self.reqparse.add_argument('description', type=str, location='json')
-
-        args = self.reqparse.parse_args()
         return service.update(
             notification_id,
-            args['label'],
-            args['plugin']['pluginOptions'],
-            args['description'],
-            args['active'],
-            args['certificates']
+            data['label'],
+            data['plugin']['plugin_options'],
+            data['description'],
+            data['active'],
+            data['certificates']
         )
 
     def delete(self, notification_id):
@@ -401,7 +351,7 @@ class CertificateNotifications(AuthenticatedResource):
     def __init__(self):
         super(CertificateNotifications, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, notifications_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/notifications
@@ -428,7 +378,7 @@ class CertificateNotifications(AuthenticatedResource):
                 "items": [
                     {
                         "description": "An example",
-                        "notificationOptions": [
+                        "options": [
                             {
                                 "name": "interval",
                                 "required": true,
@@ -470,17 +420,13 @@ class CertificateNotifications(AuthenticatedResource):
 
            :query sortBy: field to sort on
            :query sortDir: acs or desc
-           :query page: int. default is 1
-           :query filter: key value pair. format is k=v;
-           :query limit: limit number. default is 10
+           :query page: int default is 1
+           :query filter: key value pair format is k;v
+           :query count: count number default is 10
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        parser = paginated_parser.copy()
-        parser.add_argument('active', type=bool, location='args')
-        args = parser.parse_args()
-        args['certificate_id'] = certificate_id
-        return service.render(args)
+        return service.render({'certificate_id': certificate_id})
 
 
 api.add_resource(NotificationsList, '/notifications', endpoint='notifications')

lemur/plugins/base/v1.py

@@ -108,10 +108,11 @@ class IPlugin(local):
         """
         return self.resource_links
 
-    def get_option(self, name, options):
+    @staticmethod
+    def get_option(name, options):
         for o in options:
             if o.get('name') == name:
-                return o['value']
+                return o.get('value', o.get('default'))
 
 
 class Plugin(IPlugin):

lemur/plugins/bases/__init__.py

@@ -2,3 +2,4 @@ from .destination import DestinationPlugin  # noqa
 from .issuer import IssuerPlugin  # noqa
 from .source import SourcePlugin  # noqa
 from .notification import NotificationPlugin, ExpirationNotificationPlugin  # noqa
+from .export import ExportPlugin  # noqa

lemur/plugins/bases/destination.py

@@ -11,6 +11,7 @@ from lemur.plugins.base import Plugin
 
 class DestinationPlugin(Plugin):
     type = 'destination'
+    requires_key = True
 
     def upload(self):
         raise NotImplemented

lemur/plugins/bases/export.py

@@ -0,0 +1,21 @@
+"""
+.. module: lemur.bases.export
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from lemur.plugins.base import Plugin
+
+
+class ExportPlugin(Plugin):
+    """
+    This is the base class from which all supported
+    exporters will inherit from.
+    """
+    type = 'export'
+    requires_key = True
+
+    def export(self):
+        raise NotImplemented

lemur/plugins/bases/metric.py

@@ -0,0 +1,16 @@
+"""
+.. module: lemur.bases.metric
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from lemur.plugins.base import Plugin
+
+
+class MetricPlugin(Plugin):
+    type = 'metric'
+
+    def submit(self, *args, **kwargs):
+        raise NotImplemented

lemur/plugins/bases/source.py

@@ -25,6 +25,12 @@ class SourcePlugin(Plugin):
     def get_certificates(self):
         raise NotImplemented
 
+    def get_endpoints(self):
+        raise NotImplemented
+
+    def clean(self):
+        raise NotImplemented
+
     @property
     def options(self):
         return list(self.default_options) + self.additional_options

lemur/plugins/lemur_acme/__init__.py

@@ -0,0 +1,5 @@
+try:
+    VERSION = __import__('pkg_resources') \
+        .get_distribution(__name__).version
+except Exception as e:
+    VERSION = 'unknown'

lemur/plugins/lemur_acme/plugin.py

@@ -0,0 +1,210 @@
+"""
+.. module: lemur.plugins.lemur_acme.acme
+    :platform: Unix
+    :synopsis: This module is responsible for communicating with a ACME CA.
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+    Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+"""
+from flask import current_app
+
+from acme.client import Client
+from acme import jose
+from acme import messages
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+import OpenSSL.crypto
+
+from lemur.plugins.bases import IssuerPlugin
+from lemur.plugins import lemur_acme as acme
+
+from .route53 import delete_txt_record, create_txt_record, wait_for_change
+
+
+def find_dns_challenge(authz):
+    for combo in authz.body.resolved_combinations:
+        if (
+            len(combo) == 1 and
+            isinstance(combo[0].chall, acme.challenges.DNS01)
+        ):
+            yield combo[0]
+
+
+class AuthorizationRecord(object):
+    def __init__(self, host, authz, dns_challenge, change_id):
+        self.host = host
+        self.authz = authz
+        self.dns_challenge = dns_challenge
+        self.change_id = change_id
+
+
+def start_dns_challenge(acme_client, host):
+    authz = acme_client.request_domain_challenges(
+        host, acme_client.directory.new_authz
+    )
+
+    [dns_challenge] = find_dns_challenge(authz)
+
+    change_id = create_txt_record(
+        dns_challenge.validation_domain_name(host),
+        dns_challenge.validation(acme_client.key),
+
+    )
+    return AuthorizationRecord(
+        host,
+        authz,
+        dns_challenge,
+        change_id,
+    )
+
+
+def complete_dns_challenge(acme_client, authz_record):
+    wait_for_change(authz_record.change_id)
+
+    response = authz_record.dns_challenge.response(acme_client.key)
+
+    verified = response.simple_verify(
+        authz_record.dns_challenge.chall,
+        authz_record.host,
+        acme_client.key.public_key()
+    )
+    if not verified:
+        raise ValueError("Failed verification")
+
+    acme_client.answer_challenge(authz_record.dns_challenge, response)
+
+
+def request_certificate(acme_client, authorizations, csr):
+    cert_response, _ = acme_client.poll_and_request_issuance(
+        jose.util.ComparableX509(
+            OpenSSL.crypto.load_certificate_request(
+                OpenSSL.crypto.FILETYPE_ASN1,
+                csr.public_bytes(serialization.Encoding.DER),
+            )
+        ),
+        authzrs=[authz_record.authz for authz_record in authorizations],
+    )
+    pem_certificate = OpenSSL.crypto.dump_certificate(
+        OpenSSL.crypto.FILETYPE_PEM, cert_response.body
+    )
+    pem_certificate_chain = "\n".join(
+        OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
+        for cert in acme_client.fetch_chain(cert_response)
+    )
+    return pem_certificate, pem_certificate_chain
+
+
+def generate_rsa_private_key():
+    return rsa.generate_private_key(
+        public_exponent=65537, key_size=2048, backend=default_backend()
+    )
+
+
+def setup_acme_client():
+    key = current_app.config.get('ACME_PRIVATE_KEY').strip()
+    acme_email = current_app.config.get('ACME_EMAIL')
+    acme_tel = current_app.config.get('ACME_TEL')
+    acme_directory_url = current_app.config.get('ACME_DIRECTORY_URL'),
+    contact = ('mailto:{}'.format(acme_email), 'tel:{}'.format(acme_tel))
+
+    key = serialization.load_pem_private_key(
+        key, password=None, backend=default_backend()
+    )
+    return acme_client_for_private_key(acme_directory_url, key)
+
+
+def acme_client_for_private_key(acme_directory_url, private_key):
+    return Client(
+        acme_directory_url, key=jose.JWKRSA(key=private_key)
+    )
+
+
+def register(email):
+    private_key = generate_rsa_private_key()
+    acme_client = acme_client_for_private_key(current_app.config('ACME_DIRECTORY_URL'), private_key)
+
+    registration = acme_client.register(
+        messages.NewRegistration.from_data(email=email)
+    )
+    acme_client.agree_to_tos(registration)
+    return private_key
+
+
+def get_domains(options):
+    """
+    Fetches all domains currently requested
+    :param options:
+    :return:
+    """
+    domains = [options['common_name']]
+    for name in options['extensions']['sub_alt_name']['names']:
+        domains.append(name)
+    return domains
+
+
+def get_authorizations(acme_client, domains):
+    authorizations = []
+    try:
+        for domain in domains:
+            authz_record = start_dns_challenge(acme_client, domain)
+            authorizations.append(authz_record)
+
+        for authz_record in authorizations:
+            complete_dns_challenge(acme_client, authz_record)
+    finally:
+        for authz_record in authorizations:
+            dns_challenge = authz_record.dns_challenge
+            delete_txt_record(
+                authz_record.change_id,
+                dns_challenge.validation_domain_name(authz_record.host),
+                dns_challenge.validation(acme_client.key),
+            )
+
+    return authorizations
+
+
+class ACMEIssuerPlugin(IssuerPlugin):
+    title = 'Acme'
+    slug = 'acme-issuer'
+    description = 'Enables the creation of certificates via ACME CAs (including Let\'s Encrypt)'
+    version = acme.VERSION
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur.git'
+
+    def __init__(self, *args, **kwargs):
+        super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
+
+    def create_certificate(self, csr, issuer_options):
+        """
+        Creates a ACME certificate.
+
+        :param csr:
+        :param issuer_options:
+        :return: :raise Exception:
+        """
+        current_app.logger.debug("Requesting a new acme certificate: {0}".format(issuer_options))
+        acme_client = setup_acme_client()
+        domains = get_domains(issuer_options)
+        authorizations = get_authorizations(acme_client, domains)
+        pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, csr)
+        return pem_certificate, pem_certificate_chain
+
+    @staticmethod
+    def create_authority(options):
+        """
+        Creates an authority, this authority is then used by Lemur to allow a user
+        to specify which Certificate Authority they want to sign their certificate.
+
+        :param options:
+        :return:
+        """
+        role = {'username': '', 'password': '', 'name': 'acme'}
+        return current_app.config.get('ACME_ROOT'), "", [role]