Closes: #469 (#510)

* Adding coverage.

* Attempting to adding coverage.

* Adding coveragerc.

.coveragerc

@@ -0,0 +1,2 @@
+[report]
+include = lemur/*.py

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+-   repo: git://github.com/pre-commit/pre-commit-hooks
+    sha: 18d7035de5388cc7775be57f529c154bf541aab9
+    hooks:
+    -   id: trailing-whitespace
+    -   id: flake8

.travis.yml

@@ -1,6 +1,9 @@
-sudo: false
-
 language: python
+sudo: required
+dist: trusty
+
+node_js:
+  - "4.2"
 
 addons:
   postgresql: "9.4"
@@ -20,15 +23,26 @@ cache:
 env:
   global:
     - PIP_DOWNLOAD_CACHE=".pip_download_cache"
+    # do not load /etc/boto.cfg with Python 3 incompatible plugin
+    # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
+    - BOTO_CONFIG=/doesnotexist
 
 before_script:
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
+  - npm config set registry https://registry.npmjs.org
   - npm install -g bower
+  - pip install --upgrade setuptools
+
+install:
+  - pip install coveralls
 
 script:
   - make test
 
+after_success:
+  - coveralls
+
 notifications:
   email:
     kglisson@netflix.com

CHANGELOG.rst

@@ -1,13 +1,39 @@
 Changelog
 =========
 
-0.3.1 - `master`
-~~~~~~~~~~~~~~~~
+0.5 - `master`
+~~~~~~~~~~~~~~
 
 .. note:: This version is not yet released and is under active development
 
+
+0.4 - ``
+~~~~~~~~
+
+There have been quite a few issues closed in this release. Some notables:
+
+* Closed `#284 <https://github.com/Netflix/lemur/issues/284>`_ - Created new models for `Endpoints` created associated
+AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for
+future enhancements of Lemur's certificate 'deployment' capabilities.
+
+* Closed `#334 <https://github.com/Netflix/lemur/issues/334>`_ - Lemur not has the ability
+to restrict certificate expiration dates to weekdays.
+
+Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!)
+
+This will most likely be the last release to support python2.7 moving Lemur to target python3 exclusively. Please comment
+on issue #340 if this negatively affects your usage of Lemur.
+
+Upgrading
+---------
+
+See the full list of issues closed in `0.4 <https://github.com/Netflix/lemur/milestone/3>`_.
+
+.. note:: This release will need a slight migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
+
+
 0.3.0 - `2016-06-06`
-~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~
 
 This is quite a large upgrade, it is highly advised you backup your database before attempting to upgrade as this release
 requires the migration of database structure as well as data.

Makefile

@@ -1,9 +1,16 @@
 NPM_ROOT = ./node_modules
 STATIC_DIR = src/lemur/static/app
 
+USER := $(shell whoami)
+
 develop: update-submodules setup-git
 	@echo "--> Installing dependencies"
+ifeq ($(USER), root)
+	@echo "WARNING: It looks like you are installing Lemur as root. This is not generally advised."
+	npm install --unsafe-perm
+else
 	npm install
+endif
 	pip install "setuptools>=0.9.8"
 	# order matters here, base package must install first
 	pip install -e .
@@ -41,7 +48,7 @@ test: develop lint test-python
 
 testloop: develop
 	pip install pytest-xdist
-	py.test tests -f
+	coverage run --source lemur -m py.test
 
 test-cli:
 	@echo "--> Testing CLI"
@@ -60,7 +67,7 @@ test-js:
 
 test-python:
 	@echo "--> Running Python tests"
-	py.test lemur/tests || exit 1
+	coverage run --source lemur -m py.test
 	@echo ""
 
 lint: lint-python lint-js
@@ -82,4 +89,4 @@ coverage: develop
 publish:
 	python setup.py sdist bdist_wheel upload
 
-.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish
+.PHONY: develop dev-postgres dev-docs setup-git build clean update-submodules test testloop test-cli test-js test-python lint lint-python lint-js coverage publish

README.rst

@@ -9,12 +9,11 @@ Lemur
     :target: https://lemur.readthedocs.org
     :alt: Latest Docs
 
+.. image:: https://img.shields.io/badge/NetflixOSS-active-brightgreen.svg
+
 .. image:: https://travis-ci.org/Netflix/lemur.svg
     :target: https://travis-ci.org/Netflix/lemur
 
-.. image:: https://requires.io/github/Netflix/lemur/requirements.svg?branch=master
-    :target: https://requires.io/github/Netflix/lemur/requirements/?branch=master
-    :alt: Requirements Status
 
 Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs
 and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults.

docs/administration.rst

@@ -51,7 +51,7 @@ Basic Configuration
         CORS = False
 
 
-.. data:: SQLACHEMY_DATABASE_URI
+.. data:: SQLALCHEMY_DATABASE_URI
     :noindex:
 
         If you have ever used sqlalchemy before this is the standard connection string used. Lemur uses a postgres database and the connection string would look something like:
@@ -61,6 +61,11 @@ Basic Configuration
         SQLALCHEMY_DATABASE_URI = 'postgresql://<user>:<password>@<hostname>:5432/lemur'
 
 
+.. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION
+    :noindex:
+
+        Specifies whether to allow certificates created by Lemur to expire on weekends. Default is True.
+
 .. data:: LEMUR_RESTRICTED_DOMAINS
     :noindex:
 
@@ -143,7 +148,7 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_ORGANIZATION = "Netflix"
 
 
-.. data:: LEMUR_DEFAULT_ORGANIZATION_UNIT
+.. data:: LEMUR_DEFAULT_ORGANIZATIONAL_UNIT
     :noindex:
 
     ::
@@ -151,6 +156,14 @@ and are used when Lemur creates the CSR for your certificates.
         LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = "Operations"
 
 
+.. data:: LEMUR_DEFAULT_ISSUER_PLUGIN
+    :noindex:
+
+    ::
+
+        LEMUR_DEFAULT_ISSUER_PLUGIN = "verisign-issuer"
+
+
 Notification Options
 --------------------
 
@@ -174,7 +187,7 @@ Lemur supports sending certification expiration notifications through SES and SM
     Specifies which service will be delivering notification emails. Valid values are `SMTP` or `SES`
 
     .. note::
-        If using SMP as your provider you will need to define additional configuration options as specified by Flask-Mail.
+        If using SMTP as your provider you will need to define additional configuration options as specified by Flask-Mail.
         See: `Flask-Mail <https://pythonhosted.org/Flask-Mail>`_
 
         If you are using SES the email specified by the `LEMUR_MAIL` configuration will need to be verified by AWS before
@@ -268,6 +281,20 @@ For more information about how to use social logins, see: `Satellizer <https://g
 
             PING_CLIENT_ID = "client-id"
 
+.. data:: PING_REDIRECT_URI
+    :noindex:
+
+        ::
+
+            PING_REDIRECT_URI = "https://<yourlemurserver>/api/1/auth/ping"
+
+.. data:: PING_AUTH_ENDPOINT
+    :noindex:
+
+        ::
+
+            PING_AUTH_ENDPOINT = "https://<yourpingserver>/oauth2/authorize"
+
 .. data:: GOOGLE_CLIENT_ID
     :noindex:
 
@@ -334,6 +361,69 @@ for those plugins.
         This is the root to be used for your CA chain
 
 
+Digicert Issuer Plugin
+~~~~~~~~~~~~~~~~~~~~~~
+
+The following configuration properties are required to use the Digicert issuer plugin.
+
+
+.. data:: DIGICERT_URL
+    :noindex:
+
+            This is the url for the Digicert API
+
+
+.. data:: DIGICERT_API_KEY
+    :noindex:
+
+            This is the Digicert API key
+
+
+.. data:: DIGICERT_ORG_ID
+    :noindex:
+
+            This is the Digicert organization ID tied to your API key
+
+
+.. data:: DIGICERT_INTERMEDIATE
+    :noindex:
+
+            This is the intermediate to be used for your CA chain
+
+
+.. data:: DIGICERT_ROOT
+    :noindex:
+
+            This is the root to be used for your CA chain
+
+
+.. data:: DIGICERT_DEFAULT_VALIDITY
+    :noindex:
+
+            This is the default validity (in years), if no end date is specified. (Default: 1)
+
+
+
+CFSSL Issuer Plugin
+^^^^^^^^^^^^^^^^^^^
+
+The following configuration properties are required to use the the CFSSL issuer plugin.
+
+.. data:: CFSSL_URL
+    :noindex:
+
+        This is the URL for the CFSSL API
+
+.. data:: CFSSL_ROOT
+    :noindex:
+
+        This is the root to be used for your CA chain
+
+.. data:: CFSSL_INTERMEDIATE
+    :noindex:
+
+        This is the intermediate to be used for your CA chain
+
 
 AWS Source/Destination Plugin
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -682,15 +772,161 @@ Plugins
 There are several interfaces currently available to extend Lemur. These are a work in
 progress and the API is not frozen.
 
-Bundled Plugins
----------------
+Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec.
 
-Lemur includes several plugins by default. Including extensive support for AWS, VeriSign/Symantec and CloudCA services.
+Verisign/Symantec
+-----------------
 
-3rd Party Extensions
---------------------
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for the VICE 2.0 API
+
+
+Cryptography
+------------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Issuer
+:Description:
+    Toy certificate authority that creates self-signed certificate authorities.
+    Allows for the creation of arbitrary authorities and end-entity certificates.
+    This is *not* recommended for production use.
+
+
+Acme
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>,
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Issuer
+:Description:
+    Adds support for the ACME protocol (including LetsEncrypt) with domain validation being handled Route53.
+
+
+Atlas
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Metric
+:Description:
+    Adds basic support for the `Atlas <https://github.com/Netflix/atlas/wiki>`_ telemetry system.
+
+
+Email
+-----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Notification
+:Description:
+    Adds support for basic email notifications via SES.
+
+
+Slack
+-----
+
+:Authors:
+    Harm Weites <harm@weites.com>
+:Type:
+    Notification
+:Description:
+    Adds support for slack notifications.
+
+
+AWS
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Source
+:Description:
+    Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment.
+
+
+AWS
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Destination
+:Description:
+    Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment.
+
+
+Kubernetes
+----------
+
+:Authors:
+    Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+:Type:
+    Destination
+:Description:
+    Allows Lemur to upload generated certificates to the Kubernetes certificate store.
+
+
+Java
+----
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Generates java compatible .jks keystores and truststores from Lemur managed certificates.
+
+
+Openssl
+-------
+
+:Authors:
+    Kevin Glisson <kglisson@netflix.com>
+:Type:
+    Export
+:Description:
+    Leverages Openssl to support additional export formats (pkcs12)
+
+
+CFSSL
+-----
+
+:Authors:
+    Charles Hendrie <chad.hendrie@thomsonreuters.com>
+:Type:
+    Issuer
+:Description:
+    Basic support for generating certificates from the private certificate authority CFSSL
+
+
+3rd Party Plugins
+=================
+
+The following plugins are available and maintained by members of the Lemur community:
+
+Digicert
+--------
+
+:Authors:
+    Chris Dorros
+:Type:
+    Issuer
+:Description:
+    Adds support for basic Digicert
+:Links:
+    https://github.com/opendns/lemur-digicert
 
-The following extensions are available and maintained by members of the Lemur community:
 
 Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
 get it added.
@@ -724,4 +960,3 @@ These permissions are applied to the user upon login and refreshed on every requ
 .. seealso::
 
     `Flask-Principal <https://pythonhosted.org/Flask-Principal>`_
-

docs/developer/index.rst

@@ -205,7 +205,7 @@ REST API
 ========
 
 Lemur's front end is entirely API driven. Any action that you can accomplish via the UI can also be accomplished by the
-UI. The following is documents and provides examples on how to make requests to the Lemur API.
+API. The following is documents and provides examples on how to make requests to the Lemur API.
 
 Authentication
 --------------

docs/developer/internals/lemur.plugins.lemur_cfssl.rst

@@ -0,0 +1,20 @@
+lemur_cfssl Package
+===================
+
+:mod:`lemur_cfssl` Package
+--------------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+:mod:`plugin` Module
+--------------------
+
+.. automodule:: lemur.plugins.lemur_cfssl.plugin
+    :noindex:
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/developer/internals/lemur.plugins.rst

@@ -27,5 +27,6 @@ Subpackages
     lemur.plugins.base
     lemur.plugins.bases
     lemur.plugins.lemur_aws
+    lemur.plugins.lemur_cfssl
     lemur.plugins.lemur_email
     lemur.plugins.lemur_verisign

docs/developer/plugins/index.rst

@@ -137,6 +137,12 @@ Destination
 Destination plugins allow you to propagate certificates managed by Lemur to additional third parties. This provides flexibility when
 different orchestration systems have their own way of manage certificates or there is an existing system you wish to integrate with Lemur.
 
+By default destination plugins have a private key requirement. If your plugin does not require a certificates private key mark `requires_key = False`
+in the plugins base class like so::
+
+    class MyDestinationPlugin(DestinationPlugin):
+        requires_key = False
+
 The DestinationPlugin requires only one function to be implemented::
 
     def upload(self, cert, private_key, cert_chain, options, **kwargs):
@@ -277,11 +283,7 @@ The ``conftest.py`` file is our main entry-point for py.test. We need to configu
 
 .. code-block:: python
 
-   from __future__ import absolute_import
-
-   pytest_plugins = [
-       'lemur.utils.pytest'
-   ]
+   from lemur.tests.conftest import *  # noqa
 
 
 Test Cases
@@ -291,14 +293,18 @@ You can now inherit from Lemur's core test classes. These are Django-based and e
 
 .. code-block:: python
 
-   # test_myextension.py
-   from __future__ import absolute_import
+    import pytest
+    from lemur.tests.vectors import INTERNAL_CERTIFICATE_A_STR, INTERNAL_PRIVATE_KEY_A_STR
 
-   from lemur.testutils import TestCase
+    def test_export_keystore(app):
+        from lemur.plugins.base import plugins
+        p = plugins.get('java-keystore-jks')
+        options = [{'name': 'passphrase', 'value': 'test1234'}]
+        with pytest.raises(Exception):
+            p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
 
-   class MyExtensionTest(TestCase):
-       def test_simple(self):
-          assert 1 != 2
+        raw = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
+        assert raw != b""
 
 
 Running Tests
@@ -310,13 +316,14 @@ Running tests follows the py.test standard. As long as your test files and metho
 
     $ py.test -v
     ============================== test session starts ==============================
-    platform darwin -- Python 2.7.9 -- py-1.4.26 -- pytest-2.6.4/python2.7
-    plugins: django
-    collected 1 items
+    platform darwin -- Python 2.7.10, pytest-2.8.5, py-1.4.30, pluggy-0.3.1
+    cachedir: .cache
+    plugins: flask-0.10.0
+    collected 346 items
 
-    tests/test_myextension.py::MyExtensionTest::test_simple PASSED
+    lemur/plugins/lemur_acme/tests/test_acme.py::test_get_certificates PASSED
 
     =========================== 1 passed in 0.35 seconds ============================
 
 
-.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above. View the source: # TODO
+.. SeeAlso:: Lemur bundles several plugins that use the same interfaces mentioned above.

docs/requirements.txt

@@ -1,29 +1,54 @@
-Jinja2>=2.3
-Pygments>=1.2
-Sphinx>=1.3
-docutils>=0.7
-markupsafe
-sphinxcontrib-httpdomain
-Flask==0.10.1
-Flask-RESTful==0.3.3
-Flask-SQLAlchemy==2.1
-Flask-Script==2.0.5
-Flask-Migrate==1.7.0
-Flask-Bcrypt==0.7.1
-Flask-Principal==0.4.0
-Flask-Mail==0.9.1
-SQLAlchemy-Utils==0.31.4
-BeautifulSoup4
-requests==2.9.1
-psycopg2==2.6.1
+alabaster==0.7.8
+alembic==0.8.6
+aniso8601==1.1.0
 arrow==0.7.0
-boto==2.38.0  # we might make this optional
-six==1.10.0
-gunicorn==19.4.4
-pycrypto==2.6.1
-cryptography==1.1.2
-pyopenssl==0.15.1
-pyjwt==1.4.0
-xmltodict==0.9.2
-lockfile==0.12.2
+Babel==2.3.4
+bcrypt==2.0.0
+beautifulsoup4==4.4.1
+blinker==1.4
+boto==2.38.0
+cffi==1.7.0
+cryptography==1.3.2
+docutils==0.12
+enum34==1.1.6
+Flask==0.10.1
+Flask-Bcrypt==0.7.1
+Flask-Mail==0.9.1
+Flask-Migrate==1.7.0
+Flask-Principal==0.4.0
+Flask-RESTful==0.3.3
+Flask-Script==2.0.5
+Flask-SQLAlchemy==2.1
 future==0.15.2
+gunicorn==19.4.1
+idna==2.1
+imagesize==0.7.1
+inflection==0.3.1
+ipaddress==1.0.16
+itsdangerous==0.24
+Jinja2==2.8
+lockfile==0.12.2
+Mako==1.0.4
+MarkupSafe==0.23
+marshmallow==2.4.0
+marshmallow-sqlalchemy==0.8.0
+psycopg2==2.6.1
+pyasn1==0.1.9
+pycparser==2.14
+pycrypto==2.6.1
+Pygments==2.1.3
+PyJWT==1.4.0
+pyOpenSSL==0.15.1
+python-dateutil==2.5.3
+python-editor==1.0.1
+pytz==2016.4
+requests==2.9.1
+six==1.10.0
+snowballstemmer==1.2.1
+Sphinx==1.4.4
+sphinx-rtd-theme==0.1.9
+sphinxcontrib-httpdomain==1.5.0
+SQLAlchemy==1.0.13
+SQLAlchemy-Utils==0.31.4
+Werkzeug==0.11.10
+xmltodict==0.9.2

gulp/build.js

@@ -79,7 +79,7 @@ gulp.task('dev:styles', function () {
     'bower_components/angular-loading-bar/src/loading-bar.css',
     'bower_components/angular-ui-switch/angular-ui-switch.css',
     'bower_components/angular-wizard/dist/angular-wizard.css',
-    'bower_components/ng-table/ng-table.css',
+    'bower_components/ng-table/dist/ng-table.css',
     'bower_components/angularjs-toaster/toaster.css',
     'bower_components/angular-ui-select/dist/select.css',
     'lemur/static/app/styles/lemur.css'

hooks/pre-commit

@@ -17,13 +17,12 @@ if 'VIRTUAL_ENV' in os.environ:
 
 
 def py_lint(files_modified):
-    from flake8.main import DEFAULT_CONFIG
     from flake8.engine import get_style_guide
 
     # remove non-py files and files which no longer exist
     files_modified = filter(lambda x: x.endswith('.py'), files_modified)
 
-    flake8_style = get_style_guide(parse_argv=True, config_file=DEFAULT_CONFIG)
+    flake8_style = get_style_guide(parse_argv=True)
     report = flake8_style.check_files(files_modified)
 
     return report.total_errors != 0

lemur/__about__.py

@@ -9,7 +9,7 @@ __title__ = "lemur"
 __summary__ = ("Certificate management and orchestration service")
 __uri__ = "https://github.com/Netflix/lemur"
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 __author__ = "The Lemur developers"
 __email__ = "security@netflix.com"

lemur/__init__.py

@@ -24,6 +24,7 @@ from lemur.defaults.views import mod as defaults_bp
 from lemur.plugins.views import mod as plugins_bp
 from lemur.notifications.views import mod as notifications_bp
 from lemur.sources.views import mod as sources_bp
+from lemur.endpoints.views import mod as endpoints_bp
 
 from lemur.__about__ import (
     __author__, __copyright__, __email__, __license__, __summary__, __title__,
@@ -47,7 +48,8 @@ LEMUR_BLUEPRINTS = (
     defaults_bp,
     plugins_bp,
     notifications_bp,
-    sources_bp
+    sources_bp,
+    endpoints_bp
 )
 
 
@@ -63,7 +65,8 @@ def configure_hook(app):
     :param app:
     :return:
     """
-    from flask.ext.principal import PermissionDenied
+    from flask import jsonify
+    from werkzeug.exceptions import default_exceptions
     from lemur.decorators import crossdomain
     if app.config.get('CORS'):
         @app.after_request
@@ -71,17 +74,13 @@ def configure_hook(app):
         def after(response):
             return response
 
-    @app.errorhandler(500)
-    def internal_error(error):
-        metrics.send('500_status_code', 'counter', 1)
+    def make_json_handler(code):
+        def json_handler(error):
+            metrics.send('{}_status_code'.format(code), 'counter', 1)
+            response = jsonify(message=str(error))
+            response.status_code = code
+            return response
+        return json_handler
 
-    @app.errorhandler(400)
-    def response_error(error):
-        metrics.send('400_status_code', 'counter', 1)
-
-    @app.errorhandler(PermissionDenied)
-    def permission_denied_error(error):
-        metrics.send('403_status_code', 'counter', 1)
-        response = {'message': 'You are not allow to access this resource'}
-        response.status_code = 403
-        return response
+    for code, value in default_exceptions.items():
+        app.error_handler_spec[None][code] = make_json_handler(code)

lemur/auth/permissions.py

@@ -27,35 +27,23 @@ class SensitiveDomainPermission(Permission):
         super(SensitiveDomainPermission, self).__init__(RoleNeed('admin'))
 
 
-class ViewKeyPermission(Permission):
-    def __init__(self, certificate_id, owner):
-        c_need = CertificateCreatorNeed(certificate_id)
-        super(ViewKeyPermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
-
-
-class UpdateCertificatePermission(Permission):
-    def __init__(self, certificate_id, owner):
-        c_need = CertificateCreatorNeed(certificate_id)
-        super(UpdateCertificatePermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin'))
-
-
 class CertificatePermission(Permission):
-    def __init__(self, certificate_id, roles):
-        needs = [RoleNeed('admin'), CertificateCreatorNeed(certificate_id)]
+    def __init__(self, certificate_id, owner, roles):
+        needs = [RoleNeed('admin'), CertificateCreatorNeed(certificate_id), RoleNeed(owner)]
         for r in roles:
             needs.append(CertificateOwnerNeed(str(r)))
 
         super(CertificatePermission, self).__init__(*needs)
 
 
-RoleUser = namedtuple('role', ['method', 'value'])
-ViewRoleCredentialsNeed = partial(RoleUser, 'roleView')
+RoleMember = namedtuple('role', ['method', 'value'])
+RoleMemberNeed = partial(RoleMember, 'member')
 
 
-class ViewRoleCredentialsPermission(Permission):
+class RoleMemberPermission(Permission):
     def __init__(self, role_id):
-        need = ViewRoleCredentialsNeed(role_id)
-        super(ViewRoleCredentialsPermission, self).__init__(need, RoleNeed('admin'))
+        needs = [RoleNeed('admin'), RoleMemberNeed(role_id)]
+        super(RoleMemberPermission, self).__init__(*needs)
 
 
 AuthorityCreator = namedtuple('authority', ['method', 'value'])

lemur/auth/service.py

@@ -8,11 +8,9 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from __future__ import unicode_literals
-from builtins import bytes
+import sys
 import jwt
 import json
-import base64
 import binascii
 
 from functools import wraps
@@ -31,20 +29,7 @@ from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicNumbers
 
 from lemur.users import service as user_service
 from lemur.auth.permissions import CertificateCreatorNeed, \
-    AuthorityCreatorNeed, ViewRoleCredentialsNeed
-
-
-def base64url_decode(data):
-    rem = len(data) % 4
-
-    if rem > 0:
-        data += '=' * (4 - rem)
-
-    return base64.urlsafe_b64decode(bytes(data.encode('latin-1')))
-
-
-def base64url_encode(data):
-    return base64.urlsafe_b64encode(data).replace('=', '')
+    AuthorityCreatorNeed, RoleMemberNeed
 
 
 def get_rsa_public_key(n, e):
@@ -55,8 +40,13 @@ def get_rsa_public_key(n, e):
     :param e:
     :return: a RSA Public Key in PEM format
     """
-    n = int(binascii.hexlify(base64url_decode(n)), 16)
-    e = int(binascii.hexlify(base64url_decode(e)), 16)
+    if sys.version_info >= (3, 0):
+        n = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(n, 'utf-8'))), 16)
+        e = int(binascii.hexlify(jwt.utils.base64url_decode(bytes(e, 'utf-8'))), 16)
+    else:
+        n = int(binascii.hexlify(jwt.utils.base64url_decode(str(n))), 16)
+        e = int(binascii.hexlify(jwt.utils.base64url_decode(str(e))), 16)
+
     pub = RSAPublicNumbers(e, n).public_key(default_backend())
     return pub.public_bytes(
         encoding=serialization.Encoding.PEM,
@@ -75,8 +65,8 @@ def create_token(user):
     expiration_delta = timedelta(days=int(current_app.config.get('LEMUR_TOKEN_EXPIRATION', 1)))
     payload = {
         'sub': user.id,
-        'iat': datetime.now(),
-        'exp': datetime.now() + expiration_delta
+        'iat': datetime.utcnow(),
+        'exp': datetime.utcnow() + expiration_delta
     }
     token = jwt.encode(payload, current_app.config['LEMUR_TOKEN_SECRET'])
     return token.decode('unicode_escape')
@@ -138,13 +128,13 @@ def fetch_token_header(token):
         raise jwt.DecodeError('Not enough segments')
 
     try:
-        return json.loads(base64url_decode(header_segment))
+        if sys.version_info >= (3, 0):
+            return json.loads(jwt.utils.base64url_decode(header_segment).decode('utf-8'))
+        else:
+            return json.loads(jwt.utils.base64url_decode(header_segment))
     except TypeError as e:
         current_app.logger.exception(e)
         raise jwt.DecodeError('Invalid header padding')
-    except binascii.Error as e:
-        current_app.logger.exception(e)
-        raise jwt.DecodeError('Invalid header padding')
 
 
 @identity_loaded.connect
@@ -165,8 +155,8 @@ def on_identity_loaded(sender, identity):
     # identity with the roles that the user provides
     if hasattr(user, 'roles'):
         for role in user.roles:
-            identity.provides.add(ViewRoleCredentialsNeed(role.name))
             identity.provides.add(RoleNeed(role.name))
+            identity.provides.add(RoleMemberNeed(role.id))
 
     # apply ownership for authorities
     if hasattr(user, 'authorities'):

lemur/auth/views.py

@@ -5,6 +5,7 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import sys
 import jwt
 import base64
 import requests
@@ -97,6 +98,7 @@ class Login(Resource):
             # Tell Flask-Principal the identity changed
             identity_changed.send(current_app._get_current_object(),
                                   identity=Identity(user.id))
+
             metrics.send('successful_login', 'counter', 1)
             return dict(token=create_token(user))
 
@@ -139,8 +141,14 @@ class Ping(Resource):
         user_api_url = current_app.config.get('PING_USER_API_URL')
 
         # the secret and cliendId will be given to you when you signup for the provider
-        basic = base64.b64encode('{0}:{1}'.format(args['clientId'], current_app.config.get("PING_SECRET")))
-        headers = {'Authorization': 'Basic {0}'.format(basic)}
+        token = '{0}:{1}'.format(args['clientId'], current_app.config.get("PING_SECRET"))
+
+        if sys.version_info >= (3, 0):
+            basic = base64.b64encode(bytes(token, 'utf-8'))
+            headers = {'authorization': 'basic {0}'.format(basic.decode('utf-8'))}
+        else:
+            basic = base64.b64encode(token, 'utf-8')
+            headers = {'authorization': 'basic {0}'.format(basic)}
 
         # exchange authorization code for access token.
 
@@ -164,7 +172,10 @@ class Ping(Resource):
 
         # validate your token based on the key it was signed with
         try:
-            jwt.decode(id_token, secret, algorithms=[algo], audience=args['clientId'])
+            if sys.version_info >= (3, 0):
+                jwt.decode(id_token, secret.decode('utf-8'), algorithms=[algo], audience=args['clientId'])
+            else:
+                jwt.decode(id_token, secret, algorithms=[algo], audience=args['clientId'])
         except jwt.DecodeError:
             return dict(message='Token is invalid'), 403
         except jwt.ExpiredSignatureError:
@@ -190,10 +201,13 @@ class Ping(Resource):
                 role = role_service.create(group, description='This is a google group based role created by Lemur')
             roles.append(role)
 
-        # if we get an sso user create them an account
-        # we still pick a random password in case sso is down
-        if not user:
+        role = role_service.get_by_name(profile['email'])
+        if not role:
+            role = role_service.create(profile['email'], description='This is a user specific role')
+        roles.append(role)
 
+        # if we get an sso user create them an account
+        if not user:
             # every user is an operator (tied to a default role)
             if current_app.config.get('LEMUR_DEFAULT_ROLE'):
                 v = role_service.get_by_name(current_app.config.get('LEMUR_DEFAULT_ROLE'))
@@ -275,7 +289,7 @@ class Providers(Resource):
     def get(self):
         active_providers = []
 
-        for provider in current_app.config.get("ACTIVE_PROVIDERS"):
+        for provider in current_app.config.get("ACTIVE_PROVIDERS", []):
             provider = provider.lower()
 
             if provider == "google":

lemur/authorities/models.py

@@ -11,6 +11,7 @@ from sqlalchemy import Column, Integer, String, Text, func, ForeignKey, DateTime
 from sqlalchemy.dialects.postgresql import JSON
 
 from lemur.database import db
+from lemur.plugins.base import plugins
 from lemur.models import roles_authorities
 
 
@@ -38,3 +39,10 @@ class Authority(db.Model):
         self.description = kwargs.get('description')
         self.authority_certificate = kwargs['authority_certificate']
         self.plugin_name = kwargs['plugin']['slug']
+
+    @property
+    def plugin(self):
+        return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Authority(name={name})".format(name=self.name)

lemur/authorities/schemas.py

@@ -7,14 +7,16 @@
 """
 from flask import current_app
 
-from marshmallow import fields, validates_schema
+from marshmallow import fields, validates_schema, pre_load
 from marshmallow import validate
 from marshmallow.exceptions import ValidationError
 
 from lemur.schemas import PluginInputSchema, PluginOutputSchema, ExtensionSchema, AssociatedAuthoritySchema, AssociatedRoleSchema
 from lemur.users.schemas import UserNestedOutputSchema
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
-from lemur.common import validators
+from lemur.common import validators, missing
+
+from lemur.common.fields import ArrowDateTime
 
 
 class AuthorityInputSchema(LemurInputSchema):
@@ -23,8 +25,8 @@ class AuthorityInputSchema(LemurInputSchema):
     description = fields.String()
     common_name = fields.String(required=True, validate=validators.sensitive_domain)
 
-    validity_start = fields.Date()
-    validity_end = fields.Date()
+    validity_start = ArrowDateTime()
+    validity_end = ArrowDateTime()
     validity_years = fields.Integer()
 
     # certificate body fields
@@ -60,6 +62,10 @@ class AuthorityInputSchema(LemurInputSchema):
             if not data.get('parent'):
                 raise ValidationError("If generating a subca parent 'authority' must be specified.")
 
+    @pre_load
+    def ensure_dates(self, data):
+        return missing.convert_validity_years(data)
+
 
 class AuthorityUpdateSchema(LemurInputSchema):
     owner = fields.Email(required=True)
@@ -98,6 +104,7 @@ class AuthorityOutputSchema(LemurOutputSchema):
 
 
 class AuthorityNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
     id = fields.Integer()
     description = fields.String()
     name = fields.String()

lemur/authorities/service.py

@@ -44,9 +44,17 @@ def mint(**kwargs):
     Creates the authority based on the plugin provided.
     """
     issuer = kwargs['plugin']['plugin_object']
-    body, chain, roles = issuer.create_authority(kwargs)
+    values = issuer.create_authority(kwargs)
+
+    # support older plugins
+    if len(values) == 3:
+        body, chain, roles = values
+        private_key = None
+    elif len(values) == 4:
+        body, private_key, chain, roles = values
+
     roles = create_authority_roles(roles, kwargs['owner'], kwargs['plugin']['plugin_object'].title)
-    return body, chain, roles
+    return body, private_key, chain, roles
 
 
 def create_authority_roles(roles, owner, plugin_title):
@@ -88,9 +96,12 @@ def create(**kwargs):
     Creates a new authority.
     """
     kwargs['creator'] = g.user.email
-    body, chain, roles = mint(**kwargs)
+    body, private_key, chain, roles = mint(**kwargs)
+
+    g.user.roles = list(set(list(g.user.roles) + roles))
 
     kwargs['body'] = body
+    kwargs['private_key'] = private_key
     kwargs['chain'] = chain
 
     if kwargs.get('roles'):
@@ -98,16 +109,6 @@ def create(**kwargs):
     else:
         kwargs['roles'] = roles
 
-    if kwargs['type'] == 'subca':
-        description = "This is the ROOT certificate for the {0} sub certificate authority the parent \
-                                authority is {1}.".format(kwargs.get('name'), kwargs.get('parent'))
-    else:
-        description = "This is the ROOT certificate for the {0} certificate authority.".format(
-            kwargs.get('name')
-        )
-
-    kwargs['description'] = description
-
     cert = upload(**kwargs)
     kwargs['authority_certificate'] = cert
 
@@ -182,6 +183,9 @@ def render(args):
     # we make sure that a user can only use an authority they either own are are a member of - admins can see all
     if not g.current_user.is_admin:
         authority_ids = []
+        for authority in g.current_user.authorities:
+            authority_ids.append(authority.id)
+
         for role in g.current_user.roles:
             for authority in role.authorities:
                 authority_ids.append(authority.id)

lemur/certificates/models.py

@@ -7,10 +7,13 @@
 """
 import datetime
 
+import lemur.common.utils
 from flask import current_app
 
-from sqlalchemy import event, Integer, ForeignKey, String, DateTime, PassiveDefault, func, Column, Text, Boolean
 from sqlalchemy.orm import relationship
+from sqlalchemy.sql.expression import case
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy import event, Integer, ForeignKey, String, DateTime, PassiveDefault, func, Column, Text, Boolean
 
 from lemur.database import db
 from lemur.models import certificate_associations, certificate_source_associations, \
@@ -24,6 +27,7 @@ from lemur.domains.models import Domain
 
 
 def get_or_increase_name(name):
+    name = '-'.join(name.strip().split(' '))
     count = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).count()
 
     if count >= 1:
@@ -36,9 +40,9 @@ class Certificate(db.Model):
     __tablename__ = 'certificates'
     id = Column(Integer, primary_key=True)
     owner = Column(String(128), nullable=False)
-    name = Column(String(128))  # , unique=True) TODO make all names unique
+    name = Column(String(128), unique=True)
     description = Column(String(1024))
-    active = Column(Boolean, default=True)
+    notify = Column(Boolean, default=True)
 
     body = Column(Text(), nullable=False)
     chain = Column(Text())
@@ -73,8 +77,10 @@ class Certificate(db.Model):
                             secondaryjoin=id == certificate_replacement_associations.c.replaced_certificate_id,  # noqa
                             backref='replaced')
 
+    endpoints = relationship("Endpoint", backref='certificate')
+
     def __init__(self, **kwargs):
-        cert = defaults.parse_certificate(kwargs['body'])
+        cert = lemur.common.utils.parse_certificate(kwargs['body'])
 
         self.issuer = defaults.issuer(cert)
         self.cn = defaults.common_name(cert)
@@ -84,14 +90,20 @@ class Certificate(db.Model):
 
         # when destinations are appended they require a valid name.
         if kwargs.get('name'):
-            self.name = kwargs['name']
+            self.name = get_or_increase_name(kwargs['name'])
         else:
             self.name = get_or_increase_name(defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san))
 
         self.owner = kwargs['owner']
-        self.body = kwargs['body']
-        self.private_key = kwargs.get('private_key')
-        self.chain = kwargs.get('chain')
+        self.body = kwargs['body'].strip()
+
+        if kwargs.get('private_key'):
+            self.private_key = kwargs['private_key'].strip()
+
+        if kwargs.get('chain'):
+            self.chain = kwargs['chain'].strip()
+
+        self.notify = kwargs.get('notify', True)
         self.destinations = kwargs.get('destinations', [])
         self.notifications = kwargs.get('notifications', [])
         self.description = kwargs.get('description')
@@ -105,21 +117,36 @@ class Certificate(db.Model):
             self.domains.append(Domain(name=domain))
 
     @property
-    def is_expired(self):
-        if self.not_after < datetime.datetime.now():
+    def active(self):
+        return self.notify
+
+    @hybrid_property
+    def expired(self):
+        if self.not_after <= datetime.datetime.now():
             return True
 
-    @property
-    def is_unused(self):
-        if self.elb_listeners.count() == 0:
+    @expired.expression
+    def expired(cls):
+        return case(
+            [
+                (cls.now_after <= datetime.datetime.now(), True)
+            ],
+            else_=False
+        )
+
+    @hybrid_property
+    def revoked(self):
+        if 'revoked' == self.status:
             return True
 
-    @property
-    def is_revoked(self):
-        # we might not yet know the condition of the cert
-        if self.status:
-            if 'revoked' in self.status:
-                return True
+    @revoked.expression
+    def revoked(cls):
+        return case(
+            [
+                (cls.status == 'revoked', True)
+            ],
+            else_=False
+        )
 
     def get_arn(self, account_number):
         """
@@ -131,6 +158,9 @@ class Certificate(db.Model):
         """
         return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)
 
+    def __repr__(self):
+        return "Certificate(name={name})".format(name=self.name)
+
 
 @event.listens_for(Certificate.destinations, 'append')
 def update_destinations(target, value, initiator):
@@ -143,6 +173,7 @@ def update_destinations(target, value, initiator):
     :return:
     """
     destination_plugin = plugins.get(value.plugin_name)
+
     try:
         destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
     except Exception as e:
@@ -152,26 +183,28 @@ def update_destinations(target, value, initiator):
 @event.listens_for(Certificate.replaces, 'append')
 def update_replacement(target, value, initiator):
     """
-    When a certificate is marked as 'replaced' it is then marked as in-active
+    When a certificate is marked as 'replaced' we should not notify.
 
     :param target:
     :param value:
     :param initiator:
     :return:
     """
-    value.active = False
+    value.notify = False
 
 
-@event.listens_for(Certificate, 'before_update')
-def protect_active(mapper, connection, target):
-    """
-    When a certificate has a replacement do not allow it to be marked as 'active'
-
-    :param connection:
-    :param mapper:
-    :param target:
-    :return:
-    """
-    if target.active:
-        if target.replaced:
-            raise Exception("Cannot mark certificate as active, certificate has been marked as replaced.")
+# @event.listens_for(Certificate, 'before_update')
+# def protect_active(mapper, connection, target):
+#    """
+#     When a certificate has a replacement do not allow it to be marked as 'active'
+#
+#     :param connection:
+#     :param mapper:
+#     :param target:
+#     :return:
+#     """
+#     if target.active:
+#         if not target.notify:
+#             raise Exception(
+#                 "Cannot silence notification for a certificate Lemur has been found to be currently deployed onto endpoints"
+#             )

lemur/certificates/schemas.py

@@ -6,11 +6,11 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import current_app
-from marshmallow import fields, validates_schema, post_load
+from marshmallow import fields, validates_schema, post_load, pre_load
 from marshmallow.exceptions import ValidationError
 
 from lemur.schemas import AssociatedAuthoritySchema, AssociatedDestinationSchema, AssociatedCertificateSchema, \
-    AssociatedNotificationSchema, PluginInputSchema, ExtensionSchema, AssociatedRoleSchema
+    AssociatedNotificationSchema, PluginInputSchema, ExtensionSchema, AssociatedRoleSchema, EndpointNestedOutputSchema
 
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
 from lemur.destinations.schemas import DestinationNestedOutputSchema
@@ -20,16 +20,20 @@ from lemur.domains.schemas import DomainNestedOutputSchema
 from lemur.users.schemas import UserNestedOutputSchema
 
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
-from lemur.common import validators
+from lemur.common import validators, missing
 from lemur.notifications import service as notification_service
 
+from lemur.common.fields import ArrowDateTime
+
 
 class CertificateSchema(LemurInputSchema):
     owner = fields.Email(required=True)
     description = fields.String()
 
+
+class CertificateCreationSchema(CertificateSchema):
     @post_load
-    def default_notifications(self, data):
+    def default_notification(self, data):
         if not data['notifications']:
             notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
             data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']])
@@ -39,13 +43,13 @@ class CertificateSchema(LemurInputSchema):
         return data
 
 
-class CertificateInputSchema(CertificateSchema):
+class CertificateInputSchema(CertificateCreationSchema):
     name = fields.String()
     common_name = fields.String(required=True, validate=validators.sensitive_domain)
     authority = fields.Nested(AssociatedAuthoritySchema, required=True)
 
-    validity_start = fields.DateTime()
-    validity_end = fields.DateTime()
+    validity_start = ArrowDateTime()
+    validity_end = ArrowDateTime()
     validity_years = fields.Integer()
 
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
@@ -55,6 +59,8 @@ class CertificateInputSchema(CertificateSchema):
 
     csr = fields.String(validate=validators.csr)
 
+    notify = fields.Boolean(default=True)
+
     # certificate body fields
     organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
     organization = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'))
@@ -68,14 +74,32 @@ class CertificateInputSchema(CertificateSchema):
     def validate_dates(self, data):
         validators.dates(data)
 
+    @pre_load
+    def ensure_dates(self, data):
+        return missing.convert_validity_years(data)
+
 
 class CertificateEditInputSchema(CertificateSchema):
-    active = fields.Boolean()
+    notify = fields.Boolean()
+    owner = fields.String()
     destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True)
     notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True)
     replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
     roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
 
+    @post_load
+    def enforce_notifications(self, data):
+        """
+        Ensures that when an owner changes, default notifications are added for the new owner.
+        Old owner notifications are retained unless explicitly removed.
+        :param data:
+        :return:
+        """
+        if data['owner']:
+            notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
+            data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']])
+        return data
+
 
 class CertificateNestedOutputSchema(LemurOutputSchema):
     __envelope__ = False
@@ -95,9 +119,16 @@ class CertificateNestedOutputSchema(LemurOutputSchema):
     issuer = fields.Nested(AuthorityNestedOutputSchema)
 
 
+class CertificateCloneSchema(LemurOutputSchema):
+    __envelope__ = False
+    description = fields.String()
+    common_name = fields.String()
+
+
 class CertificateOutputSchema(LemurOutputSchema):
     id = fields.Integer()
     active = fields.Boolean()
+    notify = fields.Boolean()
     bits = fields.Integer()
     body = fields.String()
     chain = fields.String()
@@ -120,12 +151,12 @@ class CertificateOutputSchema(LemurOutputSchema):
     replaces = fields.Nested(CertificateNestedOutputSchema, many=True)
     authority = fields.Nested(AuthorityNestedOutputSchema)
     roles = fields.Nested(RoleNestedOutputSchema, many=True)
-    endpoints = fields.List(fields.Dict(), missing=[])
+    endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
 
 
-class CertificateUploadInputSchema(CertificateSchema):
+class CertificateUploadInputSchema(CertificateCreationSchema):
     name = fields.String()
-    active = fields.Boolean(missing=True)
+    notify = fields.Boolean(missing=True)
 
     private_key = fields.String(validate=validators.private_key)
     body = fields.String(required=True, validate=validators.public_certificate)

lemur/certificates/service.py

@@ -67,16 +67,26 @@ def get_all_certs():
     return Certificate.query.all()
 
 
-def find_duplicates(cert_body):
+def get_by_source(source_label):
+    """
+    Retrieves all certificates from a given source.
+
+    :param source_label:
+    :return:
+    """
+    return Certificate.query.filter(Certificate.sources.any(label=source_label))
+
+
+def find_duplicates(cert):
     """
     Finds certificates that already exist within Lemur. We do this by looking for
     certificate bodies that are the same. This is the most reliable way to determine
     if a certificate is already being tracked by Lemur.
 
-    :param cert_body:
+    :param cert:
     :return:
     """
-    return Certificate.query.filter_by(body=cert_body).all()
+    return Certificate.query.filter_by(body=cert['body'].strip(), chain=cert['chain'].strip()).all()
 
 
 def export(cert, export_plugin):
@@ -92,20 +102,20 @@ def export(cert, export_plugin):
     return plugin.export(cert.body, cert.chain, cert.private_key, export_plugin['pluginOptions'])
 
 
-def update(cert_id, owner, description, active, destinations, notifications, replaces, roles):
+def update(cert_id, owner, description, notify, destinations, notifications, replaces, roles):
     """
     Updates a certificate
     :param cert_id:
     :param owner:
     :param description:
-    :param active:
+    :param notify:
     :param destinations:
     :param notifications:
     :param replaces:
     :return:
     """
     cert = get(cert_id)
-    cert.active = active
+    cert.notify = notify
     cert.description = description
     cert.destinations = destinations
     cert.notifications = notifications
@@ -162,14 +172,9 @@ def import_certificate(**kwargs):
 
     :param kwargs:
     """
-    from lemur.users import service as user_service
-
     if not kwargs.get('owner'):
         kwargs['owner'] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')[0]
 
-    if not kwargs.get('creator'):
-        kwargs['creator'] = user_service.get_by_email('lemur@nobody')
-
     return upload(**kwargs)
 
 
@@ -184,13 +189,21 @@ def upload(**kwargs):
     else:
         kwargs['roles'] = roles
 
+    if kwargs.get('private_key'):
+        private_key = kwargs['private_key']
+        if not isinstance(private_key, bytes):
+            kwargs['private_key'] = private_key.encode('utf-8')
+
     cert = Certificate(**kwargs)
 
     cert = database.create(cert)
-    g.user.certificates.append(cert)
 
-    database.update(cert)
-    return cert
+    try:
+        g.user.certificates.append(cert)
+    except AttributeError:
+        current_app.logger.debug("No user to associate uploaded certificate to.")
+
+    return database.update(cert)
 
 
 def create(**kwargs):
@@ -257,7 +270,7 @@ def render(args):
 
         elif 'destination' in terms:
             query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
-        elif 'active' in filt:  # this is really weird but strcmp seems to not work here??
+        elif 'active' in filt:
             query = query.filter(Certificate.active == terms[1])
         elif 'cn' in terms:
             query = query.filter(
@@ -314,6 +327,7 @@ def create_csr(**csr_config):
         x509.NameAttribute(x509.OID_COUNTRY_NAME, csr_config['country']),
         x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, csr_config['state']),
         x509.NameAttribute(x509.OID_LOCALITY_NAME, csr_config['location']),
+        x509.NameAttribute(x509.OID_EMAIL_ADDRESS, csr_config['owner'])
     ]))
 
     builder = builder.add_extension(
@@ -387,6 +401,9 @@ def create_csr(**csr_config):
         encryption_algorithm=serialization.NoEncryption()
     )
 
+    if isinstance(private_key, bytes):
+        private_key = private_key.decode('utf-8')
+
     csr = request.public_bytes(
         encoding=serialization.Encoding.PEM
     )
@@ -442,3 +459,49 @@ def get_name_from_arn(arn):
     :return: name of the certificate as uploaded to AWS
     """
     return arn.split("/", 1)[1]
+
+
+def calculate_reissue_range(start, end):
+    """
+    Determine what the new validity_start and validity_end dates should be.
+    :param start:
+    :param end:
+    :return:
+    """
+    span = end - start
+
+    new_start = arrow.utcnow().date()
+    new_end = new_start + span
+
+    return new_start, new_end
+
+
+# TODO pull the OU, O, CN, etc + other extensions.
+def get_certificate_primitives(certificate):
+    """
+    Retrieve key primitive from a certificate such that the certificate
+    could be recreated with new expiration or be used to build upon.
+    :param certificate:
+    :return: dict of certificate primitives, should be enough to effectively re-issue
+    certificate via `create`.
+    """
+    start, end = calculate_reissue_range(certificate.not_before, certificate.not_after)
+    names = [{'name_type': 'DNSName', 'value': x.name} for x in certificate.domains]
+
+    extensions = {
+        'sub_alt_names': {
+            'names': names
+        }
+    }
+
+    return dict(
+        authority=certificate.authority,
+        common_name=certificate.cn,
+        description=certificate.description,
+        validity_start=start,
+        validity_end=end,
+        destinations=certificate.destinations,
+        roles=certificate.roles,
+        extensions=extensions,
+        owner=certificate.owner
+    )

lemur/certificates/views.py

@@ -15,7 +15,7 @@ from lemur.common.schema import validate_schema
 from lemur.common.utils import paginated_parser
 
 from lemur.auth.service import AuthenticatedResource
-from lemur.auth.permissions import ViewKeyPermission, AuthorityPermission, CertificatePermission
+from lemur.auth.permissions import AuthorityPermission, CertificatePermission
 
 from lemur.certificates import service
 from lemur.certificates.schemas import certificate_input_schema, certificate_output_schema, \
@@ -146,6 +146,39 @@ class CertificatesList(AuthenticatedResource):
               Host: example.com
               Accept: application/json, text/javascript
 
+              {
+                  "owner": "secure@example.net",
+                  "commonName": "test.example.net",
+                  "country": "US",
+                  "extensions": {
+                    "subAltNames": {
+                      "names": [
+                        {
+                          "nameType": "DNSName",
+                          "value": "*.test.example.net"
+                        },
+                        {
+                          "nameType": "DNSName",
+                          "value": "www.test.example.net"
+                        }
+                      ]
+                    }
+                  },
+                  "replacements": [{
+                    "id": 1
+                  },
+                  "notify": true,
+                  "validityEnd": "2026-01-01T08:00:00.000Z",
+                  "authority": {
+                    "name": "verisign"
+                  },
+                  "organization": "Netflix, Inc.",
+                  "location": "Los Gatos",
+                  "state": "California",
+                  "validityStart": "2016-11-11T04:19:48.000Z",
+                  "organizationalUnit": "Operations"
+              }
+
 
            **Example response**:
 
@@ -193,7 +226,9 @@ class CertificatesList(AuthenticatedResource):
                     "id": 1090,
                     "name": "*.test.example.net"
                 }],
-                "replaces": [],
+                "replaces": [{
+                    "id": 1
+                }],
                 "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
                 "roles": [{
                     "id": 464,
@@ -399,9 +434,8 @@ class CertificatePrivateKey(AuthenticatedResource):
         if not cert:
             return dict(message="Cannot find specified certificate"), 404
 
-        role = role_service.get_by_name(cert.owner)
-
-        permission = ViewKeyPermission(certificate_id, getattr(role, 'name', None))
+        owner_role = role_service.get_by_name(cert.owner)
+        permission = CertificatePermission(cert.id, owner_role, [x.name for x in cert.roles])
 
         if permission.can():
             response = make_response(jsonify(key=cert.private_key), 200)
@@ -581,14 +615,20 @@ class Certificates(AuthenticatedResource):
         """
         cert = service.get(certificate_id)
 
-        permission = CertificatePermission(cert.id, [x.name for x in cert.roles])
+        owner_role = role_service.get_by_name(cert.owner)
+        permission = CertificatePermission(cert.id, owner_role, [x.name for x in cert.roles])
 
         if permission.can():
+            for destination in data['destinations']:
+                if destination.plugin.requires_key:
+                    if not cert.private_key:
+                        return dict('Unable to add destination: {0}. Certificate does not have required private key.'.format(destination.label))
+
             return service.update(
                 certificate_id,
                 data['owner'],
                 data['description'],
-                data['active'],
+                data['notify'],
                 data['destinations'],
                 data['notifications'],
                 data['replacements'],
@@ -864,21 +904,36 @@ class CertificateExport(AuthenticatedResource):
         """
         cert = service.get(certificate_id)
 
-        permission = CertificatePermission(cert.id, [x.name for x in cert.roles])
+        owner_role = role_service.get_by_name(cert.owner)
+        permission = CertificatePermission(cert.id, owner_role, [x.name for x in cert.roles])
 
         options = data['plugin']['plugin_options']
         plugin = data['plugin']['plugin_object']
 
         if plugin.requires_key:
-            if permission.can():
-                extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
+            if cert.private_key:
+                if permission.can():
+                    extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
+                else:
+                    return dict(message='You are not authorized to export this certificate.'), 403
             else:
-                return dict(message='You are not authorized to export this certificate'), 403
+                return dict(message='Unable to export certificate, plugin: {0} requires a private key but no key was found.'.format(plugin.slug))
         else:
             extension, passphrase, data = plugin.export(cert.body, cert.chain, cert.private_key, options)
 
         # we take a hit in message size when b64 encoding
-        return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data))
+        return dict(extension=extension, passphrase=passphrase, data=base64.b64encode(data).decode('utf-8'))
+
+
+class CertificateClone(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(CertificateExport, self).__init__()
+
+    @validate_schema(None, certificate_output_schema)
+    def get(self, certificate_id):
+
+        pass
 
 
 api.add_resource(CertificatesList, '/certificates', endpoint='certificates')
@@ -887,6 +942,7 @@ api.add_resource(CertificatesStats, '/certificates/stats', endpoint='certificate
 api.add_resource(CertificatesUpload, '/certificates/upload', endpoint='certificateUpload')
 api.add_resource(CertificatePrivateKey, '/certificates/<int:certificate_id>/key', endpoint='privateKeyCertificates')
 api.add_resource(CertificateExport, '/certificates/<int:certificate_id>/export', endpoint='exportCertificate')
+api.add_resource(CertificateClone, '/certificates/<int:certificate_id>/clone', endpoint='cloneCertificate')
 api.add_resource(NotificationCertificatesList, '/notifications/<int:notification_id>/certificates',
                  endpoint='notificationCertificates')
 api.add_resource(CertificatesReplacementsList, '/certificates/<int:certificate_id>/replacements',

lemur/common/defaults.py

@@ -1,17 +1,8 @@
-import sys
-from flask import current_app
 from cryptography import x509
-from cryptography.hazmat.backends import default_backend
+from flask import current_app
 from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE
 
 
-def parse_certificate(body):
-    if sys.version_info >= (3, 0):
-        return x509.load_pem_x509_certificate(body, default_backend())
-    else:
-        return x509.load_pem_x509_certificate(bytes(body), default_backend())
-
-
 def certificate_name(common_name, issuer, not_before, not_after, san):
     """
     Create a name for our certificate. A naming standard
@@ -24,7 +15,7 @@ def certificate_name(common_name, issuer, not_before, not_after, san):
     :param not_after:
     :param issuer:
     :param not_before:
-    :rtype : str
+    :rtype: str
     :return:
     """
     if san:
@@ -132,7 +123,10 @@ def bitstrength(cert):
     :param cert:
     :return: Integer
     """
-    return cert.public_key().key_size
+    try:
+        return cert.public_key().key_size
+    except AttributeError:
+        current_app.logger.debug('Unable to get bitstrength.')
 
 
 def issuer(cert):

lemur/common/fields.py

@@ -0,0 +1,93 @@
+import arrow
+import warnings
+from datetime import datetime as dt
+from marshmallow.fields import Field
+from marshmallow import utils
+
+
+class ArrowDateTime(Field):
+    """A formatted datetime string in UTC.
+
+    Example: ``'2014-12-22T03:12:58.019077+00:00'``
+
+    Timezone-naive `datetime` objects are converted to
+    UTC (+00:00) by :meth:`Schema.dump <marshmallow.Schema.dump>`.
+    :meth:`Schema.load <marshmallow.Schema.load>` returns `datetime`
+    objects that are timezone-aware.
+
+    :param str format: Either ``"rfc"`` (for RFC822), ``"iso"`` (for ISO8601),
+        or a date format string. If `None`, defaults to "iso".
+    :param kwargs: The same keyword arguments that :class:`Field` receives.
+
+    """
+
+    DATEFORMAT_SERIALIZATION_FUNCS = {
+        'iso': utils.isoformat,
+        'iso8601': utils.isoformat,
+        'rfc': utils.rfcformat,
+        'rfc822': utils.rfcformat,
+    }
+
+    DATEFORMAT_DESERIALIZATION_FUNCS = {
+        'iso': utils.from_iso,
+        'iso8601': utils.from_iso,
+        'rfc': utils.from_rfc,
+        'rfc822': utils.from_rfc,
+    }
+
+    DEFAULT_FORMAT = 'iso'
+
+    localtime = False
+    default_error_messages = {
+        'invalid': 'Not a valid datetime.',
+        'format': '"{input}" cannot be formatted as a datetime.',
+    }
+
+    def __init__(self, format=None, **kwargs):
+        super(ArrowDateTime, self).__init__(**kwargs)
+        # Allow this to be None. It may be set later in the ``_serialize``
+        # or ``_desrialize`` methods This allows a Schema to dynamically set the
+        # dateformat, e.g. from a Meta option
+        self.dateformat = format
+
+    def _add_to_schema(self, field_name, schema):
+        super(ArrowDateTime, self)._add_to_schema(field_name, schema)
+        self.dateformat = self.dateformat or schema.opts.dateformat
+
+    def _serialize(self, value, attr, obj):
+        if value is None:
+            return None
+        self.dateformat = self.dateformat or self.DEFAULT_FORMAT
+        format_func = self.DATEFORMAT_SERIALIZATION_FUNCS.get(self.dateformat, None)
+        if format_func:
+            try:
+                return format_func(value, localtime=self.localtime)
+            except (AttributeError, ValueError) as err:
+                self.fail('format', input=value)
+        else:
+            return value.strftime(self.dateformat)
+
+    def _deserialize(self, value, attr, data):
+        if not value:  # Falsy values, e.g. '', None, [] are not valid
+            raise self.fail('invalid')
+        self.dateformat = self.dateformat or self.DEFAULT_FORMAT
+        func = self.DATEFORMAT_DESERIALIZATION_FUNCS.get(self.dateformat)
+        if func:
+            try:
+                return arrow.get(func(value))
+            except (TypeError, AttributeError, ValueError):
+                raise self.fail('invalid')
+        elif self.dateformat:
+            try:
+                return dt.datetime.strptime(value, self.dateformat)
+            except (TypeError, AttributeError, ValueError):
+                raise self.fail('invalid')
+        elif utils.dateutil_available:
+            try:
+                return arrow.get(utils.from_datestring(value))
+            except TypeError:
+                raise self.fail('invalid')
+        else:
+            warnings.warn('It is recommended that you install python-dateutil '
+                          'for improved datetime deserialization.')
+            raise self.fail('invalid')

lemur/common/managers.py

@@ -58,8 +58,8 @@ class InstanceManager(object):
                     results.append(cls())
                 else:
                     results.append(cls)
-            except Exception:
-                current_app.logger.exception('Unable to import %s', cls_path)
+            except Exception as e:
+                current_app.logger.exception('Unable to import %s. Reason: %s', cls_path, e)
                 continue
         self.cache = results
 

lemur/common/missing.py

@@ -0,0 +1,24 @@
+import arrow
+from flask import current_app
+
+from lemur.common.utils import is_weekend
+
+
+def convert_validity_years(data):
+    """
+    Convert validity years to validity_start and validity_end
+
+    :param data:
+    :return:
+    """
+    if data.get('validity_years'):
+        now = arrow.utcnow()
+        data['validity_start'] = now.isoformat()
+
+        end = now.replace(years=+int(data['validity_years']))
+        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+            if is_weekend(end):
+                end = end.replace(days=-2)
+
+        data['validity_end'] = end.isoformat()
+    return data

lemur/common/schema.py

@@ -12,8 +12,8 @@ from flask import request, current_app
 
 from sqlalchemy.orm.collections import InstrumentedList
 
-from marshmallow import Schema, post_dump, pre_load, pre_dump
 from inflection import camelize, underscore
+from marshmallow import Schema, post_dump, pre_load, pre_dump
 
 
 class LemurSchema(Schema):
@@ -136,7 +136,7 @@ def validate_schema(input_schema, output_schema):
                 resp = f(*args, **kwargs)
             except Exception as e:
                 current_app.logger.exception(e)
-                return dict(message=e.message), 500
+                return dict(message=str(e)), 500
 
             if isinstance(resp, tuple):
                 return resp[0], resp[1]

lemur/common/utils.py

@@ -6,15 +6,23 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import six
+import sys
 import string
 import random
-from functools import wraps
 
-from flask import current_app
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
 
-from flask.ext.restful import marshal
 from flask.ext.restful.reqparse import RequestParser
-from flask.ext.sqlalchemy import Pagination
+
+paginated_parser = RequestParser()
+
+paginated_parser.add_argument('count', type=int, default=10, location='args')
+paginated_parser.add_argument('page', type=int, default=1, location='args')
+paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
+paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
+paginated_parser.add_argument('filter', type=str, location='args')
 
 
 def get_psuedo_random_string():
@@ -28,51 +36,22 @@ def get_psuedo_random_string():
     return challenge
 
 
-class marshal_items(object):
-    def __init__(self, fields, envelope=None):
-        self.fields = fields
-        self.envelop = envelope
+def parse_certificate(body):
+    if sys.version_info[0] <= 2:
+        return x509.load_pem_x509_certificate(bytes(body), default_backend())
 
-    def __call__(self, f):
-        def _filter_items(items):
-            filtered_items = []
-            for item in items:
-                filtered_items.append(marshal(item, self.fields))
-            return filtered_items
+    if isinstance(body, six.string_types):
+        body = body.encode('utf-8')
 
-        @wraps(f)
-        def wrapper(*args, **kwargs):
-            try:
-                resp = f(*args, **kwargs)
-
-                # this is a bit weird way to handle non standard error codes returned from the marshaled function
-                if isinstance(resp, tuple):
-                    return resp[0], resp[1]
-
-                if isinstance(resp, Pagination):
-                    return {'items': _filter_items(resp.items), 'total': resp.total}
-
-                if isinstance(resp, list):
-                    return {'items': _filter_items(resp), 'total': len(resp)}
-
-                return marshal(resp, self.fields)
-            except Exception as e:
-                current_app.logger.exception(e)
-                # this is a little weird hack to respect flask restful parsing errors on marshaled functions
-                if hasattr(e, 'code'):
-                    if hasattr(e, 'data'):
-                        return {'message': e.data['message']}, 400
-                    else:
-                        return {'message': {'exception': 'unknown'}}, 400
-                else:
-                    return {'message': {'exception': str(e)}}, 400
-        return wrapper
+    return x509.load_pem_x509_certificate(body, default_backend())
 
 
-paginated_parser = RequestParser()
+def is_weekend(date):
+    """
+    Determines if a given date is on a weekend.
 
-paginated_parser.add_argument('count', type=int, default=10, location='args')
-paginated_parser.add_argument('page', type=int, default=1, location='args')
-paginated_parser.add_argument('sortDir', type=str, dest='sort_dir', location='args')
-paginated_parser.add_argument('sortBy', type=str, dest='sort_by', location='args')
-paginated_parser.add_argument('filter', type=str, location='args')
+    :param date:
+    :return:
+    """
+    if date.weekday() > 5:
+        return True

lemur/common/validators.py

@@ -1,13 +1,14 @@
+import re
 
-import arrow
-from marshmallow.exceptions import ValidationError
-
+from flask import current_app
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
+from marshmallow.exceptions import ValidationError
 
-from lemur.domains import service as domain_service
 from lemur.auth.permissions import SensitiveDomainPermission
+from lemur.common.utils import parse_certificate, is_weekend
+from lemur.domains import service as domain_service
 
 
 def public_certificate(body):
@@ -18,7 +19,7 @@ def public_certificate(body):
     :return:
     """
     try:
-        x509.load_pem_x509_certificate(bytes(body), default_backend())
+        parse_certificate(body)
     except Exception:
         raise ValidationError('Public certificate presented is not valid.')
 
@@ -31,7 +32,10 @@ def private_key(key):
     :return: :raise ValueError:
     """
     try:
-        serialization.load_pem_private_key(bytes(key), None, backend=default_backend())
+        if isinstance(key, bytes):
+            serialization.load_pem_private_key(key, None, backend=default_backend())
+        else:
+            serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend())
     except Exception:
         raise ValidationError('Private key presented is not valid.')
 
@@ -42,25 +46,27 @@ def sensitive_domain(domain):
     :param domain:
     :return:
     """
-    domains = domain_service.get_by_name(domain)
-    for domain in domains:
-        # we only care about non-admins
-        if not SensitiveDomainPermission().can():
-            if domain.sensitive:
-                raise ValidationError(
-                    'Domain {0} has been marked as sensitive, contact and administrator \
-                    to issue the certificate.'.format(domain))
+    restricted_domains = current_app.config.get('LEMUR_RESTRICTED_DOMAINS', [])
+    if restricted_domains:
+        domains = domain_service.get_by_name(domain)
+        for domain in domains:
+            # we only care about non-admins
+            if not SensitiveDomainPermission().can():
+                if domain.sensitive or any([re.match(pattern, domain.name) for pattern in restricted_domains]):
+                    raise ValidationError(
+                        'Domain {0} has been marked as sensitive, contact and administrator \
+                        to issue the certificate.'.format(domain))
 
 
-def oid_type(oid_type):
+def encoding(oid_encoding):
     """
     Determines if the specified oid type is valid.
-    :param oid_type:
+    :param oid_encoding:
     :return:
     """
     valid_types = ['b64asn1', 'string', 'ia5string']
-    if oid_type.lower() not in [o_type.lower() for o_type in valid_types]:
-        raise ValidationError('Invalid Oid Type: {0} choose from {1}'.format(oid_type, ",".join(valid_types)))
+    if oid_encoding.lower() not in [o_type.lower() for o_type in valid_types]:
+        raise ValidationError('Invalid Oid Encoding: {0} choose from {1}'.format(oid_encoding, ",".join(valid_types)))
 
 
 def sub_alt_type(alt_type):
@@ -94,27 +100,19 @@ def dates(data):
     if not data.get('validity_end') and data.get('validity_start'):
         raise ValidationError('If validity end is specified so must validity start.')
 
-    if data.get('validity_end') and data.get('validity_years'):
-        raise ValidationError('Cannot specify both validity end and validity years.')
-
     if data.get('validity_start') and data.get('validity_end'):
+        if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True):
+            if is_weekend(data.get('validity_end')):
+                raise ValidationError('Validity end must not land on a weekend.')
+
         if not data['validity_start'] < data['validity_end']:
             raise ValidationError('Validity start must be before validity end.')
 
         if data.get('authority'):
-            if data.get('validity_start').replace(hour=0, minute=0, second=0, tzinfo=None) < data['authority'].authority_certificate.not_before.replace(hour=0, minute=0, second=0):
+            if data.get('validity_start').date() < data['authority'].authority_certificate.not_before.date():
                 raise ValidationError('Validity start must not be before {0}'.format(data['authority'].authority_certificate.not_before))
 
-            if data.get('validity_end').replace(hour=0, minute=0, second=0, tzinfo=None) > data['authority'].authority_certificate.not_after.replace(hour=0, minute=0, second=0):
+            if data.get('validity_end').date() > data['authority'].authority_certificate.not_after.date():
                 raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
 
-    if data.get('validity_years'):
-        now = arrow.utcnow()
-        end = now.replace(years=+data['validity_years'])
-
-        if data.get('authority'):
-            if now.naive < data['authority'].authority_certificate.not_before:
-                raise ValidationError('Validity start must not be before {0}'.format(data['authority'].authority_certificate.not_before))
-
-            if end.naive > data['authority'].authority_certificate.not_after:
-                raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after))
+    return data

lemur/database.py

@@ -12,8 +12,6 @@
 from sqlalchemy import exc
 from sqlalchemy.sql import and_, or_
 from sqlalchemy.orm import make_transient
-from sqlalchemy.orm.exc import NoResultFound
-
 
 from lemur.extensions import db
 from lemur.exceptions import AttrNotFound, DuplicateError
@@ -125,10 +123,7 @@ def get(model, value, field="id"):
     :return:
     """
     query = session_query(model)
-    try:
-        return query.filter(getattr(model, field) == value).one()
-    except NoResultFound as e:
-        return
+    return query.filter(getattr(model, field) == value).scalar()
 
 
 def get_all(model, value, field="id"):

lemur/default.conf.py

@@ -3,8 +3,6 @@
 import os
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
-ADMINS = frozenset([''])
-
 THREADS_PER_PAGE = 8
 
 # General

lemur/defaults/views.py

@@ -57,7 +57,8 @@ class LemurDefaults(AuthenticatedResource):
             state=current_app.config.get('LEMUR_DEFAULT_STATE'),
             location=current_app.config.get('LEMUR_DEFAULT_LOCATION'),
             organization=current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'),
-            organizationalUnit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT')
+            organizationalUnit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'),
+            issuerPlugin=current_app.config.get('LEMUR_DEFAULT_ISSUER_PLUGIN')
         )
 
 api.add_resource(LemurDefaults, '/defaults', endpoint='default')

lemur/destinations/models.py

@@ -5,7 +5,6 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import copy
 from sqlalchemy import Column, Integer, String, Text
 from sqlalchemy_utils import JSONType
 from lemur.database import db
@@ -23,7 +22,7 @@ class Destination(db.Model):
 
     @property
     def plugin(self):
-        p = plugins.get(self.plugin_name)
-        c = copy.deepcopy(p)
-        c.options = self.options
-        return c
+        return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Destination(label={label})".format(label=self.label)

lemur/domains/models.py

@@ -17,3 +17,6 @@ class Domain(db.Model):
     id = Column(Integer, primary_key=True)
     name = Column(String(256))
     sensitive = Column(Boolean, default=False)
+
+    def __repr__(self):
+        return "Domain(name={name})".format(name=self.name)

lemur/endpoints/models.py

@@ -0,0 +1,86 @@
+"""
+.. module: lemur.endpoints.models
+    :platform: unix
+    :synopsis: This module contains all of the models need to create a authority within Lemur.
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from sqlalchemy.orm import relationship
+from sqlalchemy import Column, Integer, String, func, DateTime, PassiveDefault, Boolean, ForeignKey
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy.sql.expression import case
+
+from lemur.database import db
+
+from lemur.models import policies_ciphers
+
+
+BAD_CIPHERS = [
+    'Protocol-SSLv3',
+    'Protocol-SSLv2',
+    'Protocol-TLSv1'
+]
+
+
+class Cipher(db.Model):
+    __tablename__ = 'ciphers'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(128), nullable=False)
+
+    @hybrid_property
+    def deprecated(self):
+        return self.name in BAD_CIPHERS
+
+    @deprecated.expression
+    def deprecated(cls):
+        return case(
+            [
+                (cls.name in BAD_CIPHERS, True)
+            ],
+            else_=False
+        )
+
+
+class Policy(db.Model):
+    ___tablename__ = 'policies'
+    id = Column(Integer, primary_key=True)
+    name = Column(String(128), nullable=True)
+    ciphers = relationship('Cipher', secondary=policies_ciphers, backref='policy')
+
+
+class Endpoint(db.Model):
+    __tablename__ = 'endpoints'
+    id = Column(Integer, primary_key=True)
+    owner = Column(String(128))
+    name = Column(String(128))
+    dnsname = Column(String(256))
+    type = Column(String(128))
+    active = Column(Boolean, default=True)
+    port = Column(Integer)
+    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
+    policy_id = Column(Integer, ForeignKey('policy.id'))
+    policy = relationship('Policy', backref='endpoint')
+    certificate_id = Column(Integer, ForeignKey('certificates.id'))
+    source_id = Column(Integer, ForeignKey('sources.id'))
+    sensitive = Column(Boolean, default=False)
+    source = relationship('Source', back_populates='endpoints')
+
+    @property
+    def issues(self):
+        issues = []
+
+        for cipher in self.policy.ciphers:
+            if cipher.deprecated:
+                issues.append({'name': 'deprecated cipher', 'value': '{0} has been deprecated consider removing it.'.format(cipher.name)})
+
+        if self.certificate.expired:
+            issues.append({'name': 'expired certificate', 'value': 'There is an expired certificate attached to this endpoint consider replacing it.'})
+
+        if self.certificate.revoked:
+            issues.append({'name': 'revoked', 'value': 'There is a revoked certificate attached to this endpoint consider replacing it.'})
+
+        return issues
+
+    def __repr__(self):
+        return "Endpoint(name={name})".format(name=self.name)

lemur/endpoints/schemas.py

@@ -0,0 +1,43 @@
+"""
+.. module: lemur.endpoints.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+
+from lemur.common.schema import LemurOutputSchema
+from lemur.certificates.schemas import CertificateNestedOutputSchema
+
+
+class CipherNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    deprecated = fields.Boolean()
+    name = fields.String()
+
+
+class PolicyNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    name = fields.String()
+    ciphers = fields.Nested(CipherNestedOutputSchema, many=True)
+
+
+class EndpointOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    dnsname = fields.String()
+    owner = fields.Email()
+    type = fields.String()
+    port = fields.Integer()
+    active = fields.Boolean()
+    certificate = fields.Nested(CertificateNestedOutputSchema)
+    policy = fields.Nested(PolicyNestedOutputSchema)
+
+    issues = fields.List(fields.Dict())
+
+endpoint_output_schema = EndpointOutputSchema()
+endpoints_output_schema = EndpointOutputSchema(many=True)

lemur/endpoints/service.py

@@ -0,0 +1,143 @@
+"""
+.. module: lemur.endpoints.service
+    :platform: Unix
+    :synopsis: This module contains all of the services level functions used to
+    administer endpoints in Lemur
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+
+"""
+from lemur import database
+from lemur.extensions import metrics
+from lemur.endpoints.models import Endpoint, Policy, Cipher
+
+from sqlalchemy import func
+
+
+def get_all():
+    """
+    Get all endpoints that are currently in Lemur.
+
+    :rtype : List
+    :return:
+    """
+    query = database.session_query(Endpoint)
+    return database.find_all(query, Endpoint, {}).all()
+
+
+def get(endpoint_id):
+    """
+    Retrieves an endpoint given it's ID
+
+    :param endpoint_id:
+    :return:
+    """
+    return database.get(Endpoint, endpoint_id)
+
+
+def get_by_dnsname(endpoint_dnsname):
+    """
+    Retrieves an endpoint given it's name.
+
+    :param endpoint_dnsname:
+    :return:
+    """
+    return database.get(Endpoint, endpoint_dnsname, field='dnsname')
+
+
+def get_by_source(source_label):
+    """
+    Retrieves all endpoints for a given source.
+    :param source_label:
+    :return:
+    """
+    return Endpoint.query.filter(Endpoint.source.label == source_label).all()  # noqa
+
+
+def create(**kwargs):
+    """
+    Creates a new endpoint.
+    :param kwargs:
+    :return:
+    """
+    endpoint = Endpoint(**kwargs)
+    database.create(endpoint)
+    metrics.send('endpoint_added', 'counter', 1)
+    return endpoint
+
+
+def get_or_create_policy(**kwargs):
+    policy = database.get(Policy, kwargs['name'], field='name')
+
+    if not policy:
+        policy = Policy(**kwargs)
+        database.create(policy)
+
+    return policy
+
+
+def get_or_create_cipher(**kwargs):
+    cipher = database.get(Cipher, kwargs['name'], field='name')
+
+    if not cipher:
+        cipher = Cipher(**kwargs)
+        database.create(cipher)
+
+    return cipher
+
+
+def update(endpoint_id, **kwargs):
+    endpoint = database.get(Endpoint, endpoint_id)
+
+    endpoint.policy = kwargs['policy']
+    endpoint.certificate = kwargs['certificate']
+    database.update(endpoint)
+    return endpoint
+
+
+def render(args):
+    """
+    Helper that helps us render the REST Api responses.
+    :param args:
+    :return:
+    """
+    query = database.session_query(Endpoint)
+    filt = args.pop('filter')
+
+    if filt:
+        terms = filt.split(';')
+        if 'active' in filt:  # this is really weird but strcmp seems to not work here??
+            query = query.filter(Endpoint.active == terms[1])
+        elif 'port' in filt:
+            if terms[1] != 'null':  # ng-table adds 'null' if a number is removed
+                query = query.filter(Endpoint.port == terms[1])
+        elif 'ciphers' in filt:
+            query = query.filter(
+                Cipher.name == terms[1]
+            )
+        else:
+            query = database.filter(query, Endpoint, terms)
+
+    return database.sort_and_page(query, Endpoint, args)
+
+
+def stats(**kwargs):
+    """
+    Helper that defines some useful statistics about endpoints.
+
+    :param kwargs:
+    :return:
+    """
+    attr = getattr(Endpoint, kwargs.get('metric'))
+    query = database.db.session.query(attr, func.count(attr))
+
+    items = query.group_by(attr).all()
+
+    keys = []
+    values = []
+    for key, count in items:
+        keys.append(key)
+        values.append(count)
+
+    return {'labels': keys, 'values': values}

lemur/endpoints/views.py

@@ -0,0 +1,106 @@
+"""
+.. module: lemur.endpoints.views
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask import Blueprint
+from flask.ext.restful import reqparse, Api
+
+from lemur.common.utils import paginated_parser
+from lemur.common.schema import validate_schema
+from lemur.auth.service import AuthenticatedResource
+
+from lemur.endpoints import service
+from lemur.endpoints.schemas import endpoint_output_schema, endpoints_output_schema
+
+
+mod = Blueprint('endpoints', __name__)
+api = Api(mod)
+
+
+class EndpointsList(AuthenticatedResource):
+    """ Defines the 'endpoints' endpoint """
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(EndpointsList, self).__init__()
+
+    @validate_schema(None, endpoints_output_schema)
+    def get(self):
+        """
+        .. http:get:: /endpoints
+
+           The current list of endpoints
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /endpoints HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+
+           :query sortBy: field to sort on
+           :query sortDir: acs or desc
+           :query page: int default is 1
+           :query filter: key value pair. format is k;v
+           :query limit: limit number default is 10
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+           :note: this will only show certificates that the current user is authorized to use
+        """
+        parser = paginated_parser.copy()
+        args = parser.parse_args()
+        return service.render(args)
+
+
+class Endpoints(AuthenticatedResource):
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(Endpoints, self).__init__()
+
+    @validate_schema(None, endpoint_output_schema)
+    def get(self, endpoint_id):
+        """
+        .. http:get:: /endpoints/1
+
+           One endpoint
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              GET /endpoints/1 HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+        """
+        return service.get(endpoint_id)
+
+
+api.add_resource(EndpointsList, '/endpoints', endpoint='endpoints')
+api.add_resource(Endpoints, '/endpoints/<int:endpoint_id>', endpoint='endpoint')

lemur/exceptions.py

@@ -36,6 +36,14 @@ class IntegrityError(LemurException):
         return repr(self.message)
 
 
+class AssociatedObjectNotFound(LemurException):
+    def __init__(self, message):
+        self.message = message
+
+    def __str__(self):
+        return repr(self.message)
+
+
 class InvalidListener(LemurException):
     def __str__(self):
         return repr("Invalid listener, ensure you select a certificate if you are using a secure protocol")

lemur/factory.py

@@ -14,7 +14,7 @@ import imp
 import errno
 import pkg_resources
 
-from logging import Formatter
+from logging import Formatter, StreamHandler
 from logging.handlers import RotatingFileHandler
 
 from flask import Flask
@@ -90,12 +90,15 @@ def configure_app(app, config=None):
     :param config:
     :return:
     """
+    # respect the config first
+    if config and config != 'None':
+        app.config.from_object(from_file(config))
+
     try:
         app.config.from_envvar("LEMUR_CONF")
     except RuntimeError:
-        if config and config != 'None':
-            app.config.from_object(from_file(config))
-        elif os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
+        # look in default paths
+        if os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
             app.config.from_object(from_file(os.path.expanduser("~/.lemur/lemur.conf.py")))
         else:
             app.config.from_object(from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'default.conf.py')))
@@ -144,14 +147,19 @@ def configure_logging(app):
     app.logger.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
     app.logger.addHandler(handler)
 
+    stream_handler = StreamHandler()
+    stream_handler.setLevel(app.config.get('LOG_LEVEL'))
+    app.logger.addHandler(stream_handler)
+
 
 def install_plugins(app):
     """
     Installs new issuers that are not currently bundled with Lemur.
 
-    :param settings:
+    :param app:
     :return:
     """
+    from lemur.plugins import plugins
     from lemur.plugins.base import register
     # entry_points={
     #    'lemur.plugins': [
@@ -166,3 +174,11 @@ def install_plugins(app):
             app.logger.error("Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc()))
         else:
             register(plugin)
+
+    # ensure that we have some way to notify
+    with app.app_context():
+        try:
+            slug = app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
+            plugins.get(slug)
+        except KeyError:
+            raise Exception("Unable to location notification plugin: {slug}. Ensure that LEMUR_DEFAULT_NOTIFICATION_PLUGIN is set to a valid and installed notification plugin.".format(slug=slug))

lemur/manage.py

@@ -1,28 +1,31 @@
 from __future__ import unicode_literals    # at top of module
 
+from datetime import datetime, timedelta
+from collections import Counter
+
 import os
 import sys
 import base64
 import time
-import arrow
 import requests
 import json
+
+from tabulate import tabulate
 from gunicorn.config import make_settings
 
 from cryptography.fernet import Fernet
 
-from lockfile import LockFile, LockTimeout
-
 from flask import current_app
 from flask.ext.script import Manager, Command, Option, prompt_pass
 from flask.ext.migrate import Migrate, MigrateCommand, stamp
 from flask_script.commands import ShowUrls, Clean, Server
 
 from lemur import database
+from lemur.extensions import metrics
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
 from lemur.certificates import service as cert_service
-from lemur.sources import service as source_service
+from lemur.authorities import service as authority_service
 from lemur.notifications import service as notification_service
 
 from lemur.certificates.service import get_name_from_arn
@@ -30,7 +33,7 @@ from lemur.certificates.verify import verify_string
 
 from lemur.plugins.lemur_aws import elb
 
-from lemur.sources.service import sync as source_sync
+from lemur.sources import service as source_service
 
 from lemur import create_app
 
@@ -62,8 +65,6 @@ CONFIG_TEMPLATE = """
 import os
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
-ADMINS = frozenset([''])
-
 THREADS_PER_PAGE = 8
 
 # General
@@ -188,58 +189,6 @@ def generate_settings():
     return output
 
 
-@manager.option('-s', '--sources', dest='labels')
-def sync(labels):
-    """
-    Attempts to run several methods Certificate discovery. This is
-    run on a periodic basis and updates the Lemur datastore with the
-    information it discovers.
-    """
-    if not labels:
-        sys.stdout.write("Active\tLabel\tDescription\n")
-        for source in source_service.get_all():
-            sys.stdout.write(
-                "{active}\t{label}\t{description}!\n".format(
-                    label=source.label,
-                    description=source.description,
-                    active=source.active
-                )
-            )
-    else:
-        start_time = time.time()
-        lock_file = "/tmp/.lemur_lock"
-        sync_lock = LockFile(lock_file)
-
-        while not sync_lock.i_am_locking():
-            try:
-                sync_lock.acquire(timeout=10)    # wait up to 10 seconds
-
-                sys.stdout.write("[+] Staring to sync sources: {labels}!\n".format(labels=labels))
-                labels = labels.split(",")
-
-                if labels[0] == 'all':
-                    source_sync()
-                else:
-                    source_sync(labels=labels)
-
-                sys.stdout.write(
-                    "[+] Finished syncing sources. Run Time: {time}\n".format(
-                        time=(time.time() - start_time)
-                    )
-                )
-            except LockTimeout:
-                sys.stderr.write(
-                    "[!] Unable to acquire file lock on {file}, is there another sync running?\n".format(
-                        file=lock_file
-                    )
-                )
-                sync_lock.break_lock()
-                sync_lock.acquire()
-                sync_lock.release()
-
-        sync_lock.release()
-
-
 @manager.command
 def notify():
     """
@@ -819,51 +768,169 @@ def publish_verisign_units():
         requests.post('http://localhost:8078/metrics', data=json.dumps(metric))
 
 
-class Rolling(Command):
+@manager.command
+def publish_unapproved_verisign_certificates():
     """
-    Rotates existing certificates to a new one on an ELB
+    Query the Verisign for any certificates that need to be approved.
+    :return:
+    """
+    from lemur.plugins import plugins
+    from lemur.extensions import metrics
+    v = plugins.get('verisign-issuer')
+    certs = v.get_pending_certificates()
+    metrics.send('pending_certificates', 'gauge', certs)
+
+
+class Report(Command):
+    """
+    Defines a set of reports to be run periodically against Lemur.
     """
     option_list = (
-        Option('-w', '--window', dest='window', default=24),
+        Option('-n', '--name', dest='name', default=None, help='Name of the report to run.'),
+        Option('-d', '--duration', dest='duration', default=356, help='Number of days to run the report'),
     )
 
-    def run(self, window):
+    def run(self, name, duration):
+
+        end = datetime.utcnow()
+        start = end - timedelta(days=duration)
+        self.certificates_issued(name, start, end)
+
+    @staticmethod
+    def certificates_issued(name=None, start=None, end=None):
         """
-        Simple function that queries verisign for API units and posts the mertics to
-        Atlas API for other teams to consume.
+        Generates simple report of number of certificates issued by the authority, if no authority
+        is specified report on total number of certificates.
+
+        :param name:
+        :param start:
+        :param end:
         :return:
         """
-        end = arrow.utcnow()
-        start = end.replace(hours=-window)
-        items = Certificate.query.filter(Certificate.not_before <= end.format('YYYY-MM-DD')) \
-            .filter(Certificate.not_before >= start.format('YYYY-MM-DD')).all()
 
-        metrics = {}
-        for i in items:
-            name = "{0},{1}".format(i.owner, i.issuer)
-            if metrics.get(name):
-                metrics[name] += 1
-            else:
-                metrics[name] = 1
+        def _calculate_row(authority):
+            day_cnt = Counter()
+            month_cnt = Counter()
+            year_cnt = Counter()
 
-        for name, value in metrics.iteritems():
-            owner, issuer = name.split(",")
-            metric = [
-                {
-                    "timestamp": 1321351651,
-                    "type": "GAUGE",
-                    "name": "Issued Certificates",
-                    "tags": {"owner": owner, "issuer": issuer, "window": window},
-                    "value": value
-                }
-            ]
+            for cert in authority.certificates:
+                date = cert.date_created.date()
+                day_cnt[date.day] += 1
+                month_cnt[date.month] += 1
+                year_cnt[date.year] += 1
 
-            requests.post('http://localhost:8078/metrics', data=json.dumps(metric))
+            try:
+                day_avg = int(sum(day_cnt.values()) / len(day_cnt.keys()))
+            except ZeroDivisionError:
+                day_avg = 0
+
+            try:
+                month_avg = int(sum(month_cnt.values()) / len(month_cnt.keys()))
+            except ZeroDivisionError:
+                month_avg = 0
+
+            try:
+                year_avg = int(sum(year_cnt.values()) / len(year_cnt.keys()))
+            except ZeroDivisionError:
+                year_avg = 0
+
+            return [authority.name, authority.description, day_avg, month_avg, year_avg]
+
+        rows = []
+        if not name:
+            for authority in authority_service.get_all():
+                rows.append(_calculate_row(authority))
+
+        else:
+            authority = authority_service.get_by_name(name)
+
+            if not authority:
+                sys.stderr.write('[!] Authority {0} was not found.'.format(name))
+                sys.exit(1)
+
+            rows.append(_calculate_row(authority))
+
+        sys.stdout.write(tabulate(rows, headers=["Authority Name", "Description", "Daily Average", "Monthy Average", "Yearly Average"]) + "\n")
+
+
+class Sources(Command):
+    """
+    Defines a set of actions to take against Lemur's sources.
+    """
+    option_list = (
+        Option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.'),
+        Option('-a', '--action', choices=['sync', 'clean'], dest='action', help='Action to take on source.')
+    )
+
+    def run(self, source_strings, action):
+        sources = []
+        if not source_strings:
+            table = []
+            for source in source_service.get_all():
+                table.append([source.label, source.active, source.description])
+
+            sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description']))
+            sys.exit(1)
+
+        elif 'all' in source_strings:
+            sources = source_service.get_all()
+
+        else:
+            for source_str in source_strings:
+                source = source_service.get_by_label(source_str)
+
+                if not source:
+                    sys.stderr.write("Unable to find specified source with label: {0}".format(source_str))
+
+                sources.append(source)
+
+        for source in sources:
+            if action == 'sync':
+                self.sync(source)
+
+            if action == 'clean':
+                self.clean(source)
+
+    @staticmethod
+    def sync(source):
+        start_time = time.time()
+        sys.stdout.write("[+] Staring to sync source: {label}!\n".format(label=source.label))
+
+        user = user_service.get_by_username('lemur')
+
+        try:
+            source_service.sync(source, user)
+            sys.stdout.write(
+                "[+] Finished syncing source: {label}. Run Time: {time}\n".format(
+                    label=source.label,
+                    time=(time.time() - start_time)
+                )
+            )
+        except Exception as e:
+            current_app.logger.exception(e)
+
+            sys.stdout.write(
+                "[X] Failed syncing source {label}!\n".format(label=source.label)
+            )
+
+            metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label})
+
+    @staticmethod
+    def clean(source):
+        start_time = time.time()
+        sys.stdout.write("[+] Staring to clean source: {label}!\n".format(label=source.label))
+        source_service.clean(source)
+        sys.stdout.write(
+            "[+] Finished cleaning source: {label}. Run Time: {time}\n".format(
+                label=source.label,
+                time=(time.time() - start_time)
+            )
+        )
 
 
 def main():
     manager.add_command("start", LemurServer())
-    manager.add_command("runserver", Server(host='127.0.0.1'))
+    manager.add_command("runserver", Server(host='127.0.0.1', threaded=True))
     manager.add_command("clean", Clean())
     manager.add_command("show_urls", ShowUrls())
     manager.add_command("db", MigrateCommand)
@@ -873,7 +940,8 @@ def main():
     manager.add_command("create_role", CreateRole())
     manager.add_command("provision_elb", ProvisionELB())
     manager.add_command("rotate_elbs", RotateELBs())
-    manager.add_command("rolling", Rolling())
+    manager.add_command("sources", Sources())
+    manager.add_command("report", Report())
     manager.run()
 
 if __name__ == "__main__":

lemur/migrations/versions/29d8c8455c86_.py

@@ -0,0 +1,62 @@
+"""Adding endpoint tables
+
+Revision ID: 29d8c8455c86
+Revises: 3307381f3b88
+Create Date: 2016-06-28 16:05:25.720213
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '29d8c8455c86'
+down_revision = '3307381f3b88'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('ciphers',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=128), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('policy',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=128), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('policies_ciphers',
+    sa.Column('cipher_id', sa.Integer(), nullable=True),
+    sa.Column('policy_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['cipher_id'], ['ciphers.id'], ),
+    sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], )
+    )
+    op.create_index('policies_ciphers_ix', 'policies_ciphers', ['cipher_id', 'policy_id'], unique=False)
+    op.create_table('endpoints',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('owner', sa.String(length=128), nullable=True),
+    sa.Column('name', sa.String(length=128), nullable=True),
+    sa.Column('dnsname', sa.String(length=256), nullable=True),
+    sa.Column('type', sa.String(length=128), nullable=True),
+    sa.Column('active', sa.Boolean(), nullable=True),
+    sa.Column('port', sa.Integer(), nullable=True),
+    sa.Column('date_created', sa.DateTime(), server_default=sa.text(u'now()'), nullable=False),
+    sa.Column('policy_id', sa.Integer(), nullable=True),
+    sa.Column('certificate_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ),
+    sa.ForeignKeyConstraint(['policy_id'], ['policy.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('endpoints')
+    op.drop_index('policies_ciphers_ix', table_name='policies_ciphers')
+    op.drop_table('policies_ciphers')
+    op.drop_table('policy')
+    op.drop_table('ciphers')
+    ### end Alembic commands ###

lemur/migrations/versions/412b22cb656a_.py

@@ -43,7 +43,7 @@ def upgrade():
     # migrate existing authority_id relationship to many_to_many
     conn = op.get_bind()
     for id, authority_id in conn.execute(text('select id, authority_id from roles where authority_id is not null')):
-        stmt = text('insert into roles_authorities (role_id, authority_id) values (:role_id, :authority_id)')
+        stmt = text('insert into roles_authoritties (role_id, authority_id) values (:role_id, :authority_id)')
         stmt = stmt.bindparams(role_id=id, authority_id=authority_id)
         op.execute(stmt)
 

lemur/migrations/versions/7f71c0cea31a_.py

@@ -0,0 +1,35 @@
+"""Ensures that certificate name is unique.
+If duplicates are found, we follow the standard naming convention of appending '-X'
+with x being the number of duplicates starting at 1.
+
+Revision ID: 7f71c0cea31a
+Revises: 29d8c8455c86
+Create Date: 2016-07-28 09:39:12.736506
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '7f71c0cea31a'
+down_revision = '29d8c8455c86'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import text
+
+
+def upgrade():
+    conn = op.get_bind()
+    for name in conn.execute(text('select name from certificates group by name having count(*) > 1')):
+        for idx, id in enumerate(conn.execute(text("select id from certificates where certificates.name like :name order by id ASC").bindparams(name=name[0]))):
+            if not idx:
+                continue
+            new_name = name[0] + '-' + str(idx)
+            stmt = text('update certificates set name=:name where id=:id')
+            stmt = stmt.bindparams(name=new_name, id=id[0])
+            op.execute(stmt)
+
+    op.create_unique_constraint(None, 'certificates', ['name'])
+
+
+def downgrade():
+    op.drop_constraint(None, 'certificates', type_='unique')

lemur/migrations/versions/932525b82f1a_.py

@@ -0,0 +1,21 @@
+"""Changing the column name to the more accurately named 'notify'.
+
+Revision ID: 932525b82f1a
+Revises: 7f71c0cea31a
+Create Date: 2016-10-13 20:14:33.928029
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '932525b82f1a'
+down_revision = '7f71c0cea31a'
+
+from alembic import op
+
+
+def upgrade():
+    op.alter_column('certificates', 'active', new_column_name='notify')
+
+
+def downgrade():
+    op.alter_column('certificates', 'notify', new_column_name='active')

lemur/models.py

@@ -63,9 +63,9 @@ roles_authorities = db.Table('roles_authorities',
 Index('roles_authorities_ix', roles_authorities.c.authority_id, roles_authorities.c.role_id)
 
 roles_certificates = db.Table('roles_certificates',
-                             Column('certificate_id', Integer, ForeignKey('certificates.id')),
-                             Column('role_id', Integer, ForeignKey('roles.id'))
-                             )
+                              Column('certificate_id', Integer, ForeignKey('certificates.id')),
+                              Column('role_id', Integer, ForeignKey('roles.id'))
+                              )
 
 Index('roles_certificates_ix', roles_certificates.c.certificate_id, roles_certificates.c.role_id)
 
@@ -76,3 +76,10 @@ roles_users = db.Table('roles_users',
                        )
 
 Index('roles_users_ix', roles_users.c.user_id, roles_users.c.role_id)
+
+
+policies_ciphers = db.Table('policies_ciphers',
+                            Column('cipher_id', Integer, ForeignKey('ciphers.id')),
+                            Column('policy_id', Integer, ForeignKey('policy.id')))
+
+Index('policies_ciphers_ix', policies_ciphers.c.cipher_id, policies_ciphers.c.policy_id)

lemur/notifications/models.py

@@ -33,3 +33,6 @@ class Notification(db.Model):
     @property
     def plugin(self):
         return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Notification(label={label})".format(label=self.label)

lemur/notifications/service.py

@@ -45,6 +45,7 @@ def _get_message_data(cert):
     cert_dict['owner'] = cert.owner
     cert_dict['name'] = cert.name
     cert_dict['body'] = cert.body
+    cert_dict['endpoints'] = [{'name': x.name, 'dnsname': x.dnsname} for x in cert.endpoints]
 
     return cert_dict
 
@@ -159,6 +160,9 @@ def _is_eligible_for_notifications(cert):
     :param cert:
     :return:
     """
+    if not cert.notify:
+        return
+
     now = arrow.utcnow()
     days = (cert.not_after - now.naive).days
 
@@ -231,7 +235,7 @@ def create_default_expiration_notifications(name, recipients):
             inter.extend(options)
             n = create(
                 label="{name}_{interval}_DAY".format(name=name, interval=i),
-                plugin_name="email-notification",
+                plugin_name=current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification"),
                 options=list(inter),
                 description="Default {interval} day expiration notification".format(interval=i),
                 certificates=[]
@@ -243,27 +247,31 @@ def create_default_expiration_notifications(name, recipients):
 
 def create(label, plugin_name, options, description, certificates):
     """
-    Creates a new destination, that can then be used as a destination for certificates.
+    Creates a new notification.
 
-    :param label: Notification common name
+    :param label: Notification label
     :param plugin_name:
     :param options:
     :param description:
+    :param certificates:
     :rtype : Notification
     :return:
     """
     notification = Notification(label=label, options=options, plugin_name=plugin_name, description=description)
-    notification = database.update_list(notification, 'certificates', Certificate, certificates)
+    notification.certificates = certificates
     return database.create(notification)
 
 
 def update(notification_id, label, options, description, active, certificates):
     """
-    Updates an existing destination.
+    Updates an existing notification.
 
-    :param label: Notification common name
+    :param notification_id:
+    :param label: Notification label
     :param options:
     :param description:
+    :param active:
+    :param certificates:
     :rtype : Notification
     :return:
     """

lemur/plugins/bases/destination.py

@@ -11,6 +11,7 @@ from lemur.plugins.base import Plugin
 
 class DestinationPlugin(Plugin):
     type = 'destination'
+    requires_key = True
 
     def upload(self):
         raise NotImplemented

lemur/plugins/bases/notification.py

@@ -45,7 +45,7 @@ class ExpirationNotificationPlugin(NotificationPlugin):
     ]
 
     @property
-    def plugin_options(self):
+    def options(self):
         return list(self.default_options) + self.additional_options
 
     def send(self):

lemur/plugins/bases/source.py

@@ -25,6 +25,12 @@ class SourcePlugin(Plugin):
     def get_certificates(self):
         raise NotImplemented
 
+    def get_endpoints(self):
+        raise NotImplemented
+
+    def clean(self):
+        raise NotImplemented
+
     @property
     def options(self):
         return list(self.default_options) + self.additional_options

lemur/plugins/lemur_acme/__init__.py

@@ -0,0 +1,5 @@
+try:
+    VERSION = __import__('pkg_resources') \
+        .get_distribution(__name__).version
+except Exception as e:
+    VERSION = 'unknown'

lemur/plugins/lemur_acme/plugin.py

@@ -0,0 +1,210 @@
+"""
+.. module: lemur.plugins.lemur_acme.acme
+    :platform: Unix
+    :synopsis: This module is responsible for communicating with a ACME CA.
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+    Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+"""
+from flask import current_app
+
+from acme.client import Client
+from acme import jose
+from acme import messages
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+import OpenSSL.crypto
+
+from lemur.plugins.bases import IssuerPlugin
+from lemur.plugins import lemur_acme as acme
+
+from .route53 import delete_txt_record, create_txt_record, wait_for_change
+
+
+def find_dns_challenge(authz):
+    for combo in authz.body.resolved_combinations:
+        if (
+            len(combo) == 1 and
+            isinstance(combo[0].chall, acme.challenges.DNS01)
+        ):
+            yield combo[0]
+
+
+class AuthorizationRecord(object):
+    def __init__(self, host, authz, dns_challenge, change_id):
+        self.host = host
+        self.authz = authz
+        self.dns_challenge = dns_challenge
+        self.change_id = change_id
+
+
+def start_dns_challenge(acme_client, host):
+    authz = acme_client.request_domain_challenges(
+        host, acme_client.directory.new_authz
+    )
+
+    [dns_challenge] = find_dns_challenge(authz)
+
+    change_id = create_txt_record(
+        dns_challenge.validation_domain_name(host),
+        dns_challenge.validation(acme_client.key),
+
+    )
+    return AuthorizationRecord(
+        host,
+        authz,
+        dns_challenge,
+        change_id,
+    )
+
+
+def complete_dns_challenge(acme_client, authz_record):
+    wait_for_change(authz_record.change_id)
+
+    response = authz_record.dns_challenge.response(acme_client.key)
+
+    verified = response.simple_verify(
+        authz_record.dns_challenge.chall,
+        authz_record.host,
+        acme_client.key.public_key()
+    )
+    if not verified:
+        raise ValueError("Failed verification")
+
+    acme_client.answer_challenge(authz_record.dns_challenge, response)
+
+
+def request_certificate(acme_client, authorizations, csr):
+    cert_response, _ = acme_client.poll_and_request_issuance(
+        jose.util.ComparableX509(
+            OpenSSL.crypto.load_certificate_request(
+                OpenSSL.crypto.FILETYPE_ASN1,
+                csr.public_bytes(serialization.Encoding.DER),
+            )
+        ),
+        authzrs=[authz_record.authz for authz_record in authorizations],
+    )
+    pem_certificate = OpenSSL.crypto.dump_certificate(
+        OpenSSL.crypto.FILETYPE_PEM, cert_response.body
+    )
+    pem_certificate_chain = "\n".join(
+        OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
+        for cert in acme_client.fetch_chain(cert_response)
+    )
+    return pem_certificate, pem_certificate_chain
+
+
+def generate_rsa_private_key():
+    return rsa.generate_private_key(
+        public_exponent=65537, key_size=2048, backend=default_backend()
+    )
+
+
+def setup_acme_client():
+    key = current_app.config.get('ACME_PRIVATE_KEY').strip()
+    acme_email = current_app.config.get('ACME_EMAIL')
+    acme_tel = current_app.config.get('ACME_TEL')
+    acme_directory_url = current_app.config.get('ACME_DIRECTORY_URL'),
+    contact = ('mailto:{}'.format(acme_email), 'tel:{}'.format(acme_tel))
+
+    key = serialization.load_pem_private_key(
+        key, password=None, backend=default_backend()
+    )
+    return acme_client_for_private_key(acme_directory_url, key)
+
+
+def acme_client_for_private_key(acme_directory_url, private_key):
+    return Client(
+        acme_directory_url, key=jose.JWKRSA(key=private_key)
+    )
+
+
+def register(email):
+    private_key = generate_rsa_private_key()
+    acme_client = acme_client_for_private_key(current_app.config('ACME_DIRECTORY_URL'), private_key)
+
+    registration = acme_client.register(
+        messages.NewRegistration.from_data(email=email)
+    )
+    acme_client.agree_to_tos(registration)
+    return private_key
+
+
+def get_domains(options):
+    """
+    Fetches all domains currently requested
+    :param options:
+    :return:
+    """
+    domains = [options['common_name']]
+    for name in options['extensions']['sub_alt_name']['names']:
+        domains.append(name)
+    return domains
+
+
+def get_authorizations(acme_client, domains):
+    authorizations = []
+    try:
+        for domain in domains:
+            authz_record = start_dns_challenge(acme_client, domain)
+            authorizations.append(authz_record)
+
+        for authz_record in authorizations:
+            complete_dns_challenge(acme_client, authz_record)
+    finally:
+        for authz_record in authorizations:
+            dns_challenge = authz_record.dns_challenge
+            delete_txt_record(
+                authz_record.change_id,
+                dns_challenge.validation_domain_name(authz_record.host),
+                dns_challenge.validation(acme_client.key),
+            )
+
+    return authorizations
+
+
+class ACMEIssuerPlugin(IssuerPlugin):
+    title = 'Acme'
+    slug = 'acme-issuer'
+    description = 'Enables the creation of certificates via ACME CAs (including Let\'s Encrypt)'
+    version = acme.VERSION
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur.git'
+
+    def __init__(self, *args, **kwargs):
+        super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
+
+    def create_certificate(self, csr, issuer_options):
+        """
+        Creates a ACME certificate.
+
+        :param csr:
+        :param issuer_options:
+        :return: :raise Exception:
+        """
+        current_app.logger.debug("Requesting a new acme certificate: {0}".format(issuer_options))
+        acme_client = setup_acme_client()
+        domains = get_domains(issuer_options)
+        authorizations = get_authorizations(acme_client, domains)
+        pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, csr)
+        return pem_certificate, pem_certificate_chain
+
+    @staticmethod
+    def create_authority(options):
+        """
+        Creates an authority, this authority is then used by Lemur to allow a user
+        to specify which Certificate Authority they want to sign their certificate.
+
+        :param options:
+        :return:
+        """
+        role = {'username': '', 'password': '', 'name': 'acme'}
+        return current_app.config.get('ACME_ROOT'), "", [role]

lemur/plugins/lemur_acme/route53.py

@@ -0,0 +1,86 @@
+import time
+from lemur.plugins.lemur_aws.sts import sts_client
+
+
+@sts_client('route53')
+def wait_for_r53_change(change_id, client=None):
+    _, change_id = change_id
+
+    while True:
+        response = client.get_change(Id=change_id)
+        if response["ChangeInfo"]["Status"] == "INSYNC":
+            return
+        time.sleep(5)
+
+
+@sts_client('route53')
+def find_zone_id(domain, client=None):
+    paginator = client.get_paginator("list_hosted_zones")
+    zones = []
+    for page in paginator.paginate():
+        for zone in page["HostedZones"]:
+            if domain.endswith(zone["Name"]) or (domain + ".").endswith(zone["Name"]):
+                if not zone["Config"]["PrivateZone"]:
+                    zones.append((zone["Name"], zone["Id"]))
+
+    if not zones:
+        raise ValueError(
+            "Unable to find a Route53 hosted zone for {}".format(domain)
+        )
+
+
+@sts_client('route53')
+def change_txt_record(action, zone_id, domain, value, client=None):
+    response = client.change_resource_record_sets(
+        HostedZoneId=zone_id,
+        ChangeBatch={
+            "Changes": [
+                {
+                    "Action": action,
+                    "ResourceRecordSet": {
+                        "Name": domain,
+                        "Type": "TXT",
+                        "TTL": 300,
+                        "ResourceRecords": [
+                            # For some reason TXT records need to be
+                            # manually quoted.
+                            {"Value": '"{}"'.format(value)}
+                        ],
+                    }
+                }
+            ]
+        }
+    )
+    return response["ChangeInfo"]["Id"]
+
+
+def create_txt_record(host, value):
+    zone_id = find_zone_id(host)
+    change_id = change_txt_record(
+        "CREATE",
+        zone_id,
+        host,
+        value,
+    )
+    return zone_id, change_id
+
+
+def delete_txt_record(change_id, host, value):
+    zone_id, _ = change_id
+    change_txt_record(
+        "DELETE",
+        zone_id,
+        host,
+        value
+    )
+
+
+@sts_client('route53')
+def wait_for_change(change_id, client=None):
+    _, change_id = change_id
+
+    while True:
+        response = client.get_change(Id=change_id)
+        if response["ChangeInfo"]["Status"] == "INSYNC":
+            return
+        time.sleep(5)

lemur/plugins/lemur_acme/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -0,0 +1,4 @@
+
+def test_get_certificates(app):
+    from lemur.plugins.base import plugins
+    p = plugins.get('acme-issuer')

lemur/plugins/lemur_aws/ec2.py

@@ -0,0 +1,23 @@
+"""
+.. module: lemur.plugins.lemur_aws.elb
+    :synopsis: Module contains some often used and helpful classes that
+    are used to deal with ELBs
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from lemur.plugins.lemur_aws.sts import sts_client
+
+
+@sts_client('ec2')
+def get_regions(**kwargs):
+    regions = kwargs['client'].describe_regions()
+    return [x['RegionName'] for x in regions['Regions']]
+
+
+@sts_client('ec2')
+def get_all_instances(**kwargs):
+    """
+    Fetches all instance objects for a given account and region.
+    """
+    paginator = kwargs['client'].get_paginator('describe_instances')
+    return paginator.paginate()

lemur/plugins/lemur_aws/elb.py

@@ -5,12 +5,25 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import boto.ec2
-
+import botocore
 from flask import current_app
 
+from retrying import retry
+
 from lemur.exceptions import InvalidListener
-from lemur.plugins.lemur_aws.sts import assume_service
+from lemur.plugins.lemur_aws.sts import sts_client, assume_service
+
+
+def retry_throttled(exception):
+    """
+    Determiens if this exception is due to throttling
+    :param exception:
+    :return:
+    """
+    if isinstance(exception, botocore.exceptions.ClientError):
+        if 'Throttling' in exception.message:
+            return True
+    return False
 
 
 def is_valid(listener_tuple):
@@ -28,7 +41,6 @@ def is_valid(listener_tuple):
 
     :param listener_tuple:
     """
-    current_app.logger.debug(listener_tuple)
     lb_port, i_port, lb_protocol, arn = listener_tuple
     current_app.logger.debug(lb_protocol)
     if lb_protocol.lower() in ['ssl', 'https']:
@@ -38,41 +50,57 @@ def is_valid(listener_tuple):
     return listener_tuple
 
 
-def get_all_regions():
+@sts_client('elb')
+@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+def get_elbs(**kwargs):
     """
-    Retrieves all current EC2 regions.
+    Fetches one page elb objects for a given account and region.
+    """
+    client = kwargs.pop('client')
+    return client.describe_load_balancers(**kwargs)
 
+
+def get_all_elbs(**kwargs):
+    """
+    Fetches all elbs for a given account/region
+
+    :param kwargs:
     :return:
     """
-    regions = []
-    for r in boto.ec2.regions():
-        regions.append(r.name)
-    return regions
-
-
-def get_all_elbs(account_number, region):
-    """
-    Fetches all elb objects for a given account and region.
-
-    :param account_number:
-    :param region:
-    """
-    marker = None
     elbs = []
-    return assume_service(account_number, 'elb', region).get_all_load_balancers()
-# TODO create pull request for boto to include elb marker support
-#    while True:
-#        app.logger.debug(response.__dict__)
-#        raise Exception
-#        result = response['list_server_certificates_response']['list_server_certificates_result']
-#
-#        for elb in result['server_certificate_metadata_list']:
-#            elbs.append(elb)
-#
-#        if result['is_truncated'] == 'true':
-#            marker = result['marker']
-#        else:
-#            return elbs
+
+    while True:
+        response = get_elbs(**kwargs)
+
+        elbs += response['LoadBalancerDescriptions']
+
+        if not response.get('IsTruncated'):
+            return elbs
+
+        if response['NextMarker']:
+            kwargs.update(dict(marker=response['NextMarker']))
+
+
+@sts_client('elb')
+def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
+    """
+    Fetching all policies currently associated with an ELB.
+
+    :param load_balancer_name:
+    :return:
+    """
+    return kwargs['client'].describe_load_balancer_policies(LoadBalancerName=load_balancer_name, PolicyNames=policy_names)
+
+
+@sts_client('elb')
+def describe_load_balancer_types(policies, **kwargs):
+    """
+    Describe the policies with policy details.
+
+    :param policies:
+    :return:
+    """
+    return kwargs['client'].describe_load_balancer_policy_types(PolicyTypeNames=policies)
 
 
 def attach_certificate(account_number, region, name, port, certificate_id):
@@ -89,67 +117,67 @@ def attach_certificate(account_number, region, name, port, certificate_id):
     return assume_service(account_number, 'elb', region).set_lb_listener_SSL_certificate(name, port, certificate_id)
 
 
-def create_new_listeners(account_number, region, name, listeners=None):
-    """
-    Creates a new listener and attaches it to the ELB.
-
-    :param account_number:
-    :param region:
-    :param name:
-    :param listeners:
-    :return:
-    """
-    listeners = [is_valid(x) for x in listeners]
-    return assume_service(account_number, 'elb', region).create_load_balancer_listeners(name, listeners=listeners)
-
-
-def update_listeners(account_number, region, name, listeners, ports):
-    """
-    We assume that a listener with a specified port already exists. We can then
-    delete the old listener on the port and create a new one in it's place.
-
-    If however we are replacing a listener e.g. changing a port from 80 to 443 we need
-    to make sure we kept track of which ports we needed to delete so that we don't create
-    two listeners (one 80 and one 443)
-
-    :param account_number:
-    :param region:
-    :param name:
-    :param listeners:
-    :param ports:
-    """
-    # you cannot update a listeners port/protocol instead we remove the only one and
-    # create a new one in it's place
-    listeners = [is_valid(x) for x in listeners]
-
-    assume_service(account_number, 'elb', region).delete_load_balancer_listeners(name, ports)
-    return create_new_listeners(account_number, region, name, listeners=listeners)
-
-
-def delete_listeners(account_number, region, name, ports):
-    """
-    Deletes a listener from an ELB.
-
-    :param account_number:
-    :param region:
-    :param name:
-    :param ports:
-    :return:
-    """
-    return assume_service(account_number, 'elb', region).delete_load_balancer_listeners(name, ports)
-
-
-def get_listeners(account_number, region, name):
-    """
-    Gets the listeners configured on an elb and returns a array of tuples
-
-    :param account_number:
-    :param region:
-    :param name:
-    :return: list of tuples
-    """
-
-    conn = assume_service(account_number, 'elb', region)
-    elbs = conn.get_all_load_balancers(load_balancer_names=[name])
-    if elbs:
-        return elbs[0].listeners
+# def create_new_listeners(account_number, region, name, listeners=None):
+#     """
+#     Creates a new listener and attaches it to the ELB.
+#
+#     :param account_number:
+#     :param region:
+#     :param name:
+#     :param listeners:
+#     :return:
+#     """
+#     listeners = [is_valid(x) for x in listeners]
+#     return assume_service(account_number, 'elb', region).create_load_balancer_listeners(name, listeners=listeners)
+#
+#
+# def update_listeners(account_number, region, name, listeners, ports):
+#     """
+#     We assume that a listener with a specified port already exists. We can then
+#     delete the old listener on the port and create a new one in it's place.
+#
+#     If however we are replacing a listener e.g. changing a port from 80 to 443 we need
+#     to make sure we kept track of which ports we needed to delete so that we don't create
+#     two listeners (one 80 and one 443)
+#
+#     :param account_number:
+#     :param region:
+#     :param name:
+#     :param listeners:
+#     :param ports:
+#     """
+#     # you cannot update a listeners port/protocol instead we remove the only one and
+#     # create a new one in it's place
+#     listeners = [is_valid(x) for x in listeners]
+#
+#     assume_service(account_number, 'elb', region).delete_load_balancer_listeners(name, ports)
+#     return create_new_listeners(account_number, region, name, listeners=listeners)
+#
+#
+# def delete_listeners(account_number, region, name, ports):
+#     """
+#     Deletes a listener from an ELB.
+#
+#     :param account_number:
+#     :param region:
+#     :param name:
+#     :param ports:
+#     :return:
+#     """
+#     return assume_service(account_number, 'elb', region).delete_load_balancer_listeners(name, ports)
+#
+#
+# def get_listeners(account_number, region, name):
+#     """
+#     Gets the listeners configured on an elb and returns a array of tuples
+#
+#     :param account_number:
+#     :param region:
+#     :param name:
+#     :return: list of tuples
+#     """
+#
+#     conn = assume_service(account_number, 'elb', region)
+#     elbs = conn.get_all_load_balancers(load_balancer_names=[name])
+#     if elbs:
+#         return elbs[0].listeners

lemur/plugins/lemur_aws/iam.py

@@ -33,15 +33,15 @@ def upload_cert(account_number, name, body, private_key, cert_chain=None):
                                                                     cert_chain=str(cert_chain))
 
 
-def delete_cert(account_number, cert):
+def delete_cert(account_number, cert_name):
     """
     Delete a certificate from AWS
 
     :param account_number:
-    :param cert:
+    :param cert_name:
     :return:
     """
-    return assume_service(account_number, 'iam').delete_server_cert(cert.name)
+    return assume_service(account_number, 'iam').delete_server_cert(cert_name)
 
 
 def get_all_server_certs(account_number):

lemur/plugins/lemur_aws/plugin.py

@@ -4,20 +4,44 @@
     :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 
+    Terraform example to setup the destination bucket:
+    resource "aws_s3_bucket" "certs_log_bucket" {
+         bucket = "certs-log-access-bucket"
+        acl    = "log-delivery-write"
+    }
+
+    resource "aws_s3_bucket" "certs_lemur" {
+        bucket = "certs-lemur"
+        acl    = "private"
+
+      logging {
+        target_bucket = "${aws_s3_bucket.certs_log_bucket.id}"
+        target_prefix = "log/lemur"
+      }
+    }
+
+    The IAM role Lemur is running as should have the following actions on the destination bucket:
+
+    "S3:PutObject",
+    "S3:PutObjectAcl"
+
+    The reader should have the following actions:
+    "s3:GetObject"
+
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+.. moduleauthor:: Harm Weites <harm@weites.com>
 """
+from flask import current_app
 from boto.exception import BotoServerError
+
 from lemur.plugins.bases import DestinationPlugin, SourcePlugin
-from lemur.plugins.lemur_aws import iam, elb
+from lemur.plugins.lemur_aws.ec2 import get_regions
+from lemur.plugins.lemur_aws.elb import get_all_elbs, describe_load_balancer_policies, attach_certificate
+from lemur.plugins.lemur_aws import iam, s3
 from lemur.plugins import lemur_aws as aws
 
 
-def find_value(name, options):
-    for o in options:
-        if o['name'] == name:
-            return o['value']
-
-
 class AWSDestinationPlugin(DestinationPlugin):
     title = 'AWS'
     slug = 'aws-destination'
@@ -36,6 +60,7 @@ class AWSDestinationPlugin(DestinationPlugin):
             'helpMessage': 'Must be a valid AWS account number!',
         }
     ]
+
     # 'elb': {
     #    'name': {'type': 'name'},
     #    'region': {'type': 'str'},
@@ -43,24 +68,22 @@ class AWSDestinationPlugin(DestinationPlugin):
     # }
 
     def upload(self, name, body, private_key, cert_chain, options, **kwargs):
-        if private_key:
-            try:
-                iam.upload_cert(find_value('accountNumber', options), name, body, private_key, cert_chain=cert_chain)
-            except BotoServerError as e:
-                if e.error_code != 'EntityAlreadyExists':
-                    raise Exception(e)
+        try:
+            iam.upload_cert(self.get_option('accountNumber', options), name, body, private_key,
+                            cert_chain=cert_chain)
+        except BotoServerError as e:
+            if e.error_code != 'EntityAlreadyExists':
+                raise Exception(e)
 
-            e = find_value('elb', options)
-            if e:
-                elb.attach_certificate(kwargs['accountNumber'], ['region'], e['name'], e['port'], e['certificateId'])
-        else:
-            raise Exception("Unable to upload to AWS, private key is required")
+        e = self.get_option('elb', options)
+        if e:
+            attach_certificate(kwargs['accountNumber'], ['region'], e['name'], e['port'], e['certificateId'])
 
 
 class AWSSourcePlugin(SourcePlugin):
     title = 'AWS'
     slug = 'aws-source'
-    description = 'Discovers all SSL certificates in an AWS account'
+    description = 'Discovers all SSL certificates and ELB endpoints in an AWS account'
     version = aws.VERSION
 
     author = 'Kevin Glisson'
@@ -74,11 +97,16 @@ class AWSSourcePlugin(SourcePlugin):
             'validation': '/^[0-9]{12,12}$/',
             'helpMessage': 'Must be a valid AWS account number!',
         },
+        {
+            'name': 'regions',
+            'type': 'str',
+            'helpMessage': 'Comma separated list of regions to search in, if no region is specified we look in all regions.'
+        },
     ]
 
     def get_certificates(self, options, **kwargs):
         certs = []
-        arns = iam.get_all_server_certs(find_value('accountNumber', options))
+        arns = iam.get_all_server_certs(self.get_option('accountNumber', options))
         for arn in arns:
             cert_body, cert_chain = iam.get_cert_from_arn(arn)
             cert_name = iam.get_name_from_arn(arn)
@@ -89,3 +117,157 @@ class AWSSourcePlugin(SourcePlugin):
             )
             certs.append(cert)
         return certs
+
+    def get_endpoints(self, options, **kwargs):
+        endpoints = []
+        account_number = self.get_option('accountNumber', options)
+        regions = self.get_option('regions', options)
+
+        if not regions:
+            regions = get_regions(account_number=account_number)
+        else:
+            regions = regions.split(',')
+
+        for region in regions:
+            elbs = get_all_elbs(account_number=account_number, region=region)
+            current_app.logger.info("Describing load balancers in {0}-{1}".format(account_number, region))
+            for elb in elbs:
+                for listener in elb['ListenerDescriptions']:
+                    if not listener['Listener'].get('SSLCertificateId'):
+                        continue
+
+                    if listener['Listener']['SSLCertificateId'] == 'Invalid-Certificate':
+                        continue
+
+                    endpoint = dict(
+                        name=elb['LoadBalancerName'],
+                        dnsname=elb['DNSName'],
+                        type='elb',
+                        port=listener['Listener']['LoadBalancerPort'],
+                        certificate_name=iam.get_name_from_arn(listener['Listener']['SSLCertificateId'])
+                    )
+
+                    if listener['PolicyNames']:
+                        policy = describe_load_balancer_policies(elb['LoadBalancerName'], listener['PolicyNames'], account_number=account_number, region=region)
+                        endpoint['policy'] = format_elb_cipher_policy(policy)
+
+                    endpoints.append(endpoint)
+
+        return endpoints
+
+    def clean(self, options, **kwargs):
+        account_number = self.get_option('accountNumber', options)
+        certificates = self.get_certificates(options)
+        endpoints = self.get_endpoints(options)
+
+        orphaned = []
+        for certificate in certificates:
+            for endpoint in endpoints:
+                if certificate['name'] == endpoint['certificate_name']:
+                    break
+            else:
+                orphaned.append(certificate['name'])
+                iam.delete_cert(account_number, certificate)
+
+        return orphaned
+
+
+def format_elb_cipher_policy(policy):
+    """
+    Attempts to format cipher policy information into a common format.
+    :param policy:
+    :return:
+    """
+    ciphers = []
+    name = None
+    for descr in policy['PolicyDescriptions']:
+        for attr in descr['PolicyAttributeDescriptions']:
+            if attr['AttributeName'] == 'Reference-Security-Policy':
+                name = attr['AttributeValue']
+                continue
+
+            if attr['AttributeValue'] == 'true':
+                ciphers.append(attr['AttributeName'])
+
+    return dict(name=name, ciphers=ciphers)
+
+
+class S3DestinationPlugin(DestinationPlugin):
+    title = 'AWS-S3'
+    slug = 'aws-s3'
+    description = 'Allow the uploading of certificates to Amazon S3'
+
+    author = 'Mikhail Khodorovskiy, Harm Weites <harm@weites.com>'
+    author_url = 'https://github.com/Netflix/lemur'
+
+    options = [
+        {
+            'name': 'bucket',
+            'type': 'str',
+            'required': True,
+            'validation': '/^$|\s+/',
+            'helpMessage': 'Must be a valid S3 bucket name!',
+        },
+        {
+            'name': 'accountNumber',
+            'type': 'str',
+            'required': True,
+            'validation': '/^[0-9]{12,12}$/',
+            'helpMessage': 'A valid AWS account number with permission to access S3',
+        },
+        {
+            'name': 'region',
+            'type': 'str',
+            'default': 'eu-west-1',
+            'required': False,
+            'validation': '/^\w+-\w+-\d+$/',
+            'helpMessage': 'Availability zone to use',
+        },
+        {
+            'name': 'encrypt',
+            'type': 'bool',
+            'required': False,
+            'helpMessage': 'Availability zone to use',
+            'default': True
+        },
+        {
+            'name': 'key',
+            'type': 'str',
+            'required': False,
+            'validation': '/^$|\s+/',
+            'helpMessage': 'Must be a valid S3 object key!',
+        },
+        {
+            'name': 'caKey',
+            'type': 'str',
+            'required': False,
+            'validation': '/^$|\s+/',
+            'helpMessage': 'Must be a valid S3 object key!',
+        },
+        {
+            'name': 'certKey',
+            'type': 'str',
+            'required': False,
+            'validation': '/^$|\s+/',
+            'helpMessage': 'Must be a valid S3 object key!',
+        }
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super(S3DestinationPlugin, self).__init__(*args, **kwargs)
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        account_number = self.get_option('accountNumber', options)
+        encrypt = self.get_option('encrypt', options)
+        bucket = self.get_option('bucket', options)
+        key = self.get_option('key', options)
+        ca_key = self.get_option('caKey', options)
+        cert_key = self.get_option('certKey', options)
+
+        if key and ca_key and cert_key:
+            s3.write_to_s3(account_number, bucket, key, private_key, encrypt=encrypt)
+            s3.write_to_s3(account_number, bucket, ca_key, cert_chain, encrypt=encrypt)
+            s3.write_to_s3(account_number, bucket, cert_key, body, encrypt=encrypt)
+        else:
+            pem_body = key + '\n' + body + '\n' + cert_chain + '\n'
+            s3.write_to_s3(account_number, bucket, name, pem_body, encrypt=encrypt)

lemur/plugins/lemur_aws/s3.py

@@ -0,0 +1,26 @@
+"""
+.. module: lemur.plugins.lemur_aws.s3
+    :platform: Unix
+    :synopsis: Contains helper functions for interactive with AWS S3 Apis.
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from boto.s3.key import Key
+from lemur.plugins.lemur_aws.sts import assume_service
+
+
+def write_to_s3(account_number, bucket_name, key, data, encrypt=True):
+    """
+    Use STS to write to an S3 bucket
+
+    :param account_number:
+    :param bucket_name:
+    :param data:
+    """
+    conn = assume_service(account_number, 's3')
+    b = conn.get_bucket(bucket_name, validate=False)  # validate=False removes need for ListObjects permission
+
+    k = Key(bucket=b, name=key)
+    k.set_contents_from_string(data, encrypt_key=encrypt)
+    k.set_canned_acl("bucket-owner-read")

lemur/plugins/lemur_aws/sts.py

@@ -5,13 +5,16 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+from functools import wraps
+
 import boto
 import boto.ec2.elb
+import boto3
 
 from flask import current_app
 
 
-def assume_service(account_number, service, region=None):
+def assume_service(account_number, service, region='us-east-1'):
     conn = boto.connect_sts()
 
     role = conn.assume_role('arn:aws:iam::{0}:role/{1}'.format(
@@ -35,3 +38,47 @@ def assume_service(account_number, service, region=None):
             aws_access_key_id=role.credentials.access_key,
             aws_secret_access_key=role.credentials.secret_key,
             security_token=role.credentials.session_token)
+
+    elif service in 's3':
+        return boto.s3.connect_to_region(
+            region,
+            aws_access_key_id=role.credentials.access_key,
+            aws_secret_access_key=role.credentials.secret_key,
+            security_token=role.credentials.session_token)
+
+
+def sts_client(service, service_type='client'):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            sts = boto3.client('sts')
+            arn = 'arn:aws:iam::{0}:role/{1}'.format(
+                kwargs.pop('account_number'),
+                current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur')
+            )
+            # TODO add user specific information to RoleSessionName
+            role = sts.assume_role(RoleArn=arn, RoleSessionName='lemur')
+
+            if service_type == 'client':
+                client = boto3.client(
+                    service,
+                    region_name=kwargs.pop('region', 'us-east-1'),
+                    aws_access_key_id=role['Credentials']['AccessKeyId'],
+                    aws_secret_access_key=role['Credentials']['SecretAccessKey'],
+                    aws_session_token=role['Credentials']['SessionToken']
+                )
+                kwargs['client'] = client
+            elif service_type == 'resource':
+                resource = boto3.resource(
+                    service,
+                    region_name=kwargs.pop('region', 'us-east-1'),
+                    aws_access_key_id=role['Credentials']['AccessKeyId'],
+                    aws_secret_access_key=role['Credentials']['SecretAccessKey'],
+                    aws_session_token=role['Credentials']['SessionToken']
+                )
+                kwargs['resource'] = resource
+            return f(*args, **kwargs)
+
+        return decorated_function
+
+    return decorator

lemur/plugins/lemur_aws/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_aws/tests/test_elb.py

@@ -0,0 +1,14 @@
+import boto
+from moto import mock_sts, mock_elb
+
+
+@mock_sts()
+@mock_elb()
+def test_get_all_elbs(app):
+    from lemur.plugins.lemur_aws.elb import get_all_elbs
+    conn = boto.ec2.elb.connect_to_region('us-east-1')
+    elbs = get_all_elbs(account_number='123456789012', region='us-east-1')
+    assert not elbs
+    conn.create_load_balancer('example-lb', ['us-east-1a', 'us-east-1b'], [(443, 5443, 'tcp')])
+    elbs = get_all_elbs(account_number='123456789012', region='us-east-1')
+    assert elbs

lemur/plugins/lemur_aws/tests/test_iam.py

@@ -1,7 +1,6 @@
+import pytest
 from moto import mock_iam, mock_sts
 
-from lemur.certificates.models import Certificate
-
 from lemur.tests.vectors import EXTERNAL_VALID_STR, PRIVATE_KEY_STR
 
 
@@ -11,12 +10,12 @@ def test_get_name_from_arn():
     assert get_name_from_arn(arn) == 'tttt2.netflixtest.net-NetflixInc-20150624-20150625'
 
 
+@pytest.mark.skipif(True, reason="this fails because moto is not currently returning what boto does")
 @mock_sts()
 @mock_iam()
 def test_get_all_server_certs(app):
     from lemur.plugins.lemur_aws.iam import upload_cert, get_all_server_certs
-    cert = Certificate(EXTERNAL_VALID_STR)
-    upload_cert('123456789012', cert, PRIVATE_KEY_STR)
+    upload_cert('123456789012', 'testCert', EXTERNAL_VALID_STR.decode('utf-8'), PRIVATE_KEY_STR.decode('utf-8'))
     certs = get_all_server_certs('123456789012')
     assert len(certs) == 1
 
@@ -25,7 +24,6 @@ def test_get_all_server_certs(app):
 @mock_iam()
 def test_get_cert_from_arn(app):
     from lemur.plugins.lemur_aws.iam import upload_cert, get_cert_from_arn
-    cert = Certificate(EXTERNAL_VALID_STR)
-    upload_cert('123456789012', cert, PRIVATE_KEY_STR)
-    body, chain = get_cert_from_arn('arn:aws:iam::123456789012:server-certificate/tttt2.netflixtest.net-NetflixInc-20150624-20150625')
-    assert body.replace('\n', '') == EXTERNAL_VALID_STR.replace('\n', '')
+    upload_cert('123456789012', 'testCert', EXTERNAL_VALID_STR.decode('utf-8'), PRIVATE_KEY_STR.decode('utf-8'))
+    body, chain = get_cert_from_arn('arn:aws:iam::123456789012:server-certificate/testCert')
+    assert body.replace('\n', '') == EXTERNAL_VALID_STR.decode('utf-8').replace('\n', '')

lemur/plugins/lemur_aws/tests/test_plugin.py

@@ -0,0 +1,6 @@
+
+def test_get_certificates(app):
+    from lemur.plugins.base import plugins
+
+    p = plugins.get('aws-s3')
+    assert p

lemur/plugins/lemur_cfssl/__init__.py

@@ -0,0 +1,5 @@
+try:
+    VERSION = __import__('pkg_resources') \
+        .get_distribution(__name__).version
+except Exception as e:
+    VERSION = 'unknown'

lemur/plugins/lemur_cfssl/plugin.py

@@ -0,0 +1,64 @@
+"""
+.. module: lemur.plugins.lemur_cfssl.plugin
+    :platform: Unix
+    :synopsis: This module is responsible for communicating with the CFSSL private CA.
+    :copyright: (c) 2016 by Thomson Reuters
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Charles Hendrie <chad.hendrie@tr.com>
+"""
+
+import json
+import requests
+
+from flask import current_app
+
+from lemur.plugins.bases import IssuerPlugin
+from lemur.plugins import lemur_cfssl as cfssl
+
+
+class CfsslIssuerPlugin(IssuerPlugin):
+    title = 'CFSSL'
+    slug = 'cfssl-issuer'
+    description = 'Enables the creation of certificates by CFSSL private CA'
+    version = cfssl.VERSION
+
+    author = 'Charles Hendrie'
+    author_url = 'https://github.com/netflix/lemur.git'
+
+    def __init__(self, *args, **kwargs):
+        self.session = requests.Session()
+        super(CfsslIssuerPlugin, self).__init__(*args, **kwargs)
+
+    def create_certificate(self, csr, issuer_options):
+        """
+        Creates a CFSSL certificate.
+
+        :param csr:
+        :param issuer_options:
+        :return:
+        """
+        current_app.logger.info("Requesting a new cfssl certificate with csr: {0}".format(csr))
+
+        url = "{0}{1}".format(current_app.config.get('CFSSL_URL'), '/api/v1/cfssl/sign')
+
+        data = {'certificate_request': csr.decode('utf_8')}
+        data = json.dumps(data)
+
+        response = self.session.post(url, data=data.encode(encoding='utf_8', errors='strict'))
+        response_json = json.loads(response.content.decode('utf_8'))
+        cert = response_json['result']['certificate']
+
+        return cert, current_app.config.get('CFSSL_INTERMEDIATE'),
+
+    @staticmethod
+    def create_authority(options):
+        """
+        Creates an authority, this authority is then used by Lemur to allow a user
+        to specify which Certificate Authority they want to sign their certificate.
+
+        :param options:
+        :return:
+        """
+        role = {'username': '', 'password': '', 'name': 'cfssl'}
+        return current_app.config.get('CFSSL_ROOT'), "", [role]

lemur/plugins/lemur_cfssl/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_cfssl/tests/test_cfssl.py

@@ -0,0 +1,6 @@
+
+def test_get_certificates(app):
+    from lemur.plugins.base import plugins
+
+    p = plugins.get('cfssl-issuer')
+    assert p

lemur/plugins/lemur_cryptography/__init__.py

@@ -0,0 +1,5 @@
+try:
+    VERSION = __import__('pkg_resources') \
+        .get_distribution(__name__).version
+except Exception as e:
+    VERSION = 'unknown'

lemur/plugins/lemur_cryptography/plugin.py

@@ -0,0 +1,133 @@
+"""
+.. module: lemur.plugins.lemur_cryptography.plugin
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+import uuid
+
+from flask import current_app
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+
+from lemur.plugins.bases import IssuerPlugin
+from lemur.plugins import lemur_cryptography as cryptography_issuer
+
+
+def build_root_certificate(options):
+    private_key = rsa.generate_private_key(
+        public_exponent=65537,
+        key_size=2048,
+        backend=default_backend()
+    )
+
+    subject = issuer = x509.Name([
+        x509.NameAttribute(x509.OID_COUNTRY_NAME, options['country']),
+        x509.NameAttribute(x509.OID_STATE_OR_PROVINCE_NAME, options['state']),
+        x509.NameAttribute(x509.OID_LOCALITY_NAME, options['location']),
+        x509.NameAttribute(x509.OID_ORGANIZATION_NAME, options['organization']),
+        x509.NameAttribute(x509.OID_ORGANIZATIONAL_UNIT_NAME, options['organizational_unit']),
+        x509.NameAttribute(x509.OID_COMMON_NAME, options['common_name'])
+    ])
+
+    builder = x509.CertificateBuilder(
+        subject_name=subject,
+        issuer_name=issuer,
+        public_key=private_key.public_key(),
+        not_valid_after=options['validity_end'],
+        not_valid_before=options['validity_start'],
+        serial_number=options['first_serial']
+    )
+
+    builder.add_extension(x509.SubjectAlternativeName([x509.DNSName(options['common_name'])]), critical=False)
+
+    cert = builder.sign(private_key, hashes.SHA256(), default_backend())
+
+    cert_pem = cert.public_bytes(
+        encoding=serialization.Encoding.PEM
+    ).decode('utf-8')
+
+    private_key_pem = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,  # would like to use PKCS8 but AWS ELBs don't like it
+        encryption_algorithm=serialization.NoEncryption()
+    )
+
+    return cert_pem, private_key_pem
+
+
+def issue_certificate(csr, options):
+    csr = x509.load_pem_x509_csr(csr, default_backend())
+
+    builder = x509.CertificateBuilder(
+        issuer_name=x509.Name([
+            x509.NameAttribute(
+                x509.OID_ORGANIZATION_NAME,
+                options['authority'].authority_certificate.issuer
+            )]
+        ),
+        subject_name=csr.subject,
+        public_key=csr.public_key(),
+        not_valid_before=options['validity_start'],
+        not_valid_after=options['validity_end'],
+        extensions=csr.extensions)
+
+    # TODO figure out a better way to increment serial
+    builder = builder.serial_number(int(uuid.uuid4()))
+
+    private_key = serialization.load_pem_private_key(
+        bytes(str(options['authority'].authority_certificate.private_key).encode('utf-8')),
+        password=None,
+        backend=default_backend()
+    )
+
+    cert = builder.sign(private_key, hashes.SHA256(), default_backend())
+
+    return cert.public_bytes(
+        encoding=serialization.Encoding.PEM
+    ).decode('utf-8')
+
+
+class CryptographyIssuerPlugin(IssuerPlugin):
+    title = 'Cryptography'
+    slug = 'cryptography-issuer'
+    description = 'Enables the creation and signing of self-signed certificates'
+    version = cryptography_issuer.VERSION
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur.git'
+
+    def create_certificate(self, csr, options):
+        """
+        Creates a certificate.
+
+        :param csr:
+        :param options:
+        :return: :raise Exception:
+        """
+        current_app.logger.debug("Issuing new cryptography certificate with options: {0}".format(options))
+        cert = issue_certificate(csr, options)
+        return cert, ""
+
+    @staticmethod
+    def create_authority(options):
+        """
+        Creates an authority, this authority is then used by Lemur to allow a user
+        to specify which Certificate Authority they want to sign their certificate.
+
+        :param options:
+        :return:
+        """
+        current_app.logger.debug("Issuing new cryptography authority with options: {0}".format(options))
+        cert, private_key = build_root_certificate(options)
+        roles = [
+            {'username': '', 'password': '', 'name': options['name'] + '_admin'},
+            {'username': '', 'password': '', 'name': options['name'] + '_operator'}
+        ]
+        return cert, private_key, "", roles

lemur/plugins/lemur_cryptography/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_digicert/__init__.py

@@ -0,0 +1,5 @@
+try:
+    VERSION = __import__('pkg_resources') \
+        .get_distribution(__name__).version
+except Exception as e:
+    VERSION = 'unknown'

lemur/plugins/lemur_digicert/plugin.py

@@ -0,0 +1,251 @@
+"""
+.. module: lemur.plugins.lemur_digicert.digicert
+    :platform: Unix
+    :synopsis: This module is responsible for communicating with the DigiCert '
+    Advanced API.
+    :license: Apache, see LICENSE for more details.
+
+    DigiCert CertCentral (v2 API) Documentation
+    https://www.digicert.com/services/v2/documentation
+
+    Original Implementation:
+    Chris Dorros, github.com/opendns/lemur-digicert
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+import json
+import arrow
+import requests
+
+import pem
+from retrying import retry
+
+from flask import current_app
+
+from lemur.extensions import metrics
+from lemur.plugins.bases import IssuerPlugin, SourcePlugin
+
+from lemur.plugins import lemur_digicert as digicert
+
+
+def signature_hash(signing_algorithm):
+    """Converts Lemur's signing algorithm into a format DigiCert understands.
+
+    :param signing_algorithm:
+    :return: str digicert specific algorithm string
+    """
+    if not signing_algorithm:
+        return current_app.config.get('DIGICERT_DEFAULT_SIGNING_ALGORITHM', 'sha256')
+
+    if signing_algorithm == 'sha256WithRSA':
+        return 'sha256'
+
+    elif signing_algorithm == 'sha384WithRSA':
+        return 'sha384'
+
+    elif signing_algorithm == 'sha512WithRSA':
+        return 'sha512'
+
+    raise Exception('Unsupported signing algorithm.')
+
+
+def determine_validity_years(end_date):
+    """Given an end date determine how many years into the future that date is.
+
+    :param end_date:
+    :return: str validity in years
+    """
+    now = arrow.utcnow()
+
+    if end_date < now.replace(years=+1):
+        return 1
+    elif end_date < now.replace(years=+2):
+        return 2
+    elif end_date < now.replace(years=+3):
+        return 3
+
+    raise Exception("DigiCert issued certificates cannot exceed three"
+                    " years in validity")
+
+
+def get_issuance(options):
+    """Get the time range for certificates.
+
+    :param options:
+    :return:
+    """
+    if not options.get('validity_end'):
+        options['validity_end'] = arrow.utcnow().replace(years=current_app.config.get('DIGICERT_DEFAULT_VALIDITY', 1))
+
+    validity_years = determine_validity_years(options['validity_end'])
+    return validity_years
+
+
+def process_options(options, csr):
+    """Set the incoming issuer options to DigiCert fields/options.
+
+    :param options:
+    :param csr:
+    :return: dict or valid DigiCert options
+    """
+    data = {
+        "certificate":
+            {
+                "common_name": options['common_name'],
+                "csr": csr.decode('utf-8'),
+                "signature_hash":
+                    signature_hash(options.get('signing_algorithm')),
+            },
+        "organization":
+            {
+                "id": current_app.config.get("DIGICERT_ORG_ID")
+            },
+    }
+
+    # add SANs if present
+    if options.get('extensions', 'sub_alt_names'):
+        dns_names = []
+        for san in options['extensions']['sub_alt_names']['names']:
+            dns_names.append(san['value'])
+
+        data['certificate']['dns_names'] = dns_names
+
+    validity_years = get_issuance(options)
+    data['custom_expiration_date'] = options['validity_end'].format('YYYY-MM-DD')
+    data['validity_years'] = validity_years
+
+    return data
+
+
+def handle_response(response):
+    """
+    Handle the DigiCert API response and any errors it might have experienced.
+    :param response:
+    :return:
+    """
+    metrics.send('digicert_status_code_{0}'.format(response.status_code), 'counter', 1)
+
+    if response.status_code not in [200, 201, 302, 301]:
+        raise Exception(response.json()['message'])
+
+    return response.json()
+
+
+def verify_configuration():
+    """Verify that needed configuration variables are set before plugin startup."""
+    if not current_app.config.get('DIGICERT_API_KEY'):
+        raise Exception("No Digicert API key found. Ensure that 'DIGICERT_API_KEY' is set in the Lemur conf.")
+
+    if not current_app.config.get('DIGICERT_URL'):
+        raise Exception("No Digicert URL found. Ensure that 'DIGICERT_URL' is set in the Lemur conf.")
+
+    if not current_app.config.get('DIGICERT_ORG_ID'):
+        raise Exception("No Digicert organization ID found. Ensure that 'DIGICERT_ORG_ID' is set in Lemur conf.")
+
+    if not current_app.config.get('DIGICERT_ROOT'):
+        raise Exception("No Digicert root found. Ensure that 'DIGICERT_ROOT' is set in the Lemur conf.")
+
+    if not current_app.config.get('DIGICERT_INTERMEDIATE'):
+        raise Exception("No Digicert intermediate found. Ensure that 'DIGICERT_INTERMEDIATE is set in Lemur conf.")
+
+
+@retry(stop_max_attempt_number=10, wait_fixed=10000)
+def get_certificate_id(session, base_url, order_id):
+    """Retrieve certificate order id from Digicert API."""
+    order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id)
+    response_data = handle_response(session.get(order_url))
+    if response_data['status'] != 'issued':
+        raise Exception("Order not in issued state.")
+
+    return response_data['certificate']['id']
+
+
+class DigiCertSourcePlugin(SourcePlugin):
+    """Wrap the Digicert Certifcate API."""
+    title = 'DigiCert'
+    slug = 'digicert-source'
+    description = "Enables the use of Digicert as a source of existing certificates."
+    version = digicert.VERSION
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur.git'
+
+    def __init__(self, *args, **kwargs):
+        """Initialize source with appropriate details."""
+        verify_configuration()
+
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                'X-DC-DEVKEY': current_app.config.get('DIGICERT_API_KEY'),
+                'Content-Type': 'application/json'
+            }
+        )
+
+        super(DigiCertSourcePlugin, self).__init__(*args, **kwargs)
+
+    def get_certificates(self):
+        pass
+
+
+class DigiCertIssuerPlugin(IssuerPlugin):
+    """Wrap the Digicert Issuer API."""
+
+    title = 'DigiCert'
+    slug = 'digicert-issuer'
+    description = "Enables the creation of certificates by"
+    "the DigiCert REST API."
+    version = digicert.VERSION
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur.git'
+
+    def __init__(self, *args, **kwargs):
+        """Initialize the issuer with the appropriate details."""
+        verify_configuration()
+
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                'X-DC-DEVKEY': current_app.config.get('DIGICERT_API_KEY'),
+                'Content-Type': 'application/json'
+            }
+        )
+
+        super(DigiCertIssuerPlugin, self).__init__(*args, **kwargs)
+
+    def create_certificate(self, csr, issuer_options):
+        """Create a DigiCert certificate.
+
+        :param csr:
+        :param issuer_options:
+        :return: :raise Exception:
+        """
+        base_url = current_app.config.get('DIGICERT_URL')
+
+        # make certificate request
+        determinator_url = "{0}/services/v2/order/certificate/ssl".format(base_url)
+        data = process_options(issuer_options, csr)
+        response = self.session.post(determinator_url, data=json.dumps(data))
+        order_id = response.json()['id']
+
+        certificate_id = get_certificate_id(self.session, base_url, order_id)
+
+        # retrieve certificate
+        certificate_url = "{0}/services/v2/certificate/{1}/download/format/pem_all".format(base_url, certificate_id)
+        end_entity, intermediate, root = pem.parse(self.session.get(certificate_url).content)
+        return str(end_entity), str(intermediate)
+
+    @staticmethod
+    def create_authority(options):
+        """Create an authority.
+
+        Creates an authority, this authority is then used by Lemur to
+        allow a user to specify which Certificate Authority they want
+        to sign their certificate.
+
+        :param options:
+        :return:
+        """
+        role = {'username': '', 'password': '', 'name': 'digicert'}
+        return current_app.config.get('DIGICERT_ROOT'), "", [role]

lemur/plugins/lemur_digicert/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_digicert/tests/test_digicert.py

@@ -0,0 +1,77 @@
+import pytest
+import arrow
+from freezegun import freeze_time
+
+from lemur.tests.vectors import CSR_STR
+
+
+def test_process_options(app):
+    from lemur.plugins.lemur_digicert.plugin import process_options
+
+    names = ['one.example.com', 'two.example.com', 'three.example.com']
+
+    options = {
+        'common_name': 'example.com',
+        'owner': 'bob@example.com',
+        'description': 'test certificate',
+        'extensions': {
+            'sub_alt_names': {
+                'names': [{'name_type': 'DNSName', 'value': x} for x in names]
+            }
+        },
+        'validity_end': arrow.get(2017, 5, 7),
+        'validity_start': arrow.get(2016, 10, 30)
+    }
+
+    data = process_options(options, CSR_STR)
+
+    assert data == {
+        'certificate': {
+            'csr': CSR_STR.decode('utf-8'),
+            'common_name': 'example.com',
+            'dns_names': names,
+            'signature_hash': 'sha256'
+        },
+        'organization': {'id': 111111},
+        'validity_years': 1,
+        'custom_expiration_date': arrow.get(2017, 5, 7).format('YYYY-MM-DD')
+    }
+
+
+def test_issuance():
+    from lemur.plugins.lemur_digicert.plugin import get_issuance
+
+    with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):
+        options = {
+            'validity_end': arrow.get(2018, 5, 7),
+            'validity_start': arrow.get(2016, 10, 30)
+        }
+
+        assert get_issuance(options) == 2
+
+        options = {
+            'validity_end': arrow.get(2017, 5, 7),
+            'validity_start': arrow.get(2016, 10, 30)
+        }
+
+        assert get_issuance(options) == 1
+
+        options = {
+            'validity_end': arrow.get(2020, 5, 7),
+            'validity_start': arrow.get(2016, 10, 30)
+        }
+
+        with pytest.raises(Exception):
+            period = get_issuance(options)
+
+
+def test_signature_hash(app):
+    from lemur.plugins.lemur_digicert.plugin import signature_hash
+
+    assert signature_hash(None) == 'sha256'
+    assert signature_hash('sha256WithRSA') == 'sha256'
+    assert signature_hash('sha384WithRSA') == 'sha384'
+    assert signature_hash('sha512WithRSA') == 'sha512'
+
+    with pytest.raises(Exception):
+        signature_hash('sdfdsf')

lemur/plugins/lemur_email/templates/expiration.html

@@ -23,7 +23,7 @@
                             <table width="100%" border="0" cellspacing="0" cellpadding="0">
                                 <tbody>
                                 <tr>
-                                    <td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272;" line-height:1.5">
+                                    <td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
                                         Lemur
                                     </td>
                                 </tr>
@@ -83,12 +83,15 @@
                                                             <tr valign="middle">
                                                                 <td width="32px"></td>
                                                                 <td width="16px"></td>
-                                                                <td style="line-height:1.2"><span
-                                                                        style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.name }}</span><br><span
-                                                                        style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">{{ message.owner }}
+                                                                <td style="line-height:1.2">
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.name }}</span>
+                                                                    <br>
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
+                                                                        {{ message.endpoints | length }} Endpoints
+                                                                        <br>{{ message.owner }}
                                                                         <br>{{ message.not_after | time }}
                                                                         <a href="https://{{ hostname }}/#/certificates/{{ message.name }}" target="_blank">Details</a>
-                                                                        </span>
+                                                                    </span>
                                                                 </td>
                                                             </tr>
                                                                 {%  if not loop.last %}

lemur/plugins/lemur_email/tests/test_email.py

@@ -1,7 +1,5 @@
 from lemur.plugins.lemur_email.templates.config import env
 
-import os.path
-
 
 def test_render():
     messages = [{
@@ -12,5 +10,3 @@ def test_render():
 
     template = env.get_template('{}.html'.format('expiration'))
     body = template.render(dict(messages=messages, hostname='lemur.test.example.com'))
-    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'email.html'), 'w+') as f:
-        f.write(body.encode('utf8'))

lemur/plugins/lemur_java/plugin.py

@@ -55,6 +55,12 @@ def split_chain(chain):
 
 
 def create_truststore(cert, chain, jks_tmp, alias, passphrase):
+    if isinstance(cert, bytes):
+        cert = cert.decode('utf-8')
+
+    if isinstance(chain, bytes):
+        chain = chain.decode('utf-8')
+
     with mktempfile() as cert_tmp:
         with open(cert_tmp, 'w') as f:
             f.write(cert)
@@ -88,10 +94,18 @@ def create_truststore(cert, chain, jks_tmp, alias, passphrase):
 
 
 def create_keystore(cert, chain, jks_tmp, key, alias, passphrase):
+    if isinstance(cert, bytes):
+        cert = cert.decode('utf-8')
+
+    if isinstance(chain, bytes):
+        chain = chain.decode('utf-8')
+
+    if isinstance(key, bytes):
+        key = key.decode('utf-8')
     # Create PKCS12 keystore from private key and public certificate
     with mktempfile() as cert_tmp:
         with open(cert_tmp, 'w') as f:
-            f.writelines([key + "\n", cert + "\n", chain + "\n"])
+            f.writelines([key.strip() + "\n", cert.strip() + "\n", chain.strip() + "\n"])
 
         with mktempfile() as p12_tmp:
             run_process([
@@ -164,7 +178,7 @@ class JavaTruststoreExportPlugin(ExportPlugin):
         if self.get_option('passphrase', options):
             passphrase = self.get_option('passphrase', options)
         else:
-            passphrase = Fernet.generate_key()
+            passphrase = Fernet.generate_key().decode('utf-8')
 
         with mktemppath() as jks_tmp:
             create_truststore(body, chain, jks_tmp, alias, passphrase)
@@ -214,7 +228,7 @@ class JavaKeystoreExportPlugin(ExportPlugin):
         if self.get_option('passphrase', options):
             passphrase = self.get_option('passphrase', options)
         else:
-            passphrase = Fernet.generate_key()
+            passphrase = Fernet.generate_key().decode('utf-8')
 
         if self.get_option('alias', options):
             alias = self.get_option('alias', options)
@@ -222,9 +236,6 @@ class JavaKeystoreExportPlugin(ExportPlugin):
             alias = "blah"
 
         with mktemppath() as jks_tmp:
-            if not key:
-                raise Exception("Unable to export, no private key found.")
-
             create_keystore(body, chain, jks_tmp, key, alias, passphrase)
 
             with open(jks_tmp, 'rb') as f:

lemur/plugins/lemur_java/tests/test_java.py

@@ -1,63 +1,60 @@
-PRIVATE_KEY_STR = b"""
------BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEAsXn+QZRATxryRmGXI4fdI+0a2oBwuVh8fC/9bcqX6c5eDmgc
-rj6esmc1hpIFxMM3DvkFXX6xISkU6B5fmYDEGZLi7NvcXF3+EoA/SCkP1MFlvqhn
-EvNhb0t1fBLs0i/0gfTS/FHBZY1ekHisd/sUetCDZ7F11RxMwws0Oc8bl7j1TpRc
-awXFAsh/aWwQOwFeyWU7TtZeAE7sMyWXInBg37tKk1wlv+mN+27WijI091+amkVy
-zIV6mA5OHfqbjuqV8uQflN8jE244Qr7shtSk7LpBpWf0M6dC7dXbuUctHFhqcDjy
-3IRUl+NisKRoMtq+a0uehfmpFNSUD7F4gdUtSwIDAQABAoIBAGITsZ+aBuPwVzzv
-x286MMoeyL1BR4oVzU1v09Rtpf/uLGo3vMnKDzc19A12+rseynl6wi1FyysxIb2Y
-s2oID9a2JrOQWLmus66TsuT01CvV6J0xQSzm1MyFXdqANuF84NlEa6hGoeK1+jFK
-jr0LQukP+9484oovxnfu5CCiRHRWNZmeuekuYhI1SJf343Tr6jwvyr6KZpnIy0Yt
-axuuIZdCfY9ZV2vFG89GwwgwVQrhf14Kv5vBMZrNh1lRGsr0Sqlx5cGkPRAy90lg
-HjrRMogrtXr3AR5Pk2qqAYXzZBU2EFhJ3k2njpwOzlSj0r0ZwTmejZ89cco0sW5j
-+eQ6aRECgYEA1tkNW75fgwU52Va5VETCzG8II/pZdqNygnoc3z8EutN+1w8f6Tr+
-PdpKSICW0z7Iq4f5k/4wrA5xw1vy5RBMH0ZP29GwHTvCPiTBboR9vWvxQvZn1jb9
-wvKa0RxE18KcF0YIyTnZMubkA17QTFlvCNyZg0iCqeyFYPyqVE+R4AkCgYEA03h1
-XrqECZDDbG9HLUdGbkZNk4VzTcF6dQ3GAPY8M/H7rw5BbvH0RZLOrzl46DDVzKTg
-B1VOReAHsxBKFdkqeq1A99CLDow6vHTIEG8DwxkA7/2QPkt8MybwdApUyYnQh5/v
-CxwkRt4Mm+EiYfn5iyL8yI+vaQSRToVO/3BND7MCgYAJQSpBJG8qzqPSR9kN1zRo
-5/N60ULfSGUbV7U8rJNAlPGmw+EFA+SFt4xxmRBmIxMzyFSo2k8waiLeXmyVD2Go
-CzhPaLXkXHmegajPYOelrCulTcXlRVMi/Z5LmaMhhCGDIyInwNUpSybROllQoJ2W
-zSHTtODj/usz5U5U+WR4OQKBgHQRosI6t2wUo96peTS18UdnmP7GeZINBuymga5X
-eJW+VLkxpuKBNOTW/lCYx+8Rlte7CyebP9oEa9VxtGgniTRKUeVy9lAm0bpMkt7K
-QBNebvBKiVhX0DS3Q7U9UmpIFUfLlcXQTW0ERYFtYZTLQpeGvZ5LlyiaFDM34jM7
-7WAXAoGANDPJdQLEuimCOAMx/xoecNWeZIP6ieB0hVBrwLNxsaZlkn1KodUMuvla
-VEowbtPRdc9o3VZRh4q9cEakssTvOD70hgUZCFcMarmc37RgRvvD2fsZmDZF6qd3
-QfHplREs9F0sW+eiirczG7up4XL+CA162TtZxW+2GAiQhwhE5jA=
------END RSA PRIVATE KEY-----
-"""
+import pytest
+import six
 
-EXTERNAL_VALID_STR = b"""
------BEGIN CERTIFICATE-----
-MIID2zCCAsOgAwIBAgICA+0wDQYJKoZIhvcNAQELBQAwgZcxCzAJBgNVBAYTAlVT
-MRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlMb3MgR2F0b3MxDTALBgNV
-BAMMBHRlc3QxFjAUBgNVBAoMDU5ldGZsaXgsIEluYy4xEzARBgNVBAsMCk9wZXJh
-dGlvbnMxIzAhBgkqhkiG9w0BCQEWFGtnbGlzc29uQG5ldGZsaXguY29tMB4XDTE1
-MTEyMzIxNDIxMFoXDTE1MTEyNjIxNDIxMFowcjENMAsGA1UEAwwEdGVzdDEWMBQG
-A1UECgwNTmV0ZmxpeCwgSW5jLjETMBEGA1UECwwKT3BlcmF0aW9uczELMAkGA1UE
-BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALF5/kGUQE8a8kZhlyOH3SPt
-GtqAcLlYfHwv/W3Kl+nOXg5oHK4+nrJnNYaSBcTDNw75BV1+sSEpFOgeX5mAxBmS
-4uzb3Fxd/hKAP0gpD9TBZb6oZxLzYW9LdXwS7NIv9IH00vxRwWWNXpB4rHf7FHrQ
-g2exddUcTMMLNDnPG5e49U6UXGsFxQLIf2lsEDsBXsllO07WXgBO7DMllyJwYN+7
-SpNcJb/pjftu1ooyNPdfmppFcsyFepgOTh36m47qlfLkH5TfIxNuOEK+7IbUpOy6
-QaVn9DOnQu3V27lHLRxYanA48tyEVJfjYrCkaDLavmtLnoX5qRTUlA+xeIHVLUsC
-AwEAAaNVMFMwUQYDVR0fBEowSDBGoESgQoZAaHR0cDovL3Rlc3QuY2xvdWRjYS5j
-cmwubmV0ZmxpeC5jb20vdGVzdERlY3JpcHRpb25DQVJvb3QvY3JsLnBlbTANBgkq
-hkiG9w0BAQsFAAOCAQEAiHREBKg7zhlQ/N7hDIkxgodRSWD7CVbJGSCdkR3Pvr6+
-jHBVNTJUrYqy7sL2pIutoeiSTQEH65/Gbm30mOnNu+lvFKxTxzof6kNYv8cyc8sX
-eBuBfSrlTodPFSHXQIpOexZgA0f30LOuXegqzxgXkKg+uMXOez5Zo5pNjTUow0He
-oe+V1hfYYvL1rocCmBOkhIGWz7622FxKDawRtZTGVsGsMwMIWyvS3+KQ04K8yHhp
-bQOg9zZAoYQuHY1inKBnA0II8eW0hPpJrlZoSqN8Tp0NSBpFiUk3m7KNFP2kITIf
-tTneAgyUsgfDxNDifZryZSzg7MH31sTBcYaotSmTXw==
------END CERTIFICATE-----
-"""
+from lemur.tests.vectors import INTERNAL_CERTIFICATE_A_STR, INTERNAL_PRIVATE_KEY_A_STR
 
 
-def test_export_certificate_to_jks(app):
+def test_export_truststore(app):
     from lemur.plugins.base import plugins
-    p = plugins.get('java-export')
-    options = {'passphrase': 'test1234'}
-    raw = p.export(EXTERNAL_VALID_STR, "", PRIVATE_KEY_STR, options)
-    assert raw != b""
+
+    p = plugins.get('java-truststore-jks')
+    options = [{'name': 'passphrase', 'value': 'test1234'}]
+    actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
+
+    assert actual[0] == 'jks'
+    assert actual[1] == 'test1234'
+    assert isinstance(actual[2], bytes)
+
+
+def test_export_truststore_default_password(app):
+    from lemur.plugins.base import plugins
+
+    p = plugins.get('java-truststore-jks')
+    options = []
+    actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
+
+    assert actual[0] == 'jks'
+    assert isinstance(actual[1], six.string_types)
+    assert isinstance(actual[2], bytes)
+
+
+def test_export_keystore(app):
+    from lemur.plugins.base import plugins
+
+    p = plugins.get('java-keystore-jks')
+    options = [{'name': 'passphrase', 'value': 'test1234'}]
+
+    with pytest.raises(Exception):
+        p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
+
+    actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
+
+    assert actual[0] == 'jks'
+    assert actual[1] == 'test1234'
+    assert isinstance(actual[2], bytes)
+
+
+def test_export_keystore_default_password(app):
+    from lemur.plugins.base import plugins
+
+    p = plugins.get('java-keystore-jks')
+    options = []
+
+    with pytest.raises(Exception):
+        p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
+
+    actual = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
+
+    assert actual[0] == 'jks'
+    assert isinstance(actual[1], six.string_types)
+    assert isinstance(actual[2], bytes)

lemur/plugins/lemur_kubernetes/__init__.py

@@ -0,0 +1,5 @@
+try:
+    VERSION = __import__('pkg_resources') \
+        .get_distribution(__name__).version
+except Exception as e:
+    VERSION = 'unknown'

lemur/plugins/lemur_kubernetes/plugin.py

@@ -0,0 +1,155 @@
+"""
+.. module: lemur.plugins.lemur_kubernetes.aws
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+
+    The plugin inserts certificates and the private key as Kubernetes secret that
+     can later be used to secure service endpoints running in Kubernetes pods
+
+.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+"""
+import base64
+import urllib
+import requests
+import itertools
+
+from lemur.certificates.models import Certificate
+from lemur.plugins.bases import DestinationPlugin
+
+DEFAULT_API_VERSION = 'v1'
+
+
+def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data):
+
+    # _resolve_uri(k8s_base_uri, namespace, kind, name, api_ver=DEFAULT_API_VERSION)
+    url = _resolve_uri(k8s_base_uri, namespace, kind)
+
+    create_resp = k8s_api.post(url, json=data)
+
+    if 200 <= create_resp.status_code <= 299:
+        return None
+    elif create_resp.json()['reason'] != 'AlreadyExists':
+        return create_resp.content
+    update_resp = k8s_api.put(_resolve_uri(k8s_base_uri, namespace, kind, name), json=data)
+    if not 200 <= update_resp.status_code <= 299:
+        return update_resp.content
+    return None
+
+
+def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,):
+    api_group = 'api'
+    if '/' in api_ver:
+        api_group = 'apis'
+    return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + ('/' + namespace if namespace else '')
+
+
+def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_VERSION):
+    if not namespace:
+        namespace = 'default'
+    return "/".join(itertools.chain.from_iterable([
+        (_resolve_ns(k8s_base_uri, namespace, api_ver=api_ver),),
+        ((kind + 's').lower(),),
+        (name,) if name else (),
+    ]))
+
+
+class KubernetesDestinationPlugin(DestinationPlugin):
+    title = 'Kubernetes'
+    slug = 'kubernetes-destination'
+    description = 'Allow the uploading of certificates to Kubernetes as secret'
+
+    author = 'Mikhail Khodorovskiy'
+    author_url = 'https://github.com/mik373/lemur'
+
+    options = [
+        {
+            'name': 'kubernetesURL',
+            'type': 'str',
+            'required': True,
+            'validation': '@(https?|http)://(-\.)?([^\s/?\.#-]+\.?)+(/[^\s]*)?$@iS',
+            'helpMessage': 'Must be a valid Kubernetes server URL!',
+        },
+        {
+            'name': 'kubernetesAuthToken',
+            'type': 'str',
+            'required': True,
+            'validation': '/^$|\s+/',
+            'helpMessage': 'Must be a valid Kubernetes server Token!',
+        },
+        {
+            'name': 'kubernetesServerCertificate',
+            'type': 'str',
+            'required': True,
+            'validation': '/^$|\s+/',
+            'helpMessage': 'Must be a valid Kubernetes server Certificate!',
+        },
+        {
+            'name': 'kubernetesNamespace',
+            'type': 'str',
+            'required': True,
+            'validation': '/^$|\s+/',
+            'helpMessage': 'Must be a valid Kubernetes Namespace!',
+        },
+
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super(KubernetesDestinationPlugin, self).__init__(*args, **kwargs)
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+
+        k8_bearer = self.get_option('kubernetesAuthToken', options)
+        k8_cert = self.get_option('kubernetesServerCertificate', options)
+        k8_namespace = self.get_option('kubernetesNamespace', options)
+        k8_base_uri = self.get_option('kubernetesURL', options)
+
+        k8s_api = K8sSession(k8_bearer, k8_cert)
+
+        cert = Certificate(body=body)
+
+        # in the future once runtime properties can be passed-in - use passed-in secret name
+        secret_name = 'certs-' + urllib.quote_plus(cert.name)
+
+        err = ensure_resource(k8s_api, k8s_base_uri=k8_base_uri, namespace=k8_namespace, kind="secret", name=secret_name, data={
+            'apiVersion': 'v1',
+            'kind': 'Secret',
+            'metadata': {
+                'name': secret_name,
+            },
+            'data': {
+                'combined.pem': base64.b64encode(body + private_key),
+                'ca.crt': base64.b64encode(cert_chain),
+                'service.key': base64.b64encode(private_key),
+                'service.crt': base64.b64encode(body),
+            }
+        })
+
+        if err is not None:
+            raise Exception("Error uploading secret: " + err)
+
+
+class K8sSession(requests.Session):
+
+    def __init__(self, bearer, cert):
+        super(K8sSession, self).__init__()
+
+        self.headers.update({
+            'Authorization': 'Bearer %s' % bearer
+        })
+
+        k8_ca = '/tmp/k8.cert'
+
+        with open(k8_ca, "w") as text_file:
+            text_file.write(cert)
+
+        self.verify = k8_ca
+
+    def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=30, allow_redirects=True, proxies=None,
+                hooks=None, stream=None, verify=None, cert=None, json=None):
+        """
+        This method overrides the default timeout to be 10s.
+        """
+        return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream,
+                                               verify, cert, json)

lemur/plugins/lemur_openssl/plugin.py

@@ -6,6 +6,7 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+from io import open
 import subprocess
 
 from flask import current_app
@@ -43,6 +44,15 @@ def create_pkcs12(cert, chain, p12_tmp, key, alias, passphrase):
     :param alias:
     :param passphrase:
     """
+    if isinstance(cert, bytes):
+        cert = cert.decode('utf-8')
+
+    if isinstance(chain, bytes):
+        chain = chain.decode('utf-8')
+
+    if isinstance(key, bytes):
+        key = key.decode('utf-8')
+
     with mktempfile() as key_tmp:
         with open(key_tmp, 'w') as f:
             f.write(key)
@@ -50,7 +60,10 @@ def create_pkcs12(cert, chain, p12_tmp, key, alias, passphrase):
         # Create PKCS12 keystore from private key and public certificate
         with mktempfile() as cert_tmp:
             with open(cert_tmp, 'w') as f:
-                f.writelines([cert + "\n", chain + "\n"])
+                if chain:
+                    f.writelines([cert.strip() + "\n", chain.strip() + "\n"])
+                else:
+                    f.writelines([cert.strip() + "\n"])
 
             run_process([
                 "openssl",
@@ -120,6 +133,9 @@ class OpenSSLExportPlugin(ExportPlugin):
 
         with mktemppath() as output_tmp:
             if type == 'PKCS12 (.p12)':
+                if not key:
+                    raise Exception("Private Key required by {0}".format(type))
+
                 create_pkcs12(body, chain, output_tmp, key, alias, passphrase)
                 extension = "p12"
             else:

lemur/plugins/lemur_openssl/tests/test_openssl.py

@@ -1,63 +1,13 @@
-PRIVATE_KEY_STR = b"""
------BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEAsXn+QZRATxryRmGXI4fdI+0a2oBwuVh8fC/9bcqX6c5eDmgc
-rj6esmc1hpIFxMM3DvkFXX6xISkU6B5fmYDEGZLi7NvcXF3+EoA/SCkP1MFlvqhn
-EvNhb0t1fBLs0i/0gfTS/FHBZY1ekHisd/sUetCDZ7F11RxMwws0Oc8bl7j1TpRc
-awXFAsh/aWwQOwFeyWU7TtZeAE7sMyWXInBg37tKk1wlv+mN+27WijI091+amkVy
-zIV6mA5OHfqbjuqV8uQflN8jE244Qr7shtSk7LpBpWf0M6dC7dXbuUctHFhqcDjy
-3IRUl+NisKRoMtq+a0uehfmpFNSUD7F4gdUtSwIDAQABAoIBAGITsZ+aBuPwVzzv
-x286MMoeyL1BR4oVzU1v09Rtpf/uLGo3vMnKDzc19A12+rseynl6wi1FyysxIb2Y
-s2oID9a2JrOQWLmus66TsuT01CvV6J0xQSzm1MyFXdqANuF84NlEa6hGoeK1+jFK
-jr0LQukP+9484oovxnfu5CCiRHRWNZmeuekuYhI1SJf343Tr6jwvyr6KZpnIy0Yt
-axuuIZdCfY9ZV2vFG89GwwgwVQrhf14Kv5vBMZrNh1lRGsr0Sqlx5cGkPRAy90lg
-HjrRMogrtXr3AR5Pk2qqAYXzZBU2EFhJ3k2njpwOzlSj0r0ZwTmejZ89cco0sW5j
-+eQ6aRECgYEA1tkNW75fgwU52Va5VETCzG8II/pZdqNygnoc3z8EutN+1w8f6Tr+
-PdpKSICW0z7Iq4f5k/4wrA5xw1vy5RBMH0ZP29GwHTvCPiTBboR9vWvxQvZn1jb9
-wvKa0RxE18KcF0YIyTnZMubkA17QTFlvCNyZg0iCqeyFYPyqVE+R4AkCgYEA03h1
-XrqECZDDbG9HLUdGbkZNk4VzTcF6dQ3GAPY8M/H7rw5BbvH0RZLOrzl46DDVzKTg
-B1VOReAHsxBKFdkqeq1A99CLDow6vHTIEG8DwxkA7/2QPkt8MybwdApUyYnQh5/v
-CxwkRt4Mm+EiYfn5iyL8yI+vaQSRToVO/3BND7MCgYAJQSpBJG8qzqPSR9kN1zRo
-5/N60ULfSGUbV7U8rJNAlPGmw+EFA+SFt4xxmRBmIxMzyFSo2k8waiLeXmyVD2Go
-CzhPaLXkXHmegajPYOelrCulTcXlRVMi/Z5LmaMhhCGDIyInwNUpSybROllQoJ2W
-zSHTtODj/usz5U5U+WR4OQKBgHQRosI6t2wUo96peTS18UdnmP7GeZINBuymga5X
-eJW+VLkxpuKBNOTW/lCYx+8Rlte7CyebP9oEa9VxtGgniTRKUeVy9lAm0bpMkt7K
-QBNebvBKiVhX0DS3Q7U9UmpIFUfLlcXQTW0ERYFtYZTLQpeGvZ5LlyiaFDM34jM7
-7WAXAoGANDPJdQLEuimCOAMx/xoecNWeZIP6ieB0hVBrwLNxsaZlkn1KodUMuvla
-VEowbtPRdc9o3VZRh4q9cEakssTvOD70hgUZCFcMarmc37RgRvvD2fsZmDZF6qd3
-QfHplREs9F0sW+eiirczG7up4XL+CA162TtZxW+2GAiQhwhE5jA=
------END RSA PRIVATE KEY-----
-"""
-
-EXTERNAL_VALID_STR = b"""
------BEGIN CERTIFICATE-----
-MIID2zCCAsOgAwIBAgICA+0wDQYJKoZIhvcNAQELBQAwgZcxCzAJBgNVBAYTAlVT
-MRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlMb3MgR2F0b3MxDTALBgNV
-BAMMBHRlc3QxFjAUBgNVBAoMDU5ldGZsaXgsIEluYy4xEzARBgNVBAsMCk9wZXJh
-dGlvbnMxIzAhBgkqhkiG9w0BCQEWFGtnbGlzc29uQG5ldGZsaXguY29tMB4XDTE1
-MTEyMzIxNDIxMFoXDTE1MTEyNjIxNDIxMFowcjENMAsGA1UEAwwEdGVzdDEWMBQG
-A1UECgwNTmV0ZmxpeCwgSW5jLjETMBEGA1UECwwKT3BlcmF0aW9uczELMAkGA1UE
-BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczCC
-ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALF5/kGUQE8a8kZhlyOH3SPt
-GtqAcLlYfHwv/W3Kl+nOXg5oHK4+nrJnNYaSBcTDNw75BV1+sSEpFOgeX5mAxBmS
-4uzb3Fxd/hKAP0gpD9TBZb6oZxLzYW9LdXwS7NIv9IH00vxRwWWNXpB4rHf7FHrQ
-g2exddUcTMMLNDnPG5e49U6UXGsFxQLIf2lsEDsBXsllO07WXgBO7DMllyJwYN+7
-SpNcJb/pjftu1ooyNPdfmppFcsyFepgOTh36m47qlfLkH5TfIxNuOEK+7IbUpOy6
-QaVn9DOnQu3V27lHLRxYanA48tyEVJfjYrCkaDLavmtLnoX5qRTUlA+xeIHVLUsC
-AwEAAaNVMFMwUQYDVR0fBEowSDBGoESgQoZAaHR0cDovL3Rlc3QuY2xvdWRjYS5j
-cmwubmV0ZmxpeC5jb20vdGVzdERlY3JpcHRpb25DQVJvb3QvY3JsLnBlbTANBgkq
-hkiG9w0BAQsFAAOCAQEAiHREBKg7zhlQ/N7hDIkxgodRSWD7CVbJGSCdkR3Pvr6+
-jHBVNTJUrYqy7sL2pIutoeiSTQEH65/Gbm30mOnNu+lvFKxTxzof6kNYv8cyc8sX
-eBuBfSrlTodPFSHXQIpOexZgA0f30LOuXegqzxgXkKg+uMXOez5Zo5pNjTUow0He
-oe+V1hfYYvL1rocCmBOkhIGWz7622FxKDawRtZTGVsGsMwMIWyvS3+KQ04K8yHhp
-bQOg9zZAoYQuHY1inKBnA0II8eW0hPpJrlZoSqN8Tp0NSBpFiUk3m7KNFP2kITIf
-tTneAgyUsgfDxNDifZryZSzg7MH31sTBcYaotSmTXw==
------END CERTIFICATE-----
-"""
+import pytest
+from lemur.tests.vectors import INTERNAL_PRIVATE_KEY_A_STR, INTERNAL_CERTIFICATE_A_STR
 
 
-def test_export_certificate_to_jks(app):
+def test_export_certificate_to_pkcs12(app):
     from lemur.plugins.base import plugins
-    p = plugins.get('java-export')
-    options = {'passphrase': 'test1234'}
-    raw = p.export(EXTERNAL_VALID_STR, "", PRIVATE_KEY_STR, options)
+    p = plugins.get('openssl-export')
+    options = [{'name': 'passphrase', 'value': 'test1234'}, {'name': 'type', 'value': 'PKCS12 (.p12)'}]
+    with pytest.raises(Exception):
+        p.export(INTERNAL_CERTIFICATE_A_STR, "", "", options)
+
+    raw = p.export(INTERNAL_CERTIFICATE_A_STR, "", INTERNAL_PRIVATE_KEY_A_STR, options)
     assert raw != b""

lemur/plugins/lemur_slack/plugin.py

@@ -5,7 +5,10 @@
     :license: Apache, see LICENSE for more details.
 
 .. moduleauthor:: Harm Weites <harm@weites.com>
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import json
+import arrow
 from flask import current_app
 from lemur.plugins.bases import ExpirationNotificationPlugin
 from lemur.plugins import lemur_slack as slack
@@ -13,10 +16,42 @@ from lemur.plugins import lemur_slack as slack
 import requests
 
 
-def find_value(name, options):
-    for o in options:
-        if o['name'] == name:
-            return o['value']
+def create_certificate_url(name):
+    return 'https://{hostname}/#/certificates/{name}'.format(
+        hostname=current_app.config.get('LEMUR_HOSTNAME'),
+        name=name
+    )
+
+
+def create_expiration_attachments(messages):
+    attachments = []
+    for message in messages:
+        attachments.append({
+            'title': message['name'],
+            'title_link': create_certificate_url(message['name']),
+            'color': 'danger',
+            'fallback': '',
+            'fields': [
+                {
+                    'title': 'Owner',
+                    'value': message['owner'],
+                    'short': True
+                },
+                {
+                    'title': 'Expires',
+                    'value': arrow.get(message['not_after']).format('dddd, MMMM D, YYYY'),
+                    'short': True
+                },
+                {
+                    'title': 'Endpoints Detected',
+                    'value': len(message['endpoints']),
+                    'short': True
+                }
+            ],
+            'text': '',
+            'mrkdwn_in': ['text']
+        })
+    return attachments
 
 
 class SlackNotificationPlugin(ExpirationNotificationPlugin):
@@ -38,9 +73,9 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
         }, {
             'name': 'username',
             'type': 'str',
-            'required': True,
             'validation': '^.+$',
             'helpMessage': 'The great storyteller',
+            'default': 'Lemur'
         }, {
             'name': 'recipients',
             'type': 'str',
@@ -50,19 +85,25 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
         },
     ]
 
-    @staticmethod
-    def send(event_type, message, targets, options, **kwargs):
+    def send(self, event_type, message, targets, options, **kwargs):
         """
         A typical check can be performed using the notify command:
         `lemur notify`
         """
-        msg = 'Certificate expiry pending for certificate:\n*%s*\nCurrent state is: _%s_' % (message[0]['name'], event_type)
-        body = '{"text": "%s", "channel": "%s", "username": "%s"}' % (msg, find_value('recipients', options), find_value('username', options))
+        if event_type == 'expiration':
+            attachments = create_expiration_attachments(message)
 
-        current_app.logger.info("Sending message to Slack: %s" % body)
-        current_app.logger.debug("Sending data to Slack endpoint at %s" % find_value('webhook', options))
+        if not attachments:
+            raise Exception('Unable to create message attachments')
 
-        r = requests.post(find_value('webhook', options), body)
+        body = {
+            'text': 'Lemur Expiration Notification',
+            'attachments': attachments,
+            'channel': self.get_option('recipients', options),
+            'username': self.get_option('username', options)
+        }
+
+        r = requests.post(self.get_option('webhook', options), json.dumps(body))
         if r.status_code not in [200]:
-            current_app.logger.error("Slack response: %s" % r.status_code)
-            raise
+            raise Exception('Failed to send message')
+        current_app.logger.error("Slack response: {0} Message Body: {1}".format(r.status_code, body))

lemur/plugins/lemur_slack/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_slack/tests/test_slack.py

@@ -0,0 +1,33 @@
+
+
+def test_formatting(certificate):
+    from lemur.plugins.lemur_slack.plugin import create_expiration_attachments
+    from lemur.notifications.service import _get_message_data
+    data = [_get_message_data(certificate)]
+
+    attachment = {
+        'title': certificate.name,
+        'color': 'danger',
+        'fields': [
+            {
+                'short': True,
+                'value': 'joe@example.com',
+                'title': 'Owner'
+            },
+            {
+                'short': True,
+                'value': u'Wednesday, January 1, 2020',
+                'title': 'Expires'
+            }, {
+                'short': True,
+                'value': 0,
+                'title': 'Endpoints Detected'
+            }
+        ],
+        'title_link': 'https://lemur.example.com/#/certificates/{name}'.format(name=certificate.name),
+        'mrkdwn_in': ['text'],
+        'text': '',
+        'fallback': ''
+    }
+
+    assert attachment == create_expiration_attachments(data)[0]

lemur/plugins/lemur_verisign/plugin.py

@@ -21,7 +21,6 @@ from lemur.common.utils import get_psuedo_random_string
 # https://support.venafi.com/entries/66445046-Info-VeriSign-Error-Codes
 VERISIGN_ERRORS = {
     "0x30c5": "Domain Mismatch when enrolling for an SSL certificate, a domain in your request has not been added to verisign",
-    "0x482d": "Cannot issue SHA1 certificates expiring after 31/12/2016",
     "0x3a10": "Invalid X509 certificate format.: an unsupported certificate format was submitted",
     "0x4002": "Internal QM Error. : Internal Database connection error.",
     "0x3301": "Bad transaction id or parent cert not renewable.: User try to renew a certificate that is not yet ready for renew or the transaction id is wrong",
@@ -56,6 +55,9 @@ VERISIGN_ERRORS = {
     "0x3043": "Certificates must have a validity of at least 1 day",
     "0x950b": "CSR: Invalid State",
     "0x3105": "Organization Name Not Matched",
+    "0x300a": "Domain/SubjectAltName Mismatched -- make sure that the SANs have the proper domain suffix",
+    "0x950e": "Invalid Common Name -- make sure the CN has a proper domain suffix",
+    "0xa00e": "Pending. (Insufficient number of tokens.)"
 }
 
 
@@ -78,8 +80,8 @@ def process_options(options):
     }
 
     if options.get('validity_end'):
-        end_date, period = get_default_issuance(options)
-        data['specificEndDate'] = str(end_date)
+        period = get_default_issuance(options)
+        data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY")
         data['validityPeriod'] = period
 
     elif options.get('validity_years'):
@@ -98,19 +100,16 @@ def get_default_issuance(options):
     :param options:
     :return:
     """
-    specific_end_date = arrow.get(options['validity_end']).replace(days=-1).format("MM/DD/YYYY")
-
     now = arrow.utcnow()
-    then = arrow.get(options['validity_end'])
 
-    if then < now.replace(years=+1):
+    if options['validity_end'] < now.replace(years=+1):
         validity_period = '1Y'
-    elif then < now.replace(years=+2):
+    elif options['validity_end'] < now.replace(years=+2):
         validity_period = '2Y'
     else:
         raise Exception("Verisign issued certificates cannot exceed two years in validity")
 
-    return specific_end_date, validity_period
+    return validity_period
 
 
 def handle_response(content):
@@ -186,6 +185,25 @@ class VerisignIssuerPlugin(IssuerPlugin):
         response = self.session.post(url, headers={'content-type': 'application/x-www-form-urlencoded'})
         return handle_response(response.content)['Response']['Order']
 
+    def get_pending_certificates(self):
+        """
+        Uses Verisign to fetch the number of certificate awaiting approval.
+
+        :return:
+        """
+        url = current_app.config.get("VERISIGN_URL") + '/reportingws'
+
+        end = arrow.now()
+        start = end.replace(days=-7)
+        data = {
+            'reportType': 'summary',
+            'certProductType': 'Server',
+            'startDate': start.format("MM/DD/YYYY"),
+            'endDate': end.format("MM/DD/YYYY"),
+        }
+        response = self.session.post(url, data=data)
+        return response.json()['certificateSummary'][0]['Pending']
+
 
 class VerisignSourcePlugin(SourcePlugin):
     title = 'Verisign'

lemur/plugins/lemur_verisign/tests/test_verisign.py

@@ -1,5 +1,4 @@
 
 def test_get_certificates(app):
     from lemur.plugins.base import plugins
-    p = plugins.get('verisign-source')
-    p.get_certificates()
+    p = plugins.get('verisign-issuer')

lemur/roles/models.py

@@ -27,5 +27,8 @@ class Role(db.Model):
     authority_id = Column(Integer, ForeignKey('authorities.id'))
     authorities = relationship("Authority", secondary=roles_authorities, passive_deletes=True, backref="role", cascade='all,delete')
     user_id = Column(Integer, ForeignKey('users.id'))
-    users = relationship("User", secondary=roles_users, viewonly=True, backref="role")
+    users = relationship("User", secondary=roles_users, passive_deletes=True, backref="role")
     certificates = relationship("Certificate", secondary=roles_certificates, backref="role")
+
+    def __repr__(self):
+        return "Role(name={name})".format(name=self.name)

lemur/roles/service.py

@@ -27,7 +27,7 @@ def update(role_id, name, description, users):
     role = get(role_id)
     role.name = name
     role.description = description
-    role = database.update_list(role, 'users', User, users)
+    role.users = users
     database.update(role)
     return role
 
@@ -46,7 +46,7 @@ def create(name, password=None, description=None, username=None, users=None):
     role = Role(name=name, description=description, username=username, password=password)
 
     if users:
-        role = database.update_list(role, 'users', User, users)
+        role.users = users
 
     return database.create(role)
 

lemur/roles/views.py

@@ -7,13 +7,13 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from flask import Blueprint
-from flask import make_response, jsonify, abort, g
+from flask import Blueprint, g
+from flask import make_response, jsonify
 from flask.ext.restful import reqparse, Api
 
 from lemur.roles import service
 from lemur.auth.service import AuthenticatedResource
-from lemur.auth.permissions import ViewRoleCredentialsPermission, admin_permission
+from lemur.auth.permissions import RoleMemberPermission, admin_permission
 from lemur.common.utils import paginated_parser
 
 from lemur.common.schema import validate_schema
@@ -83,6 +83,8 @@ class RolesList(AuthenticatedResource):
         parser.add_argument('id', type=str, location='args')
 
         args = parser.parse_args()
+        if not g.current_user.is_admin:
+            args['user_id'] = g.current_user.id
         return service.render(args)
 
     @admin_permission.require(http_exception=403)
@@ -106,7 +108,9 @@ class RolesList(AuthenticatedResource):
                  "description": "this is role3",
                  "username": null,
                  "password": null,
-                 "users": []
+                 "users": [
+                    {'id': 1}
+                 ]
               }
 
            **Example response**:
@@ -171,14 +175,14 @@ class RoleViewCredentials(AuthenticatedResource):
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
-        permission = ViewRoleCredentialsPermission(role_id)
+        permission = RoleMemberPermission(role_id)
         if permission.can():
             role = service.get(role_id)
             response = make_response(jsonify(username=role.username, password=role.password), 200)
             response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store'
             response.headers['pragma'] = 'no-cache'
             return response
-        abort(403)
+        return dict(message='You are not authorized to view the credentials for this role.'), 403
 
 
 class Roles(AuthenticatedResource):
@@ -220,12 +224,11 @@ class Roles(AuthenticatedResource):
            :statuscode 403: unauthenticated
         """
         # we want to make sure that we cannot view roles that we are not members of
-        if not g.current_user.is_admin:
-            user_role_ids = set([r.id for r in g.current_user.roles])
-            if role_id not in user_role_ids:
-                return dict(message="You are not allowed to view a role which you are not a member of"), 403
+        permission = RoleMemberPermission(role_id)
+        if permission.can():
+            return service.get(role_id)
 
-        return service.get(role_id)
+        return dict(message="You are not allowed to view a role which you are not a member of."), 403
 
     @validate_schema(role_input_schema, role_output_schema)
     def put(self, role_id, data=None):
@@ -265,10 +268,10 @@ class Roles(AuthenticatedResource):
            :statuscode 200: no error
            :statuscode 403: unauthenticated
         """
-        permission = ViewRoleCredentialsPermission(role_id)
+        permission = RoleMemberPermission(role_id)
         if permission.can():
             return service.update(role_id, data['name'], data.get('description'), data.get('users'))
-        abort(403)
+        return dict(message='You are not authorized to modify this role.'), 403
 
     @admin_permission.require(http_exception=403)
     def delete(self, role_id):

lemur/schemas.py

@@ -7,17 +7,68 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from marshmallow import fields, post_load, pre_load, post_dump, validates_schema
+from sqlalchemy.orm.exc import NoResultFound
+
+from marshmallow import fields, post_load, pre_load, post_dump, validates_schema
+from marshmallow.exceptions import ValidationError
 
-from lemur.authorities.models import Authority
-from lemur.certificates.models import Certificate
 from lemur.common import validators
 from lemur.common.schema import LemurSchema, LemurInputSchema, LemurOutputSchema
-from lemur.destinations.models import Destination
-from lemur.notifications.models import Notification
+
 from lemur.plugins import plugins
 from lemur.roles.models import Role
 from lemur.users.models import User
+from lemur.authorities.models import Authority
+from lemur.certificates.models import Certificate
+from lemur.destinations.models import Destination
+from lemur.notifications.models import Notification
+
+
+def get_object_attribute(data, many=False):
+    if many:
+        ids = [d.get('id') for d in data]
+        names = [d.get('name') for d in data]
+
+        if None in ids:
+            if None in names:
+                raise ValidationError('Associated object require a name or id.')
+            else:
+                return 'name'
+        return 'id'
+    else:
+        if data.get('id'):
+            return 'id'
+        elif data.get('name'):
+            return 'name'
+        else:
+            raise ValidationError('Associated object require a name or id.')
+
+
+def fetch_objects(model, data, many=False):
+    attr = get_object_attribute(data, many=many)
+
+    if many:
+        values = [v[attr] for v in data]
+        items = model.query.filter(getattr(model, attr).in_(values)).all()
+        found = [getattr(i, attr) for i in items]
+        diff = set(values).symmetric_difference(set(found))
+
+        if diff:
+            raise ValidationError('Unable to locate {model} with {attr} {diff}'.format(
+                model=model,
+                attr=attr,
+                diff=",".join(list(diff))))
+
+        return items
+
+    else:
+        try:
+            return model.query.filter(getattr(model, attr) == data[attr]).one()
+        except NoResultFound:
+            raise ValidationError('Unable to find {model} with {attr}: {data}'.format(
+                model=model,
+                attr=attr,
+                data=data[attr]))
 
 
 class AssociatedAuthoritySchema(LemurInputSchema):
@@ -26,72 +77,52 @@ class AssociatedAuthoritySchema(LemurInputSchema):
 
     @post_load
     def get_object(self, data, many=False):
-        if data.get('id'):
-            return Authority.query.filter(Authority.id == data['id']).one()
-        elif data.get('name'):
-            return Authority.query.filter(Authority.name == data['name']).one()
+        return fetch_objects(Authority, data, many=many)
 
 
 class AssociatedRoleSchema(LemurInputSchema):
-    id = fields.Int(required=True)
+    id = fields.Int()
     name = fields.String()
 
     @post_load
     def get_object(self, data, many=False):
-        if many:
-            ids = [d['id'] for d in data]
-            return Role.query.filter(Role.id.in_(ids)).all()
-        else:
-            return Role.query.filter(Role.id == data['id']).one()
+        return fetch_objects(Role, data, many=many)
 
 
 class AssociatedDestinationSchema(LemurInputSchema):
-    id = fields.Int(required=True)
+    id = fields.Int()
     name = fields.String()
 
     @post_load
     def get_object(self, data, many=False):
-        if many:
-            ids = [d['id'] for d in data]
-            return Destination.query.filter(Destination.id.in_(ids)).all()
-        else:
-            return Destination.query.filter(Destination.id == data['id']).one()
+        return fetch_objects(Destination, data, many=many)
 
 
 class AssociatedNotificationSchema(LemurInputSchema):
-    id = fields.Int(required=True)
+    id = fields.Int()
+    name = fields.String()
 
     @post_load
     def get_object(self, data, many=False):
-        if many:
-            ids = [d['id'] for d in data]
-            return Notification.query.filter(Notification.id.in_(ids)).all()
-        else:
-            return Notification.query.filter(Notification.id == data['id']).one()
+        return fetch_objects(Notification, data, many=many)
 
 
 class AssociatedCertificateSchema(LemurInputSchema):
-    id = fields.Int(required=True)
+    id = fields.Int()
+    name = fields.String()
 
     @post_load
     def get_object(self, data, many=False):
-        if many:
-            ids = [d['id'] for d in data]
-            return Certificate.query.filter(Certificate.id.in_(ids)).all()
-        else:
-            return Certificate.query.filter(Certificate.id == data['id']).one()
+        return fetch_objects(Certificate, data, many=many)
 
 
 class AssociatedUserSchema(LemurInputSchema):
-    id = fields.Int(required=True)
+    id = fields.Int()
+    name = fields.String()
 
     @post_load
     def get_object(self, data, many=False):
-        if many:
-            ids = [d['id'] for d in data]
-            return User.query.filter(User.id.in_(ids)).all()
-        else:
-            return User.query.filter(User.id == data['id']).one()
+        return fetch_objects(User, data, many=many)
 
 
 class PluginInputSchema(LemurInputSchema):
@@ -102,8 +133,11 @@ class PluginInputSchema(LemurInputSchema):
 
     @post_load
     def get_object(self, data, many=False):
-        data['plugin_object'] = plugins.get(data['slug'])
-        return data
+        try:
+            data['plugin_object'] = plugins.get(data['slug'])
+            return data
+        except Exception:
+            raise ValidationError('Unable to find plugin: {0}'.format(data['slug']))
 
 
 class PluginOutputSchema(LemurOutputSchema):
@@ -186,7 +220,7 @@ class SubAltNameSchema(BaseExtensionSchema):
 
     @validates_schema
     def check_sensitive(self, data):
-        if data['name_type'] == 'DNSName':
+        if data.get('name_type') == 'DNSName':
             validators.sensitive_domain(data['value'])
 
 
@@ -196,7 +230,7 @@ class SubAltNamesSchema(BaseExtensionSchema):
 
 class CustomOIDSchema(BaseExtensionSchema):
     oid = fields.String()
-    oid_type = fields.String(validate=validators.oid_type)
+    encoding = fields.String(validate=validators.encoding)
     value = fields.String()
 
 
@@ -210,3 +244,14 @@ class ExtensionSchema(BaseExtensionSchema):
     authority_key_identifier = fields.Nested(AuthorityKeyIdentifierSchema)
     certificate_info_access = fields.Nested(CertificateInfoAccessSchema)
     custom = fields.List(fields.Nested(CustomOIDSchema))
+
+
+class EndpointNestedOutputSchema(LemurOutputSchema):
+    __envelope__ = False
+    id = fields.Integer()
+    description = fields.String()
+    name = fields.String()
+    dnsname = fields.String()
+    owner = fields.Email()
+    type = fields.String()
+    active = fields.Boolean()