Merge branch 'master' into linuxdst

Makefile

@@ -109,12 +109,15 @@ ifndef VIRTUAL_ENV
     $(error Please activate virtualenv first)
 endif
 	@echo "--> Updating Python requirements"
+	pip install --upgrade pip
 	pip install --upgrade pip-tools
-	pip-compile --output-file requirements-docs.txt requirements-docs.in -U
+	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
-	pip-compile --output-file requirements-dev.txt requirements-dev.in -U
+	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
-	pip-compile --output-file requirements-tests.txt requirements-tests.in -U
+	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
-	pip-compile --output-file requirements.txt requirements.in -U
+	pip-compile --output-file requirements.txt requirements.in -U --no-index
 	@echo "--> Done updating Python requirements"
+	@echo "--> Removing python-ldap from requirements-docs.txt"
+	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt
 	@echo "--> Installing new dependencies"
 	pip install -e .
 	@echo "--> Done installing new dependencies"

README.rst

@@ -5,7 +5,7 @@ Lemur
    :alt: Join the chat at https://gitter.im/Netflix/lemur
    :target: https://gitter.im/Netflix/lemur?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: https://readthedocs.io/projects/lemur/badge/?version=latest
+.. image:: https://readthedocs.org/projects/lemur/badge/?version=latest
     :target: https://lemur.readthedocs.io
     :alt: Latest Docs
 
@@ -14,6 +14,10 @@ Lemur
 .. image:: https://travis-ci.org/Netflix/lemur.svg
     :target: https://travis-ci.org/Netflix/lemur
 
+.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=master
+    :target: https://coveralls.io/github/Netflix/lemur?branch=master
+
+
 
 Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs
 and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults.

docs/administration.rst

@@ -65,6 +65,36 @@ Basic Configuration
         SQLALCHEMY_DATABASE_URI = 'postgresql://<user>:<password>@<hostname>:5432/lemur'
 
 
+.. data:: SQLALCHEMY_POOL_SIZE
+:noindex:
+
+            The default connection pool size is 5 for sqlalchemy managed connections.   Depending on the number of Lemur instances,
+            please specify per instance connection pool size.  Below is an example to set connection pool size to 10.
+
+        ::
+
+        SQLALCHEMY_POOL_SIZE = 10
+
+
+    .. warning::
+This is an optional setting but important to review and set for optimal database connection usage and for overall database performance.
+
+.. data:: SQLALCHEMY_MAX_OVERFLOW
+:noindex:
+
+        This setting allows to create connections in addition to specified number of connections in pool size.   By default, sqlalchemy
+        allows 10 connections to create in addition to the pool size.  This is also an optional setting.  If `SQLALCHEMY_POOL_SIZE` and
+        `SQLALCHEMY_MAX_OVERFLOW` are not speficied then each Lemur instance may create maximum of 15 connections.
+
+    ::
+
+        SQLALCHECK_MAX_OVERFLOW = 0
+
+
+    .. note::
+Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size.
+
+
 .. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION
     :noindex:
 
@@ -1156,6 +1186,31 @@ Digicert
     https://github.com/opendns/lemur-digicert
 
 
+InfluxDB
+--------
+
+:Authors:
+    Titouan Christophe
+:Type:
+    Metric
+:Description:
+    Sends key metrics to InfluxDB
+:Links:
+    https://github.com/titouanc/lemur-influxdb
+
+Hashicorp Vault
+---------------
+
+:Authors:
+    Ron Cohen
+:Type:
+    Issuer
+:Description:
+    Adds support for basic Vault PKI secret backend.
+:Links:
+    https://github.com/RcRonco/lemur_vault
+
+
 Have an extension that should be listed here? Submit a `pull request <https://github.com/netflix/lemur>`_ and we'll
 get it added.
 

docs/developer/index.rst

@@ -48,7 +48,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi
 * pip
 * virtualenv (ideally virtualenvwrapper)
 * node.js (for npm and building css/javascript)
-* (Optional) PostgreSQL
++* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
 
 Once you've got all that, the rest is simple:
 
@@ -77,6 +77,7 @@ Create a default Lemur configuration just as if this were a production instance:
 
 ::
 
+    lemur create_config
     lemur init
 
 You'll likely want to make some changes to the default configuration (we recommend developing against Postgres, for example). Once done, migrate your database using the following command:

docs/requirements.txt

@@ -1,36 +0,0 @@
-CloudFlare==1.7.5
-Flask==0.12
-Flask-RESTful==0.3.6
-Flask-SQLAlchemy==2.1
-Flask-Script==2.0.5
-Flask-Migrate==2.1.1
-Flask-Bcrypt==0.7.1
-Flask-Principal==0.4.0
-Flask-Mail==0.9.1
-SQLAlchemy-Utils==0.32.14
-requests==2.11.1
-ndg-httpsclient==0.4.2
-psycopg2==2.7.3
-arrow==0.10.0
-six==1.10.0
-marshmallow-sqlalchemy==0.13.1
-gunicorn==19.7.1
-marshmallow==2.13.6
-cryptography==1.9
-xmltodict==0.11.0
-pyjwt==1.5.2
-lockfile==0.12.2
-inflection==0.3.1
-future==0.16.0
-boto3==1.4.6
-acme==0.18.1
-retrying==1.3.3
-tabulate==0.7.7
-pem==17.1.0
-raven[flask]==6.1.0
-jinja2==2.9.6
-# pyldap==2.4.37  # cannot be installed on rtd - required by ldap auth provider
-paramiko==2.4.1  # required for the SFTP destination plugin
-sphinx
-sphinxcontrib-httpdomain
-sphinx-rtd-theme

lemur/__init__.py

@@ -8,7 +8,8 @@
 
 
 """
-from __future__ import absolute_import, division, print_function
+import time
+from flask import g, request
 
 from lemur import factory
 from lemur.extensions import metrics
@@ -73,17 +74,6 @@ def configure_hook(app):
     """
     from flask import jsonify
     from werkzeug.exceptions import HTTPException
-    from lemur.decorators import crossdomain
-    if app.config.get('CORS'):
-        @app.after_request
-        @crossdomain(origin=u"http://localhost:3000", methods=['PUT', 'HEAD', 'GET', 'POST', 'OPTIONS', 'DELETE'])
-        def after(response):
-            return response
-
-    @app.after_request
-    def log_status(response):
-        metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
-        return response
 
     @app.errorhandler(Exception)
     def handle_error(e):
@@ -93,3 +83,29 @@ def configure_hook(app):
 
         app.logger.exception(e)
         return jsonify(error=str(e)), code
+
+    @app.before_request
+    def before_request():
+        g.request_start_time = time.time()
+
+    @app.after_request
+    def after_request(response):
+        # Return early if we don't have the start time
+        if not hasattr(g, 'request_start_time'):
+            return response
+
+        # Get elapsed time in milliseconds
+        elapsed = time.time() - g.request_start_time
+        elapsed = int(round(1000 * elapsed))
+
+        # Collect request/response tags
+        tags = {
+            'endpoint': request.endpoint,
+            'request_method': request.method.lower(),
+            'status_code': response.status_code
+        }
+
+        # Record our response time metric
+        metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags)
+        metrics.send('status_code_{}'.format(response.status_code), 'counter', 1)
+        return response

lemur/authorities/service.py

@@ -9,6 +9,7 @@
 
 """
 from lemur import database
+from lemur.common.utils import truthiness
 from lemur.extensions import metrics
 from lemur.authorities.models import Authority
 from lemur.roles import service as role_service
@@ -170,8 +171,8 @@ def render(args):
 
     if filt:
         terms = filt.split(';')
-        if 'active' in filt:  # this is really weird but strcmp seems to not work here??
+        if 'active' in filt:
-            query = query.filter(Authority.active == terms[1])
+            query = query.filter(Authority.active == truthiness(terms[1]))
         else:
             query = database.filter(query, Authority, terms)
 

lemur/certificates/service.py

@@ -8,7 +8,7 @@
 import arrow
 
 from flask import current_app
-from sqlalchemy import func, or_, not_, cast, Boolean, Integer
+from sqlalchemy import func, or_, not_, cast, Integer
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -17,7 +17,7 @@ from cryptography.hazmat.primitives import hashes, serialization
 from lemur import database
 from lemur.extensions import metrics, signals
 from lemur.plugins.base import plugins
-from lemur.common.utils import generate_private_key
+from lemur.common.utils import generate_private_key, truthiness
 
 from lemur.roles.models import Role
 from lemur.domains.models import Domain
@@ -319,9 +319,9 @@ def render(args):
         elif 'destination' in terms:
             query = query.filter(Certificate.destinations.any(Destination.id == terms[1]))
         elif 'notify' in filt:
-            query = query.filter(Certificate.notify == cast(terms[1], Boolean))
+            query = query.filter(Certificate.notify == truthiness(terms[1]))
         elif 'active' in filt:
-            query = query.filter(Certificate.active == terms[1])
+            query = query.filter(Certificate.active == truthiness(terms[1]))
         elif 'cn' in terms:
             query = query.filter(
                 or_(

lemur/certificates/views.py

@@ -300,12 +300,14 @@ class CertificatesUpload(AuthenticatedResource):
 
               {
                  "owner": "joe@example.com",
-                 "publicCert": "-----BEGIN CERTIFICATE-----...",
+                 "body": "-----BEGIN CERTIFICATE-----...",
-                 "intermediateCert": "-----BEGIN CERTIFICATE-----...",
+                 "chain": "-----BEGIN CERTIFICATE-----...",
                  "privateKey": "-----BEGIN RSA PRIVATE KEY-----..."
                  "destinations": [],
                  "notifications": [],
                  "replacements": [],
+                 "roles": [],
+                 "notify": true,
                  "name": "cert1"
               }
 

lemur/common/utils.py

@@ -14,10 +14,11 @@ from sqlalchemy import and_, func
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives.asymmetric import rsa, ec
 
 from flask_restful.reqparse import RequestParser
 
+from lemur.constants import CERTIFICATE_KEY_TYPES
 from lemur.exceptions import InvalidConfiguration
 
 paginated_parser = RequestParser()
@@ -78,17 +79,43 @@ def generate_private_key(key_type):
     """
     Generates a new private key based on key_type.
 
-    Valid key types: RSA2048, RSA4096
+    Valid key types: RSA2048, RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1',
+        'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
+        'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1',
+        'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2'
 
     :param key_type:
     :return:
     """
-    valid_key_types = ['RSA2048', 'RSA4096']
 
-    if key_type not in valid_key_types:
+    _CURVE_TYPES = {
+        "ECCPRIME192V1": ec.SECP192R1(),
+        "ECCPRIME256V1": ec.SECP256R1(),
+
+        "ECCSECP192R1": ec.SECP192R1(),
+        "ECCSECP224R1": ec.SECP224R1(),
+        "ECCSECP256R1": ec.SECP256R1(),
+        "ECCSECP384R1": ec.SECP384R1(),
+        "ECCSECP521R1": ec.SECP521R1(),
+        "ECCSECP256K1": ec.SECP256K1(),
+
+        "ECCSECT163K1": ec.SECT163K1(),
+        "ECCSECT233K1": ec.SECT233K1(),
+        "ECCSECT283K1": ec.SECT283K1(),
+        "ECCSECT409K1": ec.SECT409K1(),
+        "ECCSECT571K1": ec.SECT571K1(),
+
+        "ECCSECT163R2": ec.SECT163R2(),
+        "ECCSECT233R1": ec.SECT233R1(),
+        "ECCSECT283R1": ec.SECT283R1(),
+        "ECCSECT409R1": ec.SECT409R1(),
+        "ECCSECT571R2": ec.SECT571R1(),
+    }
+
+    if key_type not in CERTIFICATE_KEY_TYPES:
         raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format(
             key_type=key_type,
-            choices=",".join(valid_key_types)
+            choices=",".join(CERTIFICATE_KEY_TYPES)
         ))
 
     if 'RSA' in key_type:
@@ -98,6 +125,11 @@ def generate_private_key(key_type):
             key_size=key_size,
             backend=default_backend()
         )
+    elif 'ECC' in key_type:
+        return ec.generate_private_key(
+            curve=_CURVE_TYPES[key_type],
+            backend=default_backend()
+        )
 
 
 def is_weekend(date):
@@ -175,3 +207,9 @@ def windowed_query(q, column, windowsize):
             column, windowsize):
         for row in q.filter(whereclause).order_by(column):
             yield row
+
+
+def truthiness(s):
+    """If input string resembles something truthy then return True, else False."""
+
+    return s.lower() in ('true', 'yes', 'on', 't', '1')

lemur/constants.py

@@ -9,3 +9,26 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
 
 SUCCESS_METRIC_STATUS = 'success'
 FAILURE_METRIC_STATUS = 'failure'
+
+CERTIFICATE_KEY_TYPES = [
+    'RSA2048',
+    'RSA4096',
+    'ECCPRIME192V1',
+    'ECCPRIME256V1',
+    'ECCSECP192R1',
+    'ECCSECP224R1',
+    'ECCSECP256R1',
+    'ECCSECP384R1',
+    'ECCSECP521R1',
+    'ECCSECP256K1',
+    'ECCSECT163K1',
+    'ECCSECT233K1',
+    'ECCSECT283K1',
+    'ECCSECT409K1',
+    'ECCSECT571K1',
+    'ECCSECT163R2',
+    'ECCSECT233R1',
+    'ECCSECT283R1',
+    'ECCSECT409R1',
+    'ECCSECT571R2'
+]

lemur/decorators.py

@@ -1,56 +0,0 @@
-"""
-.. module: lemur.decorators
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-"""
-from builtins import str
-
-from datetime import timedelta
-from flask import make_response, request, current_app
-
-from functools import update_wrapper
-
-
-# this is only used for dev
-def crossdomain(origin=None, methods=None, headers=None,
-                max_age=21600, attach_to_all=True,
-                automatic_options=True):  # pragma: no cover
-    if methods is not None:
-        methods = ', '.join(sorted(x.upper() for x in methods))
-
-    if headers is not None and not isinstance(headers, str):
-        headers = ', '.join(x.upper() for x in headers)
-
-    if not isinstance(origin, str):
-        origin = ', '.join(origin)
-
-    if isinstance(max_age, timedelta):
-        max_age = max_age.total_seconds()
-
-    def get_methods():
-        if methods is not None:
-            return methods
-
-        options_resp = current_app.make_default_options_response()
-        return options_resp.headers['allow']
-
-    def decorator(f):
-        def wrapped_function(*args, **kwargs):
-            if automatic_options and request.method == 'OPTIONS':
-                resp = current_app.make_default_options_response()
-            else:
-                resp = make_response(f(*args, **kwargs))
-            if not attach_to_all and request.method != 'OPTIONS':
-                return resp
-
-            h = resp.headers
-            h['Access-Control-Allow-Origin'] = origin
-            h['Access-Control-Allow-Methods'] = get_methods()
-            h['Access-Control-Max-Age'] = str(max_age)
-            h['Access-Control-Allow-Headers'] = "Origin, X-Requested-With, Content-Type, Accept, Authorization "
-            h['Access-Control-Allow-Credentials'] = 'true'
-            return resp
-
-        f.provide_automatic_options = False
-        return update_wrapper(wrapped_function, f)
-    return decorator

lemur/endpoints/service.py

@@ -13,6 +13,7 @@ import arrow
 from sqlalchemy import func
 
 from lemur import database
+from lemur.common.utils import truthiness
 from lemur.endpoints.models import Endpoint, Policy, Cipher
 from lemur.extensions import metrics
 
@@ -142,7 +143,7 @@ def render(args):
     if filt:
         terms = filt.split(';')
         if 'active' in filt:  # this is really weird but strcmp seems to not work here??
-            query = query.filter(Endpoint.active == terms[1])
+            query = query.filter(Endpoint.active == truthiness(terms[1]))
         elif 'port' in filt:
             if terms[1] != 'null':  # ng-table adds 'null' if a number is removed
                 query = query.filter(Endpoint.port == terms[1])

lemur/extensions.py

@@ -26,3 +26,6 @@ sentry = Sentry()
 
 from blinker import Namespace
 signals = Namespace()
+
+from flask_cors import CORS
+cors = CORS()

lemur/factory.py

@@ -21,7 +21,7 @@ from flask import Flask
 
 from lemur.certificates.hooks import activate_debug_dump
 from lemur.common.health import mod as health
-from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry
+from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry, cors
 
 
 DEFAULT_BLUEPRINTS = (
@@ -125,6 +125,10 @@ def configure_extensions(app):
     metrics.init_app(app)
     sentry.init_app(app)
 
+    if app.config['CORS']:
+        app.config['CORS_HEADERS'] = 'Content-Type'
+        cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True)
+
 
 def configure_blueprints(app, blueprints):
     """

lemur/manage.py

@@ -208,16 +208,16 @@ class InitializeApp(Command):
         if operator_role:
             sys.stdout.write("[-] Operator role already created, skipping...!\n")
         else:
-            # we create an admin role
+            # we create an operator role
             operator_role = role_service.create('operator', description='This is the Lemur operator role.')
             sys.stdout.write("[+] Created 'operator' role\n")
 
         read_only_role = role_service.get_by_name('read-only')
 
         if read_only_role:
-            sys.stdout.write("[-] Operator role already created, skipping...!\n")
+            sys.stdout.write("[-] Read only role already created, skipping...!\n")
         else:
-            # we create an admin role
+            # we create an read only role
             read_only_role = role_service.create('read-only', description='This is the Lemur read only role.')
             sys.stdout.write("[+] Created 'read-only' role\n")
 
@@ -237,9 +237,6 @@ class InitializeApp(Command):
         else:
             sys.stdout.write("[-] Default user has already been created, skipping...!\n")
 
-        sys.stdout.write("[+] Creating expiration email notifications!\n")
-        sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format("LEMUR_SECURITY_TEAM_EMAIL"))
-
         intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", [])
         sys.stdout.write(
             "[!] Creating {num} notifications for {intervals} days as specified by LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS\n".format(
@@ -249,14 +246,21 @@ class InitializeApp(Command):
         )
 
         recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
+        sys.stdout.write("[+] Creating expiration email notifications!\n")
+        sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format(recipients))
         notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients)
 
-        days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30)
+        _DEFAULT_ROTATION_INTERVAL = 'default'
-        sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
+        default_rotation_interval = policy_service.get_by_name(_DEFAULT_ROTATION_INTERVAL)
-            days=days
+
-        ))
+        if default_rotation_interval:
+            sys.stdout.write("[-] Default rotation interval policy already created, skipping...!\n")
+        else:
+            days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30)
+            sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
+                days=days))
+            policy_service.create(days=days, name=_DEFAULT_ROTATION_INTERVAL)
 
-        policy_service.create(days=days, name='default')
         sys.stdout.write("[/] Done!\n")
 
 

lemur/notifications/service.py

@@ -12,6 +12,7 @@ from flask import current_app
 
 from lemur import database
 from lemur.certificates.models import Certificate
+from lemur.common.utils import truthiness
 from lemur.notifications.models import Notification
 
 
@@ -169,10 +170,8 @@ def render(args):
 
     if filt:
         terms = filt.split(';')
-        if terms[0] == 'active' and terms[1] == 'false':
+        if terms[0] == 'active':
-            query = query.filter(Notification.active == False)  # noqa
+            query = query.filter(Notification.active == truthiness(terms[1]))
-        elif terms[0] == 'active' and terms[1] == 'true':
-            query = query.filter(Notification.active == True)  # noqa
         else:
             query = database.filter(query, Notification, terms)
 

lemur/pending_certificates/service.py

@@ -5,9 +5,10 @@
 """
 import arrow
 
-from sqlalchemy import or_, cast, Boolean, Integer
+from sqlalchemy import or_, cast, Integer
 
 from lemur import database
+from lemur.common.utils import truthiness
 from lemur.plugins.base import plugins
 
 from lemur.roles.models import Role
@@ -181,9 +182,9 @@ def render(args):
         elif 'destination' in terms:
             query = query.filter(PendingCertificate.destinations.any(Destination.id == terms[1]))
         elif 'notify' in filt:
-            query = query.filter(PendingCertificate.notify == cast(terms[1], Boolean))
+            query = query.filter(PendingCertificate.notify == truthiness(terms[1]))
         elif 'active' in filt:
-            query = query.filter(PendingCertificate.active == terms[1])
+            query = query.filter(PendingCertificate.active == truthiness(terms[1]))
         elif 'cn' in terms:
             query = query.filter(
                 or_(

lemur/plugins/lemur_digicert/plugin.py

@@ -157,7 +157,7 @@ def map_cis_fields(options, csr):
         "csr": csr,
         "signature_hash": signature_hash(options.get('signing_algorithm')),
         "validity": {
-            "valid_to": options['validity_end'].format('YYYY-MM-DD')
+            "valid_to": options['validity_end'].format('YYYY-MM-DDTHH:MM') + 'Z'
         },
         "organization": {
             "name": options['organization'],
@@ -491,6 +491,11 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
 
         self.session.headers.pop('Accept')
         end_entity = pem.parse(certificate_pem)[0]
+
+        if 'ECC' in issuer_options['key_type']:
+            return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_ECC_CIS_INTERMEDIATE'), data['id']
+
+        # By default return RSA
         return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_CIS_INTERMEDIATE'), data['id']
 
     def revoke_certificate(self, certificate, comments):

lemur/plugins/lemur_digicert/tests/test_digicert.py

@@ -103,7 +103,7 @@ def test_map_cis_fields(app):
         'signature_hash': 'sha256',
         'organization': {'name': 'Example, Inc.', 'units': ['Example Org']},
         'validity': {
-            'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DD')
+            'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DDTHH:MM') + 'Z'
         },
         'profile_name': None
     }
@@ -132,7 +132,7 @@ def test_map_cis_fields(app):
             'signature_hash': 'sha256',
             'organization': {'name': 'Example, Inc.', 'units': ['Example Org']},
             'validity': {
-                'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DD')
+                'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DDTHH:MM') + 'Z'
             },
             'profile_name': None
         }

lemur/plugins/lemur_statsd/docs/requirements.txt

@@ -0,0 +1 @@
+datadog==0.14.0

lemur/plugins/lemur_statsd/lemur_statsd/__init__.py

@@ -0,0 +1,4 @@
+try:
+    VERSION = __import__('pkg_resources').get_distribution(__name__).version
+except Exception as e:
+    VERSION = 'Unknown'

lemur/plugins/lemur_statsd/lemur_statsd/plugin.py

@@ -0,0 +1,45 @@
+import lemur_statsd as plug
+
+from flask import current_app
+from lemur.plugins.bases.metric import MetricPlugin
+from datadog import DogStatsd
+
+
+class StatsdMetricPlugin(MetricPlugin):
+    title = 'Statsd'
+    slug = 'statsd-metrics'
+    description = 'Adds support for sending metrics to Statsd'
+    version = plug.VERSION
+
+    def __init__(self):
+        host = current_app.config.get('STATSD_HOST')
+        port = current_app.config.get('STATSD_PORT')
+        prefix = current_app.config.get('STATSD_PREFIX')
+
+        self.statsd = DogStatsd(host=host, port=port, namespace=prefix)
+
+    def submit(self, metric_name, metric_type, metric_value, metric_tags=None, options=None):
+        valid_types = ['COUNTER', 'GAUGE', 'TIMER']
+        tags = []
+
+        if metric_type.upper() not in valid_types:
+            raise Exception(
+                "Invalid Metric Type for Statsd, '{metric}' choose from: {options}".format(
+                    metric=metric_type, options=','.join(valid_types)
+                )
+            )
+
+        if metric_tags:
+            if not isinstance(metric_tags, dict):
+                raise Exception("Invalid Metric Tags for Statsd: Tags must be in dict format")
+            else:
+                tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items())
+
+        if metric_type.upper() == 'COUNTER':
+            self.statsd.increment(metric_name, metric_value, tags)
+        elif metric_type.upper() == 'GAUGE':
+            self.statsd.gauge(metric_name, metric_value, tags)
+        elif metric_type.upper() == 'TIMER':
+            self.statsd.timing(metric_name, metric_value, tags)
+
+        return

lemur/plugins/lemur_statsd/setup.py

@@ -0,0 +1,24 @@
+"""Basic package information"""
+from __future__ import absolute_import
+from setuptools import setup, find_packages
+
+install_requires = [
+    'lemur',
+    'datadog'
+]
+
+setup(
+    name='lemur_statsd',
+    version='1.0.0',
+    author='Cloudflare Security Engineering',
+    author_email='',
+    include_package_data=True,
+    packages=find_packages(),
+    zip_safe=False,
+    install_requires=install_requires,
+    entry_points={
+        'lemur.plugins': [
+            'statsd = lemur_statsd.plugin:StatsdMetricPlugin',
+        ]
+    }
+)

lemur/policies/service.py

@@ -18,6 +18,15 @@ def get(policy_id):
     return database.get(RotationPolicy, policy_id)
 
 
+def get_by_name(policy_name):
+    """
+    Retrieves policy by its name.
+    :param policy_name:
+    :return:
+    """
+    return database.get_all(RotationPolicy, policy_name, field='name').all()
+
+
 def delete(policy_id):
     """
     Delete a rotation policy.

lemur/static/app/angular/authorities/authority/options.tpl.html

@@ -20,7 +20,8 @@
       Key Type
     </label>
     <div class="col-sm-10">
-      <select class="form-control" ng-model="authority.keyType" ng-options="option for option in ['RSA2048', 'RSA4096']" ng-init="authority.keyType = 'RSA2048'"></select>
+      <select class="form-control" ng-model="authority.keyType" ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
+      'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2']" ng-init="authority.keyType = 'RSA2048'"></select>
     </div>
   </div>
   <div ng-show="authority.sensitivity == 'high'" class="form-group">

lemur/static/app/angular/certificates/certificate/options.tpl.html

@@ -32,7 +32,10 @@
         </label>
         <div class="col-sm-10">
           <select class="form-control" ng-model="certificate.keyType"
-                  ng-options="option for option in ['RSA2048', 'RSA4096']"
+                  ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1',
+                                                    'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
+                                                    'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1',
+                                                    'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2']"
                   ng-init="certificate.keyType = 'RSA2048'"></select>
         </div>
       </div>

lemur/tests/test_certificates.py

@@ -717,3 +717,11 @@ def test_certificates_upload_patch(client, token, status):
 def test_sensitive_sort(client):
     resp = client.get(api.url_for(CertificatesList) + '?sortBy=private_key&sortDir=asc', headers=VALID_ADMIN_HEADER_TOKEN)
     assert "'private_key' is not sortable or filterable" in resp.json['message']
+
+
+def test_boolean_filter(client):
+    resp = client.get(api.url_for(CertificatesList) + '?filter=notify;true', headers=VALID_ADMIN_HEADER_TOKEN)
+    assert resp.status_code == 200
+    # Also don't crash with invalid input (we currently treat that as false)
+    resp = client.get(api.url_for(CertificatesList) + '?filter=notify;whatisthis', headers=VALID_ADMIN_HEADER_TOKEN)
+    assert resp.status_code == 200

lemur/tests/test_utils.py

@@ -6,9 +6,27 @@ def test_generate_private_key():
 
     assert generate_private_key('RSA2048')
     assert generate_private_key('RSA4096')
+    assert generate_private_key('ECCPRIME192V1')
+    assert generate_private_key('ECCPRIME256V1')
+    assert generate_private_key('ECCSECP192R1')
+    assert generate_private_key('ECCSECP224R1')
+    assert generate_private_key('ECCSECP256R1')
+    assert generate_private_key('ECCSECP384R1')
+    assert generate_private_key('ECCSECP521R1')
+    assert generate_private_key('ECCSECP256K1')
+    assert generate_private_key('ECCSECT163K1')
+    assert generate_private_key('ECCSECT233K1')
+    assert generate_private_key('ECCSECT283K1')
+    assert generate_private_key('ECCSECT409K1')
+    assert generate_private_key('ECCSECT571K1')
+    assert generate_private_key('ECCSECT163R2')
+    assert generate_private_key('ECCSECT233R1')
+    assert generate_private_key('ECCSECT283R1')
+    assert generate_private_key('ECCSECT409R1')
+    assert generate_private_key('ECCSECT571R2')
 
     with pytest.raises(Exception):
-        generate_private_key('ECC')
+        generate_private_key('LEMUR')
 
 
 def test_get_authority_key():

requirements-dev.in

@@ -1,3 +1,5 @@
+# Run `make up-reqs` to update pinned dependencies in requirement text files
+
 flake8>=3.2,<4.0
 pre-commit
 invoke

requirements-dev.txt

@@ -2,15 +2,15 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements-dev.txt requirements-dev.in
+#    pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
 #
-aspy.yaml==1.0.0          # via pre-commit
+aspy.yaml==1.1.0          # via pre-commit
-cached-property==1.4.0    # via pre-commit
+cached-property==1.4.2    # via pre-commit
-certifi==2018.1.18        # via requests
+certifi==2018.4.16        # via requests
 cfgv==1.0.0               # via pre-commit
 chardet==3.0.4            # via requests
 flake8==3.5.0
-identify==1.0.8           # via pre-commit
+identify==1.0.13          # via pre-commit
 idna==2.6                 # via requests
 invoke==0.22.1
 mccabe==0.6.1             # via flake8
@@ -23,7 +23,7 @@ pyyaml==3.12              # via aspy.yaml, pre-commit
 requests-toolbelt==0.8.0  # via twine
 requests==2.18.4          # via requests-toolbelt, twine
 six==1.11.0               # via cfgv, pre-commit
-tqdm==4.19.9              # via twine
+tqdm==4.23.0              # via twine
 twine==1.11.0
 urllib3==1.22             # via requests
 virtualenv==15.2.0        # via pre-commit

requirements-docs.in

@@ -1,3 +1,7 @@
+# Note: python-ldap from requirements breaks due to readthedocs.io not having the correct header files
+# The `make up-reqs` will update all requirement text files, and forcibly remove python-ldap
+# from requirements-docs.txt
+-r requirements.txt
 sphinx
 sphinxcontrib-httpdomain
 sphinx-rtd-theme

requirements-docs.txt

@@ -2,26 +2,78 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements-docs.txt requirements-docs.in
+#    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
+acme==0.23.0
 alabaster==0.7.10         # via sphinx
+alembic-autogenerate-enums==0.0.2
+alembic==0.9.9
+aniso8601==3.0.0
+arrow==0.12.1
+asn1crypto==0.24.0
 babel==2.5.3              # via sphinx
-certifi==2018.1.18        # via requests
+bcrypt==3.1.4
-chardet==3.0.4            # via requests
+blinker==1.4
-docutils==0.14            # via sphinx
+boto3==1.7.6
-idna==2.6                 # via requests
+botocore==1.10.6
+cffi==1.11.5
+click==6.7
+cryptography==2.2.2
+docutils==0.14
+flask-bcrypt==0.7.1
+flask-cors==3.0.3
+flask-mail==0.9.1
+flask-migrate==2.1.1
+flask-principal==0.4.0
+flask-restful==0.3.6
+flask-script==2.0.6
+flask-sqlalchemy==2.3.2
+flask==0.12
+future==0.16.0
+gunicorn==19.7.1
+idna==2.6
 imagesize==1.0.0          # via sphinx
-jinja2==2.10              # via sphinx
+inflection==0.3.1
-markupsafe==1.0           # via jinja2
+itsdangerous==0.24
+jinja2==2.10
+jmespath==0.9.3
+josepy==1.1.0
+lockfile==0.12.2
+mako==1.0.7
+markupsafe==1.0
+marshmallow-sqlalchemy==0.13.2
+marshmallow==2.15.0
+mock==2.0.0
+ndg-httpsclient==0.4.4
 packaging==17.1           # via sphinx
+paramiko==2.4.1
+pbr==4.0.2
+pem==17.1.0
+psycopg2==2.7.4
+pyasn1-modules==0.2.1
+pyasn1==0.4.2
+pycparser==2.18
 pygments==2.2.0           # via sphinx
+pyjwt==1.6.1
+pynacl==1.2.1
+pyopenssl==17.2.0
 pyparsing==2.2.0          # via packaging
-pytz==2018.3              # via babel
+pyrfc3339==1.0
-requests==2.18.4          # via sphinx
+python-dateutil==2.7.2
-six==1.11.0               # via packaging, sphinx, sphinxcontrib-httpdomain
+python-editor==1.0.3
+pytz==2018.4
+raven[flask]==6.7.0
+requests[security]==2.11.1
+retrying==1.3.3
+s3transfer==0.1.13
+six==1.11.0
 snowballstemmer==1.2.1    # via sphinx
-sphinx-rtd-theme==0.2.4
+sphinx-rtd-theme==0.3.0
-sphinx==1.7.2
+sphinx==1.7.3
 sphinxcontrib-httpdomain==1.6.1
 sphinxcontrib-websupport==1.0.1  # via sphinx
-urllib3==1.22             # via requests
+sqlalchemy-utils==0.33.2
+sqlalchemy==1.2.7
+tabulate==0.8.2
+werkzeug==0.14.1
+xmltodict==0.11.0

requirements-tests.in

@@ -1,3 +1,5 @@
+# Run `make up-reqs` to update pinned dependencies in requirement text files
+
 coverage
 factory-boy
 Faker

requirements-tests.txt

@@ -2,26 +2,26 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements-tests.txt requirements-tests.in
+#    pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in
 #
 asn1crypto==0.24.0        # via cryptography
 attrs==17.4.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.6.19             # via moto
+boto3==1.7.6              # via moto
 boto==2.48.0              # via moto
-botocore==1.9.19          # via boto3, moto, s3transfer
+botocore==1.10.6          # via boto3, moto, s3transfer
-certifi==2018.1.18        # via requests
+certifi==2018.4.16        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
 click==6.7                # via flask
-cookies==2.2.1            # via moto
+cookies==2.2.1            # via moto, responses
 coverage==4.5.1
 cryptography==2.2.2       # via moto
 docker-pycreds==0.2.2     # via docker
-docker==3.1.4             # via moto
+docker==3.2.1             # via moto
 docutils==0.14            # via botocore
 factory-boy==2.10.0
-faker==0.8.12
+faker==0.8.13
 flask==0.12.2             # via pytest-flask
 freezegun==0.3.10
 idna==2.6                 # via cryptography, requests
@@ -33,24 +33,25 @@ jsonpickle==0.9.6         # via aws-xray-sdk
 markupsafe==1.0           # via jinja2
 mock==2.0.0               # via moto
 more-itertools==4.1.0     # via pytest
-moto==1.3.1
+moto==1.3.3
 nose==1.3.7
-pbr==4.0.0                # via mock
+pbr==4.0.2                # via mock
 pluggy==0.6.0             # via pytest
 py==1.5.3                 # via pytest
 pyaml==17.12.1            # via moto
 pycparser==2.18           # via cffi
 pyflakes==1.6.0
 pytest-flask==0.10.0
-pytest-mock==1.7.1
+pytest-mock==1.9.0
 pytest==3.5.0
 python-dateutil==2.6.1    # via botocore, faker, freezegun, moto
-pytz==2018.3              # via moto
+pytz==2018.4              # via moto
 pyyaml==3.12              # via pyaml
 requests-mock==1.4.0
-requests==2.18.4          # via aws-xray-sdk, docker, moto, requests-mock
+requests==2.18.4          # via aws-xray-sdk, docker, moto, requests-mock, responses
+responses==0.9.0          # via moto
 s3transfer==0.1.13        # via boto3
-six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, websocket-client
+six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client
 text-unidecode==1.2       # via faker
 urllib3==1.22             # via requests
 websocket-client==0.47.0  # via docker

requirements.in

@@ -1,3 +1,5 @@
+# Run `make up-reqs` to update pinned dependencies in requirement text files
+
 acme
 alembic-autogenerate-enums
 arrow
@@ -11,6 +13,7 @@ Flask-RESTful==0.3.6
 Flask-Script==2.0.6
 Flask-SQLAlchemy
 Flask==0.12
+Flask-Cors
 future
 gunicorn
 inflection

requirements.txt

@@ -2,9 +2,9 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file requirements.txt requirements.in
+#    pip-compile --no-index --output-file requirements.txt requirements.in
 #
-acme==0.22.2
+acme==0.23.0
 alembic-autogenerate-enums==0.0.2
 alembic==0.9.9            # via flask-migrate
 aniso8601==3.0.0          # via flask-restful
@@ -12,13 +12,14 @@ arrow==0.12.1
 asn1crypto==0.24.0        # via cryptography
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.6.19
+boto3==1.7.6
-botocore==1.9.19          # via boto3, s3transfer
+botocore==1.10.6          # via boto3, s3transfer
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 click==6.7                # via flask
 cryptography==2.2.2
 docutils==0.14            # via botocore
 flask-bcrypt==0.7.1
+flask-cors==3.0.3
 flask-mail==0.9.1
 flask-migrate==2.1.1
 flask-principal==0.4.0
@@ -33,7 +34,7 @@ inflection==0.3.1
 itsdangerous==0.24        # via flask
 jinja2==2.10
 jmespath==0.9.3           # via boto3, botocore
-josepy==1.0.1             # via acme
+josepy==1.1.0             # via acme
 lockfile==0.12.2
 mako==1.0.7               # via alembic
 markupsafe==1.0           # via jinja2, mako
@@ -42,7 +43,7 @@ marshmallow==2.15.0
 mock==2.0.0               # via acme
 ndg-httpsclient==0.4.4
 paramiko==2.4.1
-pbr==4.0.0                # via mock
+pbr==4.0.2                # via mock
 pem==17.1.0
 psycopg2==2.7.4
 pyasn1-modules==0.2.1     # via python-ldap
@@ -52,17 +53,17 @@ pyjwt==1.6.1
 pynacl==1.2.1             # via paramiko
 pyopenssl==17.2.0
 pyrfc3339==1.0            # via acme
-python-dateutil==2.6.1    # via alembic, arrow, botocore
+python-dateutil==2.7.2    # via alembic, arrow, botocore
 python-editor==1.0.3      # via alembic
 python-ldap==3.0.0
-pytz==2018.3              # via acme, flask-restful, pyrfc3339
+pytz==2018.4              # via acme, flask-restful, pyrfc3339
-raven[flask]==6.6.0
+raven[flask]==6.7.0
 requests[security]==2.11.1
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.11.0
-sqlalchemy-utils==0.33.1
+sqlalchemy-utils==0.33.2
-sqlalchemy==1.2.5         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.2.7         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
 werkzeug==0.14.1          # via flask
 xmltodict==0.11.0

setup.py

@@ -16,14 +16,20 @@ import datetime
 
 from distutils import log
 from distutils.core import Command
-from pip.download import PipSession
-from pip.req import parse_requirements
 from setuptools.command.develop import develop
 from setuptools.command.install import install
 from setuptools.command.sdist import sdist
 from setuptools import setup, find_packages
 from subprocess import check_output
 
+import pip
+if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0):
+    from pip._internal.download import PipSession
+    from pip._internal.req import parse_requirements
+else:
+    from pip.download import PipSession
+    from pip.req import parse_requirements
+
 ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__)))
 
 # When executing the setup.py, we need to be able to import ourselves, this