diff --git a/Makefile b/Makefile index d3d3ecc2..f740faab 100644 --- a/Makefile +++ b/Makefile @@ -109,12 +109,15 @@ ifndef VIRTUAL_ENV $(error Please activate virtualenv first) endif @echo "--> Updating Python requirements" + pip install --upgrade pip pip install --upgrade pip-tools - pip-compile --output-file requirements-docs.txt requirements-docs.in -U - pip-compile --output-file requirements-dev.txt requirements-dev.in -U - pip-compile --output-file requirements-tests.txt requirements-tests.in -U - pip-compile --output-file requirements.txt requirements.in -U + pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index + pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index + pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index + pip-compile --output-file requirements.txt requirements.in -U --no-index @echo "--> Done updating Python requirements" + @echo "--> Removing python-ldap from requirements-docs.txt" + grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt @echo "--> Installing new dependencies" pip install -e . @echo "--> Done installing new dependencies" diff --git a/README.rst b/README.rst index 9582f756..d42bc810 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ Lemur :alt: Join the chat at https://gitter.im/Netflix/lemur :target: https://gitter.im/Netflix/lemur?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -.. image:: https://readthedocs.io/projects/lemur/badge/?version=latest +.. image:: https://readthedocs.org/projects/lemur/badge/?version=latest :target: https://lemur.readthedocs.io :alt: Latest Docs @@ -14,6 +14,10 @@ Lemur .. image:: https://travis-ci.org/Netflix/lemur.svg :target: https://travis-ci.org/Netflix/lemur +.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=master + :target: https://coveralls.io/github/Netflix/lemur?branch=master + + Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults. diff --git a/docs/administration.rst b/docs/administration.rst index 3c0ca457..aab7cd58 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -65,6 +65,36 @@ Basic Configuration SQLALCHEMY_DATABASE_URI = 'postgresql://:@:5432/lemur' +.. data:: SQLALCHEMY_POOL_SIZE +:noindex: + + The default connection pool size is 5 for sqlalchemy managed connections. Depending on the number of Lemur instances, + please specify per instance connection pool size. Below is an example to set connection pool size to 10. + + :: + + SQLALCHEMY_POOL_SIZE = 10 + + + .. warning:: +This is an optional setting but important to review and set for optimal database connection usage and for overall database performance. + +.. data:: SQLALCHEMY_MAX_OVERFLOW +:noindex: + + This setting allows to create connections in addition to specified number of connections in pool size. By default, sqlalchemy + allows 10 connections to create in addition to the pool size. This is also an optional setting. If `SQLALCHEMY_POOL_SIZE` and + `SQLALCHEMY_MAX_OVERFLOW` are not speficied then each Lemur instance may create maximum of 15 connections. + + :: + + SQLALCHECK_MAX_OVERFLOW = 0 + + + .. note:: +Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size. + + .. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION :noindex: @@ -1156,6 +1186,31 @@ Digicert https://github.com/opendns/lemur-digicert +InfluxDB +-------- + +:Authors: + Titouan Christophe +:Type: + Metric +:Description: + Sends key metrics to InfluxDB +:Links: + https://github.com/titouanc/lemur-influxdb + +Hashicorp Vault +--------------- + +:Authors: + Ron Cohen +:Type: + Issuer +:Description: + Adds support for basic Vault PKI secret backend. +:Links: + https://github.com/RcRonco/lemur_vault + + Have an extension that should be listed here? Submit a `pull request `_ and we'll get it added. diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 8b6da224..4c46566a 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -48,7 +48,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi * pip * virtualenv (ideally virtualenvwrapper) * node.js (for npm and building css/javascript) -* (Optional) PostgreSQL ++* `PostgreSQL `_ Once you've got all that, the rest is simple: @@ -77,6 +77,7 @@ Create a default Lemur configuration just as if this were a production instance: :: + lemur create_config lemur init You'll likely want to make some changes to the default configuration (we recommend developing against Postgres, for example). Once done, migrate your database using the following command: diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 5cb8b8b6..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,36 +0,0 @@ -CloudFlare==1.7.5 -Flask==0.12 -Flask-RESTful==0.3.6 -Flask-SQLAlchemy==2.1 -Flask-Script==2.0.5 -Flask-Migrate==2.1.1 -Flask-Bcrypt==0.7.1 -Flask-Principal==0.4.0 -Flask-Mail==0.9.1 -SQLAlchemy-Utils==0.32.14 -requests==2.11.1 -ndg-httpsclient==0.4.2 -psycopg2==2.7.3 -arrow==0.10.0 -six==1.10.0 -marshmallow-sqlalchemy==0.13.1 -gunicorn==19.7.1 -marshmallow==2.13.6 -cryptography==1.9 -xmltodict==0.11.0 -pyjwt==1.5.2 -lockfile==0.12.2 -inflection==0.3.1 -future==0.16.0 -boto3==1.4.6 -acme==0.18.1 -retrying==1.3.3 -tabulate==0.7.7 -pem==17.1.0 -raven[flask]==6.1.0 -jinja2==2.9.6 -# pyldap==2.4.37 # cannot be installed on rtd - required by ldap auth provider -paramiko==2.4.1 # required for the SFTP destination plugin -sphinx -sphinxcontrib-httpdomain -sphinx-rtd-theme diff --git a/lemur/__init__.py b/lemur/__init__.py index 6f29733c..c3661f4e 100644 --- a/lemur/__init__.py +++ b/lemur/__init__.py @@ -8,7 +8,8 @@ """ -from __future__ import absolute_import, division, print_function +import time +from flask import g, request from lemur import factory from lemur.extensions import metrics @@ -73,17 +74,6 @@ def configure_hook(app): """ from flask import jsonify from werkzeug.exceptions import HTTPException - from lemur.decorators import crossdomain - if app.config.get('CORS'): - @app.after_request - @crossdomain(origin=u"http://localhost:3000", methods=['PUT', 'HEAD', 'GET', 'POST', 'OPTIONS', 'DELETE']) - def after(response): - return response - - @app.after_request - def log_status(response): - metrics.send('status_code_{}'.format(response.status_code), 'counter', 1) - return response @app.errorhandler(Exception) def handle_error(e): @@ -93,3 +83,29 @@ def configure_hook(app): app.logger.exception(e) return jsonify(error=str(e)), code + + @app.before_request + def before_request(): + g.request_start_time = time.time() + + @app.after_request + def after_request(response): + # Return early if we don't have the start time + if not hasattr(g, 'request_start_time'): + return response + + # Get elapsed time in milliseconds + elapsed = time.time() - g.request_start_time + elapsed = int(round(1000 * elapsed)) + + # Collect request/response tags + tags = { + 'endpoint': request.endpoint, + 'request_method': request.method.lower(), + 'status_code': response.status_code + } + + # Record our response time metric + metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags) + metrics.send('status_code_{}'.format(response.status_code), 'counter', 1) + return response diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 453677ec..0b475e0b 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -9,6 +9,7 @@ """ from lemur import database +from lemur.common.utils import truthiness from lemur.extensions import metrics from lemur.authorities.models import Authority from lemur.roles import service as role_service @@ -170,8 +171,8 @@ def render(args): if filt: terms = filt.split(';') - if 'active' in filt: # this is really weird but strcmp seems to not work here?? - query = query.filter(Authority.active == terms[1]) + if 'active' in filt: + query = query.filter(Authority.active == truthiness(terms[1])) else: query = database.filter(query, Authority, terms) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 62f1ab6c..ce90b4f0 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -8,7 +8,7 @@ import arrow from flask import current_app -from sqlalchemy import func, or_, not_, cast, Boolean, Integer +from sqlalchemy import func, or_, not_, cast, Integer from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -17,7 +17,7 @@ from cryptography.hazmat.primitives import hashes, serialization from lemur import database from lemur.extensions import metrics, signals from lemur.plugins.base import plugins -from lemur.common.utils import generate_private_key +from lemur.common.utils import generate_private_key, truthiness from lemur.roles.models import Role from lemur.domains.models import Domain @@ -319,9 +319,9 @@ def render(args): elif 'destination' in terms: query = query.filter(Certificate.destinations.any(Destination.id == terms[1])) elif 'notify' in filt: - query = query.filter(Certificate.notify == cast(terms[1], Boolean)) + query = query.filter(Certificate.notify == truthiness(terms[1])) elif 'active' in filt: - query = query.filter(Certificate.active == terms[1]) + query = query.filter(Certificate.active == truthiness(terms[1])) elif 'cn' in terms: query = query.filter( or_( diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 80355950..59f96522 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -300,12 +300,14 @@ class CertificatesUpload(AuthenticatedResource): { "owner": "joe@example.com", - "publicCert": "-----BEGIN CERTIFICATE-----...", - "intermediateCert": "-----BEGIN CERTIFICATE-----...", + "body": "-----BEGIN CERTIFICATE-----...", + "chain": "-----BEGIN CERTIFICATE-----...", "privateKey": "-----BEGIN RSA PRIVATE KEY-----..." "destinations": [], "notifications": [], "replacements": [], + "roles": [], + "notify": true, "name": "cert1" } diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 621a2c39..02f55340 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -14,10 +14,11 @@ from sqlalchemy import and_, func from cryptography import x509 from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric import rsa, ec from flask_restful.reqparse import RequestParser +from lemur.constants import CERTIFICATE_KEY_TYPES from lemur.exceptions import InvalidConfiguration paginated_parser = RequestParser() @@ -78,17 +79,43 @@ def generate_private_key(key_type): """ Generates a new private key based on key_type. - Valid key types: RSA2048, RSA4096 + Valid key types: RSA2048, RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', + 'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1', + 'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', + 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2' :param key_type: :return: """ - valid_key_types = ['RSA2048', 'RSA4096'] - if key_type not in valid_key_types: + _CURVE_TYPES = { + "ECCPRIME192V1": ec.SECP192R1(), + "ECCPRIME256V1": ec.SECP256R1(), + + "ECCSECP192R1": ec.SECP192R1(), + "ECCSECP224R1": ec.SECP224R1(), + "ECCSECP256R1": ec.SECP256R1(), + "ECCSECP384R1": ec.SECP384R1(), + "ECCSECP521R1": ec.SECP521R1(), + "ECCSECP256K1": ec.SECP256K1(), + + "ECCSECT163K1": ec.SECT163K1(), + "ECCSECT233K1": ec.SECT233K1(), + "ECCSECT283K1": ec.SECT283K1(), + "ECCSECT409K1": ec.SECT409K1(), + "ECCSECT571K1": ec.SECT571K1(), + + "ECCSECT163R2": ec.SECT163R2(), + "ECCSECT233R1": ec.SECT233R1(), + "ECCSECT283R1": ec.SECT283R1(), + "ECCSECT409R1": ec.SECT409R1(), + "ECCSECT571R2": ec.SECT571R1(), + } + + if key_type not in CERTIFICATE_KEY_TYPES: raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format( key_type=key_type, - choices=",".join(valid_key_types) + choices=",".join(CERTIFICATE_KEY_TYPES) )) if 'RSA' in key_type: @@ -98,6 +125,11 @@ def generate_private_key(key_type): key_size=key_size, backend=default_backend() ) + elif 'ECC' in key_type: + return ec.generate_private_key( + curve=_CURVE_TYPES[key_type], + backend=default_backend() + ) def is_weekend(date): @@ -175,3 +207,9 @@ def windowed_query(q, column, windowsize): column, windowsize): for row in q.filter(whereclause).order_by(column): yield row + + +def truthiness(s): + """If input string resembles something truthy then return True, else False.""" + + return s.lower() in ('true', 'yes', 'on', 't', '1') diff --git a/lemur/constants.py b/lemur/constants.py index 04824c12..0ee9bc40 100644 --- a/lemur/constants.py +++ b/lemur/constants.py @@ -9,3 +9,26 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}" SUCCESS_METRIC_STATUS = 'success' FAILURE_METRIC_STATUS = 'failure' + +CERTIFICATE_KEY_TYPES = [ + 'RSA2048', + 'RSA4096', + 'ECCPRIME192V1', + 'ECCPRIME256V1', + 'ECCSECP192R1', + 'ECCSECP224R1', + 'ECCSECP256R1', + 'ECCSECP384R1', + 'ECCSECP521R1', + 'ECCSECP256K1', + 'ECCSECT163K1', + 'ECCSECT233K1', + 'ECCSECT283K1', + 'ECCSECT409K1', + 'ECCSECT571K1', + 'ECCSECT163R2', + 'ECCSECT233R1', + 'ECCSECT283R1', + 'ECCSECT409R1', + 'ECCSECT571R2' +] diff --git a/lemur/decorators.py b/lemur/decorators.py deleted file mode 100644 index 7b93ec70..00000000 --- a/lemur/decorators.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -.. module: lemur.decorators - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. -""" -from builtins import str - -from datetime import timedelta -from flask import make_response, request, current_app - -from functools import update_wrapper - - -# this is only used for dev -def crossdomain(origin=None, methods=None, headers=None, - max_age=21600, attach_to_all=True, - automatic_options=True): # pragma: no cover - if methods is not None: - methods = ', '.join(sorted(x.upper() for x in methods)) - - if headers is not None and not isinstance(headers, str): - headers = ', '.join(x.upper() for x in headers) - - if not isinstance(origin, str): - origin = ', '.join(origin) - - if isinstance(max_age, timedelta): - max_age = max_age.total_seconds() - - def get_methods(): - if methods is not None: - return methods - - options_resp = current_app.make_default_options_response() - return options_resp.headers['allow'] - - def decorator(f): - def wrapped_function(*args, **kwargs): - if automatic_options and request.method == 'OPTIONS': - resp = current_app.make_default_options_response() - else: - resp = make_response(f(*args, **kwargs)) - if not attach_to_all and request.method != 'OPTIONS': - return resp - - h = resp.headers - h['Access-Control-Allow-Origin'] = origin - h['Access-Control-Allow-Methods'] = get_methods() - h['Access-Control-Max-Age'] = str(max_age) - h['Access-Control-Allow-Headers'] = "Origin, X-Requested-With, Content-Type, Accept, Authorization " - h['Access-Control-Allow-Credentials'] = 'true' - return resp - - f.provide_automatic_options = False - return update_wrapper(wrapped_function, f) - return decorator diff --git a/lemur/endpoints/service.py b/lemur/endpoints/service.py index 09c7c418..55bacde2 100644 --- a/lemur/endpoints/service.py +++ b/lemur/endpoints/service.py @@ -13,6 +13,7 @@ import arrow from sqlalchemy import func from lemur import database +from lemur.common.utils import truthiness from lemur.endpoints.models import Endpoint, Policy, Cipher from lemur.extensions import metrics @@ -142,7 +143,7 @@ def render(args): if filt: terms = filt.split(';') if 'active' in filt: # this is really weird but strcmp seems to not work here?? - query = query.filter(Endpoint.active == terms[1]) + query = query.filter(Endpoint.active == truthiness(terms[1])) elif 'port' in filt: if terms[1] != 'null': # ng-table adds 'null' if a number is removed query = query.filter(Endpoint.port == terms[1]) diff --git a/lemur/extensions.py b/lemur/extensions.py index b5a18569..76abcab1 100644 --- a/lemur/extensions.py +++ b/lemur/extensions.py @@ -26,3 +26,6 @@ sentry = Sentry() from blinker import Namespace signals = Namespace() + +from flask_cors import CORS +cors = CORS() diff --git a/lemur/factory.py b/lemur/factory.py index 107fb70f..97f7f6ca 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -21,7 +21,7 @@ from flask import Flask from lemur.certificates.hooks import activate_debug_dump from lemur.common.health import mod as health -from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry +from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry, cors DEFAULT_BLUEPRINTS = ( @@ -125,6 +125,10 @@ def configure_extensions(app): metrics.init_app(app) sentry.init_app(app) + if app.config['CORS']: + app.config['CORS_HEADERS'] = 'Content-Type' + cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True) + def configure_blueprints(app, blueprints): """ diff --git a/lemur/manage.py b/lemur/manage.py index 8f45cfb4..919f70e4 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -208,16 +208,16 @@ class InitializeApp(Command): if operator_role: sys.stdout.write("[-] Operator role already created, skipping...!\n") else: - # we create an admin role + # we create an operator role operator_role = role_service.create('operator', description='This is the Lemur operator role.') sys.stdout.write("[+] Created 'operator' role\n") read_only_role = role_service.get_by_name('read-only') if read_only_role: - sys.stdout.write("[-] Operator role already created, skipping...!\n") + sys.stdout.write("[-] Read only role already created, skipping...!\n") else: - # we create an admin role + # we create an read only role read_only_role = role_service.create('read-only', description='This is the Lemur read only role.') sys.stdout.write("[+] Created 'read-only' role\n") @@ -237,9 +237,6 @@ class InitializeApp(Command): else: sys.stdout.write("[-] Default user has already been created, skipping...!\n") - sys.stdout.write("[+] Creating expiration email notifications!\n") - sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format("LEMUR_SECURITY_TEAM_EMAIL")) - intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", []) sys.stdout.write( "[!] Creating {num} notifications for {intervals} days as specified by LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS\n".format( @@ -249,14 +246,21 @@ class InitializeApp(Command): ) recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL') + sys.stdout.write("[+] Creating expiration email notifications!\n") + sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format(recipients)) notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients) - days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30) - sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format( - days=days - )) + _DEFAULT_ROTATION_INTERVAL = 'default' + default_rotation_interval = policy_service.get_by_name(_DEFAULT_ROTATION_INTERVAL) + + if default_rotation_interval: + sys.stdout.write("[-] Default rotation interval policy already created, skipping...!\n") + else: + days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30) + sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format( + days=days)) + policy_service.create(days=days, name=_DEFAULT_ROTATION_INTERVAL) - policy_service.create(days=days, name='default') sys.stdout.write("[/] Done!\n") diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index efbfd512..e26ff47f 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -12,6 +12,7 @@ from flask import current_app from lemur import database from lemur.certificates.models import Certificate +from lemur.common.utils import truthiness from lemur.notifications.models import Notification @@ -169,10 +170,8 @@ def render(args): if filt: terms = filt.split(';') - if terms[0] == 'active' and terms[1] == 'false': - query = query.filter(Notification.active == False) # noqa - elif terms[0] == 'active' and terms[1] == 'true': - query = query.filter(Notification.active == True) # noqa + if terms[0] == 'active': + query = query.filter(Notification.active == truthiness(terms[1])) else: query = database.filter(query, Notification, terms) diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index fed2ddd4..9046e0c8 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -5,9 +5,10 @@ """ import arrow -from sqlalchemy import or_, cast, Boolean, Integer +from sqlalchemy import or_, cast, Integer from lemur import database +from lemur.common.utils import truthiness from lemur.plugins.base import plugins from lemur.roles.models import Role @@ -181,9 +182,9 @@ def render(args): elif 'destination' in terms: query = query.filter(PendingCertificate.destinations.any(Destination.id == terms[1])) elif 'notify' in filt: - query = query.filter(PendingCertificate.notify == cast(terms[1], Boolean)) + query = query.filter(PendingCertificate.notify == truthiness(terms[1])) elif 'active' in filt: - query = query.filter(PendingCertificate.active == terms[1]) + query = query.filter(PendingCertificate.active == truthiness(terms[1])) elif 'cn' in terms: query = query.filter( or_( diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 00fe9519..bfe41b4c 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -157,7 +157,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get('signing_algorithm')), "validity": { - "valid_to": options['validity_end'].format('YYYY-MM-DD') + "valid_to": options['validity_end'].format('YYYY-MM-DDTHH:MM') + 'Z' }, "organization": { "name": options['organization'], @@ -491,6 +491,11 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): self.session.headers.pop('Accept') end_entity = pem.parse(certificate_pem)[0] + + if 'ECC' in issuer_options['key_type']: + return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_ECC_CIS_INTERMEDIATE'), data['id'] + + # By default return RSA return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_CIS_INTERMEDIATE'), data['id'] def revoke_certificate(self, certificate, comments): diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index dca24892..5f14f04b 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -103,7 +103,7 @@ def test_map_cis_fields(app): 'signature_hash': 'sha256', 'organization': {'name': 'Example, Inc.', 'units': ['Example Org']}, 'validity': { - 'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DD') + 'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DDTHH:MM') + 'Z' }, 'profile_name': None } @@ -132,7 +132,7 @@ def test_map_cis_fields(app): 'signature_hash': 'sha256', 'organization': {'name': 'Example, Inc.', 'units': ['Example Org']}, 'validity': { - 'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DD') + 'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DDTHH:MM') + 'Z' }, 'profile_name': None } diff --git a/lemur/plugins/lemur_statsd/docs/requirements.txt b/lemur/plugins/lemur_statsd/docs/requirements.txt new file mode 100644 index 00000000..73a2e5b6 --- /dev/null +++ b/lemur/plugins/lemur_statsd/docs/requirements.txt @@ -0,0 +1 @@ +datadog==0.14.0 diff --git a/lemur/plugins/lemur_statsd/lemur_statsd/__init__.py b/lemur/plugins/lemur_statsd/lemur_statsd/__init__.py new file mode 100644 index 00000000..3a751848 --- /dev/null +++ b/lemur/plugins/lemur_statsd/lemur_statsd/__init__.py @@ -0,0 +1,4 @@ +try: + VERSION = __import__('pkg_resources').get_distribution(__name__).version +except Exception as e: + VERSION = 'Unknown' diff --git a/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py new file mode 100644 index 00000000..a6a87c66 --- /dev/null +++ b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py @@ -0,0 +1,45 @@ +import lemur_statsd as plug + +from flask import current_app +from lemur.plugins.bases.metric import MetricPlugin +from datadog import DogStatsd + + +class StatsdMetricPlugin(MetricPlugin): + title = 'Statsd' + slug = 'statsd-metrics' + description = 'Adds support for sending metrics to Statsd' + version = plug.VERSION + + def __init__(self): + host = current_app.config.get('STATSD_HOST') + port = current_app.config.get('STATSD_PORT') + prefix = current_app.config.get('STATSD_PREFIX') + + self.statsd = DogStatsd(host=host, port=port, namespace=prefix) + + def submit(self, metric_name, metric_type, metric_value, metric_tags=None, options=None): + valid_types = ['COUNTER', 'GAUGE', 'TIMER'] + tags = [] + + if metric_type.upper() not in valid_types: + raise Exception( + "Invalid Metric Type for Statsd, '{metric}' choose from: {options}".format( + metric=metric_type, options=','.join(valid_types) + ) + ) + + if metric_tags: + if not isinstance(metric_tags, dict): + raise Exception("Invalid Metric Tags for Statsd: Tags must be in dict format") + else: + tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items()) + + if metric_type.upper() == 'COUNTER': + self.statsd.increment(metric_name, metric_value, tags) + elif metric_type.upper() == 'GAUGE': + self.statsd.gauge(metric_name, metric_value, tags) + elif metric_type.upper() == 'TIMER': + self.statsd.timing(metric_name, metric_value, tags) + + return diff --git a/lemur/plugins/lemur_statsd/setup.py b/lemur/plugins/lemur_statsd/setup.py new file mode 100644 index 00000000..6c4c2dd6 --- /dev/null +++ b/lemur/plugins/lemur_statsd/setup.py @@ -0,0 +1,24 @@ +"""Basic package information""" +from __future__ import absolute_import +from setuptools import setup, find_packages + +install_requires = [ + 'lemur', + 'datadog' +] + +setup( + name='lemur_statsd', + version='1.0.0', + author='Cloudflare Security Engineering', + author_email='', + include_package_data=True, + packages=find_packages(), + zip_safe=False, + install_requires=install_requires, + entry_points={ + 'lemur.plugins': [ + 'statsd = lemur_statsd.plugin:StatsdMetricPlugin', + ] + } +) diff --git a/lemur/policies/service.py b/lemur/policies/service.py index 603fb3af..c6719a03 100644 --- a/lemur/policies/service.py +++ b/lemur/policies/service.py @@ -18,6 +18,15 @@ def get(policy_id): return database.get(RotationPolicy, policy_id) +def get_by_name(policy_name): + """ + Retrieves policy by its name. + :param policy_name: + :return: + """ + return database.get_all(RotationPolicy, policy_name, field='name').all() + + def delete(policy_id): """ Delete a rotation policy. diff --git a/lemur/static/app/angular/authorities/authority/options.tpl.html b/lemur/static/app/angular/authorities/authority/options.tpl.html index 57fc29e6..245716cb 100644 --- a/lemur/static/app/angular/authorities/authority/options.tpl.html +++ b/lemur/static/app/angular/authorities/authority/options.tpl.html @@ -20,7 +20,8 @@ Key Type
- +
diff --git a/lemur/static/app/angular/certificates/certificate/options.tpl.html b/lemur/static/app/angular/certificates/certificate/options.tpl.html index a52ee387..fb1d59a1 100644 --- a/lemur/static/app/angular/certificates/certificate/options.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/options.tpl.html @@ -32,7 +32,10 @@
diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index f2b1f7e9..a5bebe15 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -717,3 +717,11 @@ def test_certificates_upload_patch(client, token, status): def test_sensitive_sort(client): resp = client.get(api.url_for(CertificatesList) + '?sortBy=private_key&sortDir=asc', headers=VALID_ADMIN_HEADER_TOKEN) assert "'private_key' is not sortable or filterable" in resp.json['message'] + + +def test_boolean_filter(client): + resp = client.get(api.url_for(CertificatesList) + '?filter=notify;true', headers=VALID_ADMIN_HEADER_TOKEN) + assert resp.status_code == 200 + # Also don't crash with invalid input (we currently treat that as false) + resp = client.get(api.url_for(CertificatesList) + '?filter=notify;whatisthis', headers=VALID_ADMIN_HEADER_TOKEN) + assert resp.status_code == 200 diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index cc03fcd5..62d021a4 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -6,9 +6,27 @@ def test_generate_private_key(): assert generate_private_key('RSA2048') assert generate_private_key('RSA4096') + assert generate_private_key('ECCPRIME192V1') + assert generate_private_key('ECCPRIME256V1') + assert generate_private_key('ECCSECP192R1') + assert generate_private_key('ECCSECP224R1') + assert generate_private_key('ECCSECP256R1') + assert generate_private_key('ECCSECP384R1') + assert generate_private_key('ECCSECP521R1') + assert generate_private_key('ECCSECP256K1') + assert generate_private_key('ECCSECT163K1') + assert generate_private_key('ECCSECT233K1') + assert generate_private_key('ECCSECT283K1') + assert generate_private_key('ECCSECT409K1') + assert generate_private_key('ECCSECT571K1') + assert generate_private_key('ECCSECT163R2') + assert generate_private_key('ECCSECT233R1') + assert generate_private_key('ECCSECT283R1') + assert generate_private_key('ECCSECT409R1') + assert generate_private_key('ECCSECT571R2') with pytest.raises(Exception): - generate_private_key('ECC') + generate_private_key('LEMUR') def test_get_authority_key(): diff --git a/requirements-dev.in b/requirements-dev.in index 4e560267..de8b60d3 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,3 +1,5 @@ +# Run `make up-reqs` to update pinned dependencies in requirement text files + flake8>=3.2,<4.0 pre-commit invoke diff --git a/requirements-dev.txt b/requirements-dev.txt index d647716d..356db421 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,15 +2,15 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-dev.txt requirements-dev.in +# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # -aspy.yaml==1.0.0 # via pre-commit -cached-property==1.4.0 # via pre-commit -certifi==2018.1.18 # via requests +aspy.yaml==1.1.0 # via pre-commit +cached-property==1.4.2 # via pre-commit +certifi==2018.4.16 # via requests cfgv==1.0.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.8 # via pre-commit +identify==1.0.13 # via pre-commit idna==2.6 # via requests invoke==0.22.1 mccabe==0.6.1 # via flake8 @@ -23,7 +23,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.18.4 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -tqdm==4.19.9 # via twine +tqdm==4.23.0 # via twine twine==1.11.0 urllib3==1.22 # via requests virtualenv==15.2.0 # via pre-commit diff --git a/requirements-docs.in b/requirements-docs.in index 8f1c7fe9..cf598240 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -1,3 +1,7 @@ +# Note: python-ldap from requirements breaks due to readthedocs.io not having the correct header files +# The `make up-reqs` will update all requirement text files, and forcibly remove python-ldap +# from requirements-docs.txt +-r requirements.txt sphinx sphinxcontrib-httpdomain sphinx-rtd-theme \ No newline at end of file diff --git a/requirements-docs.txt b/requirements-docs.txt index 373468c1..9d388183 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,26 +2,78 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-docs.txt requirements-docs.in +# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # +acme==0.23.0 alabaster==0.7.10 # via sphinx +alembic-autogenerate-enums==0.0.2 +alembic==0.9.9 +aniso8601==3.0.0 +arrow==0.12.1 +asn1crypto==0.24.0 babel==2.5.3 # via sphinx -certifi==2018.1.18 # via requests -chardet==3.0.4 # via requests -docutils==0.14 # via sphinx -idna==2.6 # via requests +bcrypt==3.1.4 +blinker==1.4 +boto3==1.7.6 +botocore==1.10.6 +cffi==1.11.5 +click==6.7 +cryptography==2.2.2 +docutils==0.14 +flask-bcrypt==0.7.1 +flask-cors==3.0.3 +flask-mail==0.9.1 +flask-migrate==2.1.1 +flask-principal==0.4.0 +flask-restful==0.3.6 +flask-script==2.0.6 +flask-sqlalchemy==2.3.2 +flask==0.12 +future==0.16.0 +gunicorn==19.7.1 +idna==2.6 imagesize==1.0.0 # via sphinx -jinja2==2.10 # via sphinx -markupsafe==1.0 # via jinja2 +inflection==0.3.1 +itsdangerous==0.24 +jinja2==2.10 +jmespath==0.9.3 +josepy==1.1.0 +lockfile==0.12.2 +mako==1.0.7 +markupsafe==1.0 +marshmallow-sqlalchemy==0.13.2 +marshmallow==2.15.0 +mock==2.0.0 +ndg-httpsclient==0.4.4 packaging==17.1 # via sphinx +paramiko==2.4.1 +pbr==4.0.2 +pem==17.1.0 +psycopg2==2.7.4 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 +pycparser==2.18 pygments==2.2.0 # via sphinx +pyjwt==1.6.1 +pynacl==1.2.1 +pyopenssl==17.2.0 pyparsing==2.2.0 # via packaging -pytz==2018.3 # via babel -requests==2.18.4 # via sphinx -six==1.11.0 # via packaging, sphinx, sphinxcontrib-httpdomain +pyrfc3339==1.0 +python-dateutil==2.7.2 +python-editor==1.0.3 +pytz==2018.4 +raven[flask]==6.7.0 +requests[security]==2.11.1 +retrying==1.3.3 +s3transfer==0.1.13 +six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.2.4 -sphinx==1.7.2 +sphinx-rtd-theme==0.3.0 +sphinx==1.7.3 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx -urllib3==1.22 # via requests +sqlalchemy-utils==0.33.2 +sqlalchemy==1.2.7 +tabulate==0.8.2 +werkzeug==0.14.1 +xmltodict==0.11.0 diff --git a/requirements-tests.in b/requirements-tests.in index 44a5c59f..02a2b0ae 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -1,3 +1,5 @@ +# Run `make up-reqs` to update pinned dependencies in requirement text files + coverage factory-boy Faker diff --git a/requirements-tests.txt b/requirements-tests.txt index a3ca0578..0c74eacb 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,26 +2,26 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-tests.txt requirements-tests.in +# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.6.19 # via moto +boto3==1.7.6 # via moto boto==2.48.0 # via moto -botocore==1.9.19 # via boto3, moto, s3transfer -certifi==2018.1.18 # via requests +botocore==1.10.6 # via boto3, moto, s3transfer +certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==6.7 # via flask -cookies==2.2.1 # via moto +cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto docker-pycreds==0.2.2 # via docker -docker==3.1.4 # via moto +docker==3.2.1 # via moto docutils==0.14 # via botocore factory-boy==2.10.0 -faker==0.8.12 +faker==0.8.13 flask==0.12.2 # via pytest-flask freezegun==0.3.10 idna==2.6 # via cryptography, requests @@ -33,24 +33,25 @@ jsonpickle==0.9.6 # via aws-xray-sdk markupsafe==1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.1.0 # via pytest -moto==1.3.1 +moto==1.3.3 nose==1.3.7 -pbr==4.0.0 # via mock +pbr==4.0.2 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto pycparser==2.18 # via cffi pyflakes==1.6.0 pytest-flask==0.10.0 -pytest-mock==1.7.1 +pytest-mock==1.9.0 pytest==3.5.0 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto -pytz==2018.3 # via moto +pytz==2018.4 # via moto pyyaml==3.12 # via pyaml requests-mock==1.4.0 -requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock +requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses +responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 -six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, websocket-client +six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.22 # via requests websocket-client==0.47.0 # via docker diff --git a/requirements.in b/requirements.in index 4dad196d..a5fe2317 100644 --- a/requirements.in +++ b/requirements.in @@ -1,3 +1,5 @@ +# Run `make up-reqs` to update pinned dependencies in requirement text files + acme alembic-autogenerate-enums arrow @@ -11,6 +13,7 @@ Flask-RESTful==0.3.6 Flask-Script==2.0.6 Flask-SQLAlchemy Flask==0.12 +Flask-Cors future gunicorn inflection diff --git a/requirements.txt b/requirements.txt index 129972ff..23f1f28f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements.txt requirements.in +# pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.22.2 +acme==0.23.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -12,13 +12,14 @@ arrow==0.12.1 asn1crypto==0.24.0 # via cryptography bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.6.19 -botocore==1.9.19 # via boto3, s3transfer +boto3==1.7.6 +botocore==1.10.6 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask cryptography==2.2.2 docutils==0.14 # via botocore flask-bcrypt==0.7.1 +flask-cors==3.0.3 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -33,7 +34,7 @@ inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 jmespath==0.9.3 # via boto3, botocore -josepy==1.0.1 # via acme +josepy==1.1.0 # via acme lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.0 # via jinja2, mako @@ -42,7 +43,7 @@ marshmallow==2.15.0 mock==2.0.0 # via acme ndg-httpsclient==0.4.4 paramiko==2.4.1 -pbr==4.0.0 # via mock +pbr==4.0.2 # via mock pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 # via python-ldap @@ -52,17 +53,17 @@ pyjwt==1.6.1 pynacl==1.2.1 # via paramiko pyopenssl==17.2.0 pyrfc3339==1.0 # via acme -python-dateutil==2.6.1 # via alembic, arrow, botocore +python-dateutil==2.7.2 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.0.0 -pytz==2018.3 # via acme, flask-restful, pyrfc3339 -raven[flask]==6.6.0 +pytz==2018.4 # via acme, flask-restful, pyrfc3339 +raven[flask]==6.7.0 requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.1 -sqlalchemy==1.2.5 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.33.2 +sqlalchemy==1.2.7 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 werkzeug==0.14.1 # via flask xmltodict==0.11.0 diff --git a/setup.py b/setup.py index 9f5e2990..3cb731d2 100644 --- a/setup.py +++ b/setup.py @@ -16,14 +16,20 @@ import datetime from distutils import log from distutils.core import Command -from pip.download import PipSession -from pip.req import parse_requirements from setuptools.command.develop import develop from setuptools.command.install import install from setuptools.command.sdist import sdist from setuptools import setup, find_packages from subprocess import check_output +import pip +if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0): + from pip._internal.download import PipSession + from pip._internal.req import parse_requirements +else: + from pip.download import PipSession + from pip.req import parse_requirements + ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__))) # When executing the setup.py, we need to be able to import ourselves, this