From 2dc6478c34de9270caed57ce3a4c28921df670d8 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 9 Apr 2018 07:46:08 -0700 Subject: [PATCH 01/27] update requirements --- requirements-dev.txt | 6 +++--- requirements-docs.txt | 2 +- requirements-tests.txt | 10 +++++----- requirements.txt | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d647716d..1b11eb2b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,12 +5,12 @@ # pip-compile --output-file requirements-dev.txt requirements-dev.in # aspy.yaml==1.0.0 # via pre-commit -cached-property==1.4.0 # via pre-commit +cached-property==1.4.2 # via pre-commit certifi==2018.1.18 # via requests cfgv==1.0.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.8 # via pre-commit +identify==1.0.9 # via pre-commit idna==2.6 # via requests invoke==0.22.1 mccabe==0.6.1 # via flake8 @@ -23,7 +23,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.18.4 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -tqdm==4.19.9 # via twine +tqdm==4.21.0 # via twine twine==1.11.0 urllib3==1.22 # via requests virtualenv==15.2.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 373468c1..35fce72b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -20,7 +20,7 @@ pytz==2018.3 # via babel requests==2.18.4 # via sphinx six==1.11.0 # via packaging, sphinx, sphinxcontrib-httpdomain snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.2.4 +sphinx-rtd-theme==0.3.0 sphinx==1.7.2 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index a3ca0578..983aeb0e 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,9 +7,9 @@ asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.6.19 # via moto +boto3==1.7.2 # via moto boto==2.48.0 # via moto -botocore==1.9.19 # via boto3, moto, s3transfer +botocore==1.10.2 # via boto3, moto, s3transfer certifi==2018.1.18 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -18,7 +18,7 @@ cookies==2.2.1 # via moto coverage==4.5.1 cryptography==2.2.2 # via moto docker-pycreds==0.2.2 # via docker -docker==3.1.4 # via moto +docker==3.2.1 # via moto docutils==0.14 # via botocore factory-boy==2.10.0 faker==0.8.12 @@ -35,14 +35,14 @@ mock==2.0.0 # via moto more-itertools==4.1.0 # via pytest moto==1.3.1 nose==1.3.7 -pbr==4.0.0 # via mock +pbr==4.0.1 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto pycparser==2.18 # via cffi pyflakes==1.6.0 pytest-flask==0.10.0 -pytest-mock==1.7.1 +pytest-mock==1.8.0 pytest==3.5.0 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.3 # via moto diff --git a/requirements.txt b/requirements.txt index 129972ff..dafb2bdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file requirements.txt requirements.in # -acme==0.22.2 +acme==0.23.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -12,8 +12,8 @@ arrow==0.12.1 asn1crypto==0.24.0 # via cryptography bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.6.19 -botocore==1.9.19 # via boto3, s3transfer +boto3==1.7.2 +botocore==1.10.2 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask cryptography==2.2.2 @@ -42,7 +42,7 @@ marshmallow==2.15.0 mock==2.0.0 # via acme ndg-httpsclient==0.4.4 paramiko==2.4.1 -pbr==4.0.0 # via mock +pbr==4.0.1 # via mock pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 # via python-ldap @@ -61,8 +61,8 @@ requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.1 -sqlalchemy==1.2.5 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.33.2 +sqlalchemy==1.2.6 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 werkzeug==0.14.1 # via flask xmltodict==0.11.0 From 8e2b2123f1dd80aaba978eb671a8650956e3272b Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Mon, 2 Apr 2018 18:33:51 +0300 Subject: [PATCH 02/27] Fix filtering on boolean columns, broken with SQLAlchemy 1.2 upgrade SQLAlchemy 1.2 does not allow comparing string values to boolean columns. This caused errors like: sqlalchemy.exc.StatementError: (builtins.TypeError) Not a boolean value: 'true' For more details see http://docs.sqlalchemy.org/en/latest/changelog/migration_12.html#boolean-datatype-now-enforces-strict-true-false-none-values --- lemur/authorities/service.py | 5 +++-- lemur/certificates/service.py | 8 ++++---- lemur/common/utils.py | 6 ++++++ lemur/endpoints/service.py | 3 ++- lemur/notifications/service.py | 7 +++---- lemur/pending_certificates/service.py | 7 ++++--- lemur/tests/test_certificates.py | 8 ++++++++ 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 453677ec..0b475e0b 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -9,6 +9,7 @@ """ from lemur import database +from lemur.common.utils import truthiness from lemur.extensions import metrics from lemur.authorities.models import Authority from lemur.roles import service as role_service @@ -170,8 +171,8 @@ def render(args): if filt: terms = filt.split(';') - if 'active' in filt: # this is really weird but strcmp seems to not work here?? - query = query.filter(Authority.active == terms[1]) + if 'active' in filt: + query = query.filter(Authority.active == truthiness(terms[1])) else: query = database.filter(query, Authority, terms) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 62f1ab6c..ce90b4f0 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -8,7 +8,7 @@ import arrow from flask import current_app -from sqlalchemy import func, or_, not_, cast, Boolean, Integer +from sqlalchemy import func, or_, not_, cast, Integer from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -17,7 +17,7 @@ from cryptography.hazmat.primitives import hashes, serialization from lemur import database from lemur.extensions import metrics, signals from lemur.plugins.base import plugins -from lemur.common.utils import generate_private_key +from lemur.common.utils import generate_private_key, truthiness from lemur.roles.models import Role from lemur.domains.models import Domain @@ -319,9 +319,9 @@ def render(args): elif 'destination' in terms: query = query.filter(Certificate.destinations.any(Destination.id == terms[1])) elif 'notify' in filt: - query = query.filter(Certificate.notify == cast(terms[1], Boolean)) + query = query.filter(Certificate.notify == truthiness(terms[1])) elif 'active' in filt: - query = query.filter(Certificate.active == terms[1]) + query = query.filter(Certificate.active == truthiness(terms[1])) elif 'cn' in terms: query = query.filter( or_( diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 621a2c39..4db31a4e 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -175,3 +175,9 @@ def windowed_query(q, column, windowsize): column, windowsize): for row in q.filter(whereclause).order_by(column): yield row + + +def truthiness(s): + """If input string resembles something truthy then return True, else False.""" + + return s.lower() in ('true', 'yes', 'on', 't', '1') diff --git a/lemur/endpoints/service.py b/lemur/endpoints/service.py index 09c7c418..55bacde2 100644 --- a/lemur/endpoints/service.py +++ b/lemur/endpoints/service.py @@ -13,6 +13,7 @@ import arrow from sqlalchemy import func from lemur import database +from lemur.common.utils import truthiness from lemur.endpoints.models import Endpoint, Policy, Cipher from lemur.extensions import metrics @@ -142,7 +143,7 @@ def render(args): if filt: terms = filt.split(';') if 'active' in filt: # this is really weird but strcmp seems to not work here?? - query = query.filter(Endpoint.active == terms[1]) + query = query.filter(Endpoint.active == truthiness(terms[1])) elif 'port' in filt: if terms[1] != 'null': # ng-table adds 'null' if a number is removed query = query.filter(Endpoint.port == terms[1]) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index efbfd512..e26ff47f 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -12,6 +12,7 @@ from flask import current_app from lemur import database from lemur.certificates.models import Certificate +from lemur.common.utils import truthiness from lemur.notifications.models import Notification @@ -169,10 +170,8 @@ def render(args): if filt: terms = filt.split(';') - if terms[0] == 'active' and terms[1] == 'false': - query = query.filter(Notification.active == False) # noqa - elif terms[0] == 'active' and terms[1] == 'true': - query = query.filter(Notification.active == True) # noqa + if terms[0] == 'active': + query = query.filter(Notification.active == truthiness(terms[1])) else: query = database.filter(query, Notification, terms) diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index fed2ddd4..9046e0c8 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -5,9 +5,10 @@ """ import arrow -from sqlalchemy import or_, cast, Boolean, Integer +from sqlalchemy import or_, cast, Integer from lemur import database +from lemur.common.utils import truthiness from lemur.plugins.base import plugins from lemur.roles.models import Role @@ -181,9 +182,9 @@ def render(args): elif 'destination' in terms: query = query.filter(PendingCertificate.destinations.any(Destination.id == terms[1])) elif 'notify' in filt: - query = query.filter(PendingCertificate.notify == cast(terms[1], Boolean)) + query = query.filter(PendingCertificate.notify == truthiness(terms[1])) elif 'active' in filt: - query = query.filter(PendingCertificate.active == terms[1]) + query = query.filter(PendingCertificate.active == truthiness(terms[1])) elif 'cn' in terms: query = query.filter( or_( diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index f2b1f7e9..a5bebe15 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -717,3 +717,11 @@ def test_certificates_upload_patch(client, token, status): def test_sensitive_sort(client): resp = client.get(api.url_for(CertificatesList) + '?sortBy=private_key&sortDir=asc', headers=VALID_ADMIN_HEADER_TOKEN) assert "'private_key' is not sortable or filterable" in resp.json['message'] + + +def test_boolean_filter(client): + resp = client.get(api.url_for(CertificatesList) + '?filter=notify;true', headers=VALID_ADMIN_HEADER_TOKEN) + assert resp.status_code == 200 + # Also don't crash with invalid input (we currently treat that as false) + resp = client.get(api.url_for(CertificatesList) + '?filter=notify;whatisthis', headers=VALID_ADMIN_HEADER_TOKEN) + assert resp.status_code == 200 From 8ca4f730e821796aedddc06121cbdc06c4f6de2d Mon Sep 17 00:00:00 2001 From: Will Bengtson Date: Tue, 10 Apr 2018 13:23:09 -0700 Subject: [PATCH 03/27] lemur_digicert: Do not truncate valid_to anymore (#1187) * lemur_digicert: Do not truncate valid_to anymore The valid_to field for Digicert supports YYYY-MM-DDTHH:MM:SSZ so we should stop truncating * lemur_digicert: Update unit tests for valid_to --- lemur/plugins/lemur_digicert/plugin.py | 2 +- lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 00fe9519..5692af86 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -157,7 +157,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get('signing_algorithm')), "validity": { - "valid_to": options['validity_end'].format('YYYY-MM-DD') + "valid_to": options['validity_end'].format('YYYY-MM-DDTHH:MM:SSZ') }, "organization": { "name": options['organization'], diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index dca24892..91f27ad4 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -103,7 +103,7 @@ def test_map_cis_fields(app): 'signature_hash': 'sha256', 'organization': {'name': 'Example, Inc.', 'units': ['Example Org']}, 'validity': { - 'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DD') + 'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DDTHH:MM:SSZ') }, 'profile_name': None } @@ -132,7 +132,7 @@ def test_map_cis_fields(app): 'signature_hash': 'sha256', 'organization': {'name': 'Example, Inc.', 'units': ['Example Org']}, 'validity': { - 'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DD') + 'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DDTHH:MM:SSZ') }, 'profile_name': None } From a9baaf4da4b100ae745dcacc1ad73f685451ef33 Mon Sep 17 00:00:00 2001 From: Mihir Jham Date: Tue, 10 Apr 2018 15:15:03 -0700 Subject: [PATCH 04/27] add(plugins): Added a statsd plugin for lemur (#1189) --- .../lemur_statsd/docs/requirements.txt | 1 + .../lemur_statsd/lemur_statsd/__init__.py | 4 ++ .../lemur_statsd/lemur_statsd/plugin.py | 45 +++++++++++++++++++ lemur/plugins/lemur_statsd/setup.py | 24 ++++++++++ 4 files changed, 74 insertions(+) create mode 100644 lemur/plugins/lemur_statsd/docs/requirements.txt create mode 100644 lemur/plugins/lemur_statsd/lemur_statsd/__init__.py create mode 100644 lemur/plugins/lemur_statsd/lemur_statsd/plugin.py create mode 100644 lemur/plugins/lemur_statsd/setup.py diff --git a/lemur/plugins/lemur_statsd/docs/requirements.txt b/lemur/plugins/lemur_statsd/docs/requirements.txt new file mode 100644 index 00000000..73a2e5b6 --- /dev/null +++ b/lemur/plugins/lemur_statsd/docs/requirements.txt @@ -0,0 +1 @@ +datadog==0.14.0 diff --git a/lemur/plugins/lemur_statsd/lemur_statsd/__init__.py b/lemur/plugins/lemur_statsd/lemur_statsd/__init__.py new file mode 100644 index 00000000..3a751848 --- /dev/null +++ b/lemur/plugins/lemur_statsd/lemur_statsd/__init__.py @@ -0,0 +1,4 @@ +try: + VERSION = __import__('pkg_resources').get_distribution(__name__).version +except Exception as e: + VERSION = 'Unknown' diff --git a/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py new file mode 100644 index 00000000..a6a87c66 --- /dev/null +++ b/lemur/plugins/lemur_statsd/lemur_statsd/plugin.py @@ -0,0 +1,45 @@ +import lemur_statsd as plug + +from flask import current_app +from lemur.plugins.bases.metric import MetricPlugin +from datadog import DogStatsd + + +class StatsdMetricPlugin(MetricPlugin): + title = 'Statsd' + slug = 'statsd-metrics' + description = 'Adds support for sending metrics to Statsd' + version = plug.VERSION + + def __init__(self): + host = current_app.config.get('STATSD_HOST') + port = current_app.config.get('STATSD_PORT') + prefix = current_app.config.get('STATSD_PREFIX') + + self.statsd = DogStatsd(host=host, port=port, namespace=prefix) + + def submit(self, metric_name, metric_type, metric_value, metric_tags=None, options=None): + valid_types = ['COUNTER', 'GAUGE', 'TIMER'] + tags = [] + + if metric_type.upper() not in valid_types: + raise Exception( + "Invalid Metric Type for Statsd, '{metric}' choose from: {options}".format( + metric=metric_type, options=','.join(valid_types) + ) + ) + + if metric_tags: + if not isinstance(metric_tags, dict): + raise Exception("Invalid Metric Tags for Statsd: Tags must be in dict format") + else: + tags = map(lambda e: "{0}:{1}".format(*e), metric_tags.items()) + + if metric_type.upper() == 'COUNTER': + self.statsd.increment(metric_name, metric_value, tags) + elif metric_type.upper() == 'GAUGE': + self.statsd.gauge(metric_name, metric_value, tags) + elif metric_type.upper() == 'TIMER': + self.statsd.timing(metric_name, metric_value, tags) + + return diff --git a/lemur/plugins/lemur_statsd/setup.py b/lemur/plugins/lemur_statsd/setup.py new file mode 100644 index 00000000..6c4c2dd6 --- /dev/null +++ b/lemur/plugins/lemur_statsd/setup.py @@ -0,0 +1,24 @@ +"""Basic package information""" +from __future__ import absolute_import +from setuptools import setup, find_packages + +install_requires = [ + 'lemur', + 'datadog' +] + +setup( + name='lemur_statsd', + version='1.0.0', + author='Cloudflare Security Engineering', + author_email='', + include_package_data=True, + packages=find_packages(), + zip_safe=False, + install_requires=install_requires, + entry_points={ + 'lemur.plugins': [ + 'statsd = lemur_statsd.plugin:StatsdMetricPlugin', + ] + } +) From 12622d5847ca8de2a08593837585b395ebc0cd21 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Tue, 10 Apr 2018 15:55:02 -0700 Subject: [PATCH 05/27] Adding metrics for request timings. (#1190) --- lemur/__init__.py | 40 +++++++++++++++++++++--------- lemur/decorators.py | 56 ------------------------------------------ lemur/extensions.py | 3 +++ lemur/factory.py | 4 ++- requirements-tests.txt | 8 +++--- requirements.in | 1 + requirements.txt | 7 +++--- 7 files changed, 43 insertions(+), 76 deletions(-) delete mode 100644 lemur/decorators.py diff --git a/lemur/__init__.py b/lemur/__init__.py index 6f29733c..c3661f4e 100644 --- a/lemur/__init__.py +++ b/lemur/__init__.py @@ -8,7 +8,8 @@ """ -from __future__ import absolute_import, division, print_function +import time +from flask import g, request from lemur import factory from lemur.extensions import metrics @@ -73,17 +74,6 @@ def configure_hook(app): """ from flask import jsonify from werkzeug.exceptions import HTTPException - from lemur.decorators import crossdomain - if app.config.get('CORS'): - @app.after_request - @crossdomain(origin=u"http://localhost:3000", methods=['PUT', 'HEAD', 'GET', 'POST', 'OPTIONS', 'DELETE']) - def after(response): - return response - - @app.after_request - def log_status(response): - metrics.send('status_code_{}'.format(response.status_code), 'counter', 1) - return response @app.errorhandler(Exception) def handle_error(e): @@ -93,3 +83,29 @@ def configure_hook(app): app.logger.exception(e) return jsonify(error=str(e)), code + + @app.before_request + def before_request(): + g.request_start_time = time.time() + + @app.after_request + def after_request(response): + # Return early if we don't have the start time + if not hasattr(g, 'request_start_time'): + return response + + # Get elapsed time in milliseconds + elapsed = time.time() - g.request_start_time + elapsed = int(round(1000 * elapsed)) + + # Collect request/response tags + tags = { + 'endpoint': request.endpoint, + 'request_method': request.method.lower(), + 'status_code': response.status_code + } + + # Record our response time metric + metrics.send('response_time', 'TIMER', elapsed, metric_tags=tags) + metrics.send('status_code_{}'.format(response.status_code), 'counter', 1) + return response diff --git a/lemur/decorators.py b/lemur/decorators.py deleted file mode 100644 index 7b93ec70..00000000 --- a/lemur/decorators.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -.. module: lemur.decorators - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. -""" -from builtins import str - -from datetime import timedelta -from flask import make_response, request, current_app - -from functools import update_wrapper - - -# this is only used for dev -def crossdomain(origin=None, methods=None, headers=None, - max_age=21600, attach_to_all=True, - automatic_options=True): # pragma: no cover - if methods is not None: - methods = ', '.join(sorted(x.upper() for x in methods)) - - if headers is not None and not isinstance(headers, str): - headers = ', '.join(x.upper() for x in headers) - - if not isinstance(origin, str): - origin = ', '.join(origin) - - if isinstance(max_age, timedelta): - max_age = max_age.total_seconds() - - def get_methods(): - if methods is not None: - return methods - - options_resp = current_app.make_default_options_response() - return options_resp.headers['allow'] - - def decorator(f): - def wrapped_function(*args, **kwargs): - if automatic_options and request.method == 'OPTIONS': - resp = current_app.make_default_options_response() - else: - resp = make_response(f(*args, **kwargs)) - if not attach_to_all and request.method != 'OPTIONS': - return resp - - h = resp.headers - h['Access-Control-Allow-Origin'] = origin - h['Access-Control-Allow-Methods'] = get_methods() - h['Access-Control-Max-Age'] = str(max_age) - h['Access-Control-Allow-Headers'] = "Origin, X-Requested-With, Content-Type, Accept, Authorization " - h['Access-Control-Allow-Credentials'] = 'true' - return resp - - f.provide_automatic_options = False - return update_wrapper(wrapped_function, f) - return decorator diff --git a/lemur/extensions.py b/lemur/extensions.py index b5a18569..76abcab1 100644 --- a/lemur/extensions.py +++ b/lemur/extensions.py @@ -26,3 +26,6 @@ sentry = Sentry() from blinker import Namespace signals = Namespace() + +from flask_cors import CORS +cors = CORS() diff --git a/lemur/factory.py b/lemur/factory.py index 107fb70f..93c18e71 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -21,7 +21,7 @@ from flask import Flask from lemur.certificates.hooks import activate_debug_dump from lemur.common.health import mod as health -from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry +from lemur.extensions import db, migrate, principal, smtp_mail, metrics, sentry, cors DEFAULT_BLUEPRINTS = ( @@ -124,6 +124,8 @@ def configure_extensions(app): smtp_mail.init_app(app) metrics.init_app(app) sentry.init_app(app) + app.config['CORS_HEADERS'] = 'Content-Type' + cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True) def configure_blueprints(app, blueprints): diff --git a/requirements-tests.txt b/requirements-tests.txt index 983aeb0e..9b359d72 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,9 +7,9 @@ asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.2 # via moto +boto3==1.7.3 # via moto boto==2.48.0 # via moto -botocore==1.10.2 # via boto3, moto, s3transfer +botocore==1.10.3 # via boto3, moto, s3transfer certifi==2018.1.18 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -42,10 +42,10 @@ pyaml==17.12.1 # via moto pycparser==2.18 # via cffi pyflakes==1.6.0 pytest-flask==0.10.0 -pytest-mock==1.8.0 +pytest-mock==1.9.0 pytest==3.5.0 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto -pytz==2018.3 # via moto +pytz==2018.4 # via moto pyyaml==3.12 # via pyaml requests-mock==1.4.0 requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock diff --git a/requirements.in b/requirements.in index 4dad196d..90a48170 100644 --- a/requirements.in +++ b/requirements.in @@ -11,6 +11,7 @@ Flask-RESTful==0.3.6 Flask-Script==2.0.6 Flask-SQLAlchemy Flask==0.12 +Flask-Cors future gunicorn inflection diff --git a/requirements.txt b/requirements.txt index dafb2bdf..1018354f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,13 +12,14 @@ arrow==0.12.1 asn1crypto==0.24.0 # via cryptography bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.2 -botocore==1.10.2 # via boto3, s3transfer +boto3==1.7.3 +botocore==1.10.3 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask cryptography==2.2.2 docutils==0.14 # via botocore flask-bcrypt==0.7.1 +flask-cors==3.0.3 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -55,7 +56,7 @@ pyrfc3339==1.0 # via acme python-dateutil==2.6.1 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.0.0 -pytz==2018.3 # via acme, flask-restful, pyrfc3339 +pytz==2018.4 # via acme, flask-restful, pyrfc3339 raven[flask]==6.6.0 requests[security]==2.11.1 retrying==1.3.3 From 6a762d463f3d216e01d57ae9616730b04974a23c Mon Sep 17 00:00:00 2001 From: lmitul Date: Tue, 10 Apr 2018 16:50:58 -0700 Subject: [PATCH 06/27] Documenting connection pool config settings (#1197) --- docs/administration.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 3c0ca457..1d50f280 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -65,6 +65,36 @@ Basic Configuration SQLALCHEMY_DATABASE_URI = 'postgresql://:@:5432/lemur' +.. data:: SQLALCHEMY_POOL_SIZE +:noindex: + + The default connection pool size is 5 for sqlalchemy managed connections. Depending on the number of Lemur instances, + please specify per instance connection pool size. Below is an example to set connection pool size to 10. + + :: + + SQLALCHEMY_POOL_SIZE = 10 + + + .. warning:: +This is an optional setting but important to review and set for optimal database connection usage and for overall database performance. + +.. data:: SQLALCHEMY_MAX_OVERFLOW +:noindex: + + This setting allows to create connections in addition to specified number of connections in pool size. By default, sqlalchemy + allows 10 connections to create in addition to the pool size. This is also an optional setting. If `SQLALCHEMY_POOL_SIZE` and + `SQLALCHEMY_MAX_OVERFLOW` are not speficied then each Lemur instance may create maximum of 15 connections. + + :: + + SQLALCHECK_MAX_OVERFLOW = 0 + + + .. note:: +Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size. + + .. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION :noindex: From c6bd93fe85234237a5de6356bc880b4f15b991dc Mon Sep 17 00:00:00 2001 From: "Patrick R. Donahue" Date: Tue, 10 Apr 2018 16:54:02 -0700 Subject: [PATCH 07/27] PostgreSQL is required, not optional due to JSON column usage, so link to quickstart instructions and add create_config statement. (#1198) --- docs/developer/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 8b6da224..4c46566a 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -48,7 +48,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi * pip * virtualenv (ideally virtualenvwrapper) * node.js (for npm and building css/javascript) -* (Optional) PostgreSQL ++* `PostgreSQL `_ Once you've got all that, the rest is simple: @@ -77,6 +77,7 @@ Create a default Lemur configuration just as if this were a production instance: :: + lemur create_config lemur init You'll likely want to make some changes to the default configuration (we recommend developing against Postgres, for example). Once done, migrate your database using the following command: From 52cb1453337f632858ea7d3e5b55500d5ad584e3 Mon Sep 17 00:00:00 2001 From: Will Bengtson Date: Tue, 10 Apr 2018 16:54:17 -0700 Subject: [PATCH 08/27] ecc: add the support for ECC (#1191) * ecc: add the support for ECC update generate_private_key to support ECC. Move key types to constant. Update UI for the new key types * ecc: Remove extra line to fix linting * ecc: Fix flake8 lint problems * Update options.tpl.html --- lemur/common/utils.py | 42 ++++++++++++++++--- lemur/constants.py | 23 ++++++++++ lemur/plugins/lemur_digicert/plugin.py | 5 +++ .../authorities/authority/options.tpl.html | 3 +- .../certificates/certificate/options.tpl.html | 5 ++- lemur/tests/test_utils.py | 20 ++++++++- 6 files changed, 90 insertions(+), 8 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 4db31a4e..02f55340 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -14,10 +14,11 @@ from sqlalchemy import and_, func from cryptography import x509 from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.asymmetric import rsa, ec from flask_restful.reqparse import RequestParser +from lemur.constants import CERTIFICATE_KEY_TYPES from lemur.exceptions import InvalidConfiguration paginated_parser = RequestParser() @@ -78,17 +79,43 @@ def generate_private_key(key_type): """ Generates a new private key based on key_type. - Valid key types: RSA2048, RSA4096 + Valid key types: RSA2048, RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', + 'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1', + 'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', + 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2' :param key_type: :return: """ - valid_key_types = ['RSA2048', 'RSA4096'] - if key_type not in valid_key_types: + _CURVE_TYPES = { + "ECCPRIME192V1": ec.SECP192R1(), + "ECCPRIME256V1": ec.SECP256R1(), + + "ECCSECP192R1": ec.SECP192R1(), + "ECCSECP224R1": ec.SECP224R1(), + "ECCSECP256R1": ec.SECP256R1(), + "ECCSECP384R1": ec.SECP384R1(), + "ECCSECP521R1": ec.SECP521R1(), + "ECCSECP256K1": ec.SECP256K1(), + + "ECCSECT163K1": ec.SECT163K1(), + "ECCSECT233K1": ec.SECT233K1(), + "ECCSECT283K1": ec.SECT283K1(), + "ECCSECT409K1": ec.SECT409K1(), + "ECCSECT571K1": ec.SECT571K1(), + + "ECCSECT163R2": ec.SECT163R2(), + "ECCSECT233R1": ec.SECT233R1(), + "ECCSECT283R1": ec.SECT283R1(), + "ECCSECT409R1": ec.SECT409R1(), + "ECCSECT571R2": ec.SECT571R1(), + } + + if key_type not in CERTIFICATE_KEY_TYPES: raise Exception("Invalid key type: {key_type}. Supported key types: {choices}".format( key_type=key_type, - choices=",".join(valid_key_types) + choices=",".join(CERTIFICATE_KEY_TYPES) )) if 'RSA' in key_type: @@ -98,6 +125,11 @@ def generate_private_key(key_type): key_size=key_size, backend=default_backend() ) + elif 'ECC' in key_type: + return ec.generate_private_key( + curve=_CURVE_TYPES[key_type], + backend=default_backend() + ) def is_weekend(date): diff --git a/lemur/constants.py b/lemur/constants.py index 04824c12..0ee9bc40 100644 --- a/lemur/constants.py +++ b/lemur/constants.py @@ -9,3 +9,26 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}" SUCCESS_METRIC_STATUS = 'success' FAILURE_METRIC_STATUS = 'failure' + +CERTIFICATE_KEY_TYPES = [ + 'RSA2048', + 'RSA4096', + 'ECCPRIME192V1', + 'ECCPRIME256V1', + 'ECCSECP192R1', + 'ECCSECP224R1', + 'ECCSECP256R1', + 'ECCSECP384R1', + 'ECCSECP521R1', + 'ECCSECP256K1', + 'ECCSECT163K1', + 'ECCSECT233K1', + 'ECCSECT283K1', + 'ECCSECT409K1', + 'ECCSECT571K1', + 'ECCSECT163R2', + 'ECCSECT233R1', + 'ECCSECT283R1', + 'ECCSECT409R1', + 'ECCSECT571R2' +] diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 5692af86..ae6a5924 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -491,6 +491,11 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): self.session.headers.pop('Accept') end_entity = pem.parse(certificate_pem)[0] + + if 'ECC' in issuer_options['key_type']: + return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_ECC_CIS_INTERMEDIATE'), data['id'] + + # By default return RSA return "\n".join(str(end_entity).splitlines()), current_app.config.get('DIGICERT_CIS_INTERMEDIATE'), data['id'] def revoke_certificate(self, certificate, comments): diff --git a/lemur/static/app/angular/authorities/authority/options.tpl.html b/lemur/static/app/angular/authorities/authority/options.tpl.html index 57fc29e6..245716cb 100644 --- a/lemur/static/app/angular/authorities/authority/options.tpl.html +++ b/lemur/static/app/angular/authorities/authority/options.tpl.html @@ -20,7 +20,8 @@ Key Type
- +
diff --git a/lemur/static/app/angular/certificates/certificate/options.tpl.html b/lemur/static/app/angular/certificates/certificate/options.tpl.html index a52ee387..fb1d59a1 100644 --- a/lemur/static/app/angular/certificates/certificate/options.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/options.tpl.html @@ -32,7 +32,10 @@
diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index cc03fcd5..62d021a4 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -6,9 +6,27 @@ def test_generate_private_key(): assert generate_private_key('RSA2048') assert generate_private_key('RSA4096') + assert generate_private_key('ECCPRIME192V1') + assert generate_private_key('ECCPRIME256V1') + assert generate_private_key('ECCSECP192R1') + assert generate_private_key('ECCSECP224R1') + assert generate_private_key('ECCSECP256R1') + assert generate_private_key('ECCSECP384R1') + assert generate_private_key('ECCSECP521R1') + assert generate_private_key('ECCSECP256K1') + assert generate_private_key('ECCSECT163K1') + assert generate_private_key('ECCSECT233K1') + assert generate_private_key('ECCSECT283K1') + assert generate_private_key('ECCSECT409K1') + assert generate_private_key('ECCSECT571K1') + assert generate_private_key('ECCSECT163R2') + assert generate_private_key('ECCSECT233R1') + assert generate_private_key('ECCSECT283R1') + assert generate_private_key('ECCSECT409R1') + assert generate_private_key('ECCSECT571R2') with pytest.raises(Exception): - generate_private_key('ECC') + generate_private_key('LEMUR') def test_get_authority_key(): From 6cd2205f1f600a8d03d0ab898c6e7d0b72cfac3f Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 12 Apr 2018 08:52:47 -0700 Subject: [PATCH 09/27] up-reqs --- Makefile | 8 ++++---- lemur/common/schema.py | 2 +- lemur/migrations/alembic.ini | 2 +- requirements-dev.txt | 10 ++++++---- requirements-docs.txt | 5 +++-- requirements-tests.txt | 19 +++++++++++++------ requirements.txt | 14 ++++++++++---- 7 files changed, 38 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index d3d3ecc2..92181461 100644 --- a/Makefile +++ b/Makefile @@ -110,10 +110,10 @@ ifndef VIRTUAL_ENV endif @echo "--> Updating Python requirements" pip install --upgrade pip-tools - pip-compile --output-file requirements-docs.txt requirements-docs.in -U - pip-compile --output-file requirements-dev.txt requirements-dev.in -U - pip-compile --output-file requirements-tests.txt requirements-tests.in -U - pip-compile --output-file requirements.txt requirements.in -U + pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index + pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index + pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index + pip-compile --output-file requirements.txt requirements.in -U --no-index @echo "--> Done updating Python requirements" @echo "--> Installing new dependencies" pip install -e . diff --git a/lemur/common/schema.py b/lemur/common/schema.py index 1e081f8c..940f267c 100644 --- a/lemur/common/schema.py +++ b/lemur/common/schema.py @@ -148,7 +148,7 @@ def validate_schema(input_schema, output_schema): request_data = request.get_json() else: request_data = request.args - + print(data) data, errors = input_schema.load(request_data) if errors: diff --git a/lemur/migrations/alembic.ini b/lemur/migrations/alembic.ini index f8ed4801..6c3428e7 100644 --- a/lemur/migrations/alembic.ini +++ b/lemur/migrations/alembic.ini @@ -8,7 +8,7 @@ # the 'revision' command, regardless of autogenerate # revision_environment = false - +script_location = . # Logging configuration [loggers] keys = root,sqlalchemy,alembic diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b11eb2b..9c2d4400 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,15 +2,17 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-dev.txt requirements-dev.in +# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # -aspy.yaml==1.0.0 # via pre-commit +aspy.yaml==1.1.0 # via pre-commit cached-property==1.4.2 # via pre-commit certifi==2018.1.18 # via requests cfgv==1.0.0 # via pre-commit chardet==3.0.4 # via requests +configparser==3.5.0 # via flake8 +enum34==1.1.6 # via flake8 flake8==3.5.0 -identify==1.0.9 # via pre-commit +identify==1.0.11 # via pre-commit idna==2.6 # via requests invoke==0.22.1 mccabe==0.6.1 # via flake8 @@ -23,7 +25,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.18.4 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -tqdm==4.21.0 # via twine +tqdm==4.22.0 # via twine twine==1.11.0 urllib3==1.22 # via requests virtualenv==15.2.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 35fce72b..a11881c2 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-docs.txt requirements-docs.in +# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # alabaster==0.7.10 # via sphinx babel==2.5.3 # via sphinx @@ -16,7 +16,7 @@ markupsafe==1.0 # via jinja2 packaging==17.1 # via sphinx pygments==2.2.0 # via sphinx pyparsing==2.2.0 # via packaging -pytz==2018.3 # via babel +pytz==2018.4 # via babel requests==2.18.4 # via sphinx six==1.11.0 # via packaging, sphinx, sphinxcontrib-httpdomain snowballstemmer==1.2.1 # via sphinx @@ -24,4 +24,5 @@ sphinx-rtd-theme==0.3.0 sphinx==1.7.2 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx +typing==3.6.4 # via sphinx urllib3==1.22 # via requests diff --git a/requirements-tests.txt b/requirements-tests.txt index 9b359d72..b7b79fd4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,14 +2,17 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-tests.txt requirements-tests.in +# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.3 # via moto +backports.ssl-match-hostname==3.5.0.1 # via docker +backports.tempfile==1.0 # via moto +backports.weakref==1.0.post1 # via backports.tempfile +boto3==1.7.4 # via moto boto==2.48.0 # via moto -botocore==1.10.3 # via boto3, moto, s3transfer +botocore==1.10.4 # via boto3, moto, s3transfer certifi==2018.1.18 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -20,22 +23,26 @@ cryptography==2.2.2 # via moto docker-pycreds==0.2.2 # via docker docker==3.2.1 # via moto docutils==0.14 # via botocore +enum34==1.1.6 # via cryptography factory-boy==2.10.0 -faker==0.8.12 +faker==0.8.13 flask==0.12.2 # via pytest-flask freezegun==0.3.10 +funcsigs==1.0.2 # via mock, pytest +futures==3.2.0 # via s3transfer idna==2.6 # via cryptography, requests +ipaddress==1.0.19 # via cryptography, docker, faker itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==0.9.6 # via aws-xray-sdk markupsafe==1.0 # via jinja2 -mock==2.0.0 # via moto +mock==2.0.0 # via moto, pytest-mock more-itertools==4.1.0 # via pytest moto==1.3.1 nose==1.3.7 -pbr==4.0.1 # via mock +pbr==4.0.2 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.txt b/requirements.txt index 1018354f..2f7c9458 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements.txt requirements.in +# pip-compile --no-index --output-file requirements.txt requirements.in # acme==0.23.0 alembic-autogenerate-enums==0.0.2 @@ -10,14 +10,17 @@ alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography +backports.functools-lru-cache==1.5 # via arrow bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.3 -botocore==1.10.3 # via boto3, s3transfer +boto3==1.7.4 +botocore==1.10.4 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask +contextlib2==0.5.5 # via raven cryptography==2.2.2 docutils==0.14 # via botocore +enum34==1.1.6 # via cryptography flask-bcrypt==0.7.1 flask-cors==3.0.3 flask-mail==0.9.1 @@ -27,10 +30,13 @@ flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 +funcsigs==1.0.2 # via mock future==0.16.0 +futures==3.2.0 # via s3transfer gunicorn==19.7.1 idna==2.6 # via cryptography inflection==0.3.1 +ipaddress==1.0.19 # via cryptography itsdangerous==0.24 # via flask jinja2==2.10 jmespath==0.9.3 # via boto3, botocore @@ -43,7 +49,7 @@ marshmallow==2.15.0 mock==2.0.0 # via acme ndg-httpsclient==0.4.4 paramiko==2.4.1 -pbr==4.0.1 # via mock +pbr==4.0.2 # via mock pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 # via python-ldap From acb1eab24eccc05a7b0eea0c397ebaf68f1cd581 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 12 Apr 2018 08:58:04 -0700 Subject: [PATCH 10/27] fix_errors --- lemur/common/schema.py | 2 +- lemur/migrations/alembic.ini | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/common/schema.py b/lemur/common/schema.py index 940f267c..51e257c7 100644 --- a/lemur/common/schema.py +++ b/lemur/common/schema.py @@ -148,7 +148,7 @@ def validate_schema(input_schema, output_schema): request_data = request.get_json() else: request_data = request.args - print(data) + data, errors = input_schema.load(request_data) if errors: diff --git a/lemur/migrations/alembic.ini b/lemur/migrations/alembic.ini index 6c3428e7..b2e42cfa 100644 --- a/lemur/migrations/alembic.ini +++ b/lemur/migrations/alembic.ini @@ -8,7 +8,6 @@ # the 'revision' command, regardless of autogenerate # revision_environment = false -script_location = . # Logging configuration [loggers] keys = root,sqlalchemy,alembic From c88c0b0127954e31bef0485598d9d1f828b6dcd6 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 12 Apr 2018 08:59:06 -0700 Subject: [PATCH 11/27] fix_changes --- lemur/common/schema.py | 2 +- lemur/migrations/alembic.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/common/schema.py b/lemur/common/schema.py index 51e257c7..1e081f8c 100644 --- a/lemur/common/schema.py +++ b/lemur/common/schema.py @@ -148,7 +148,7 @@ def validate_schema(input_schema, output_schema): request_data = request.get_json() else: request_data = request.args - + data, errors = input_schema.load(request_data) if errors: diff --git a/lemur/migrations/alembic.ini b/lemur/migrations/alembic.ini index b2e42cfa..f8ed4801 100644 --- a/lemur/migrations/alembic.ini +++ b/lemur/migrations/alembic.ini @@ -8,6 +8,7 @@ # the 'revision' command, regardless of autogenerate # revision_environment = false + # Logging configuration [loggers] keys = root,sqlalchemy,alembic From f302408712d17d396c79935088a89ac2e410b463 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 12 Apr 2018 12:34:08 -0700 Subject: [PATCH 12/27] py3 --- requirements-dev.txt | 2 -- requirements-docs.txt | 1 - requirements-tests.txt | 9 +-------- requirements.txt | 6 ------ 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9c2d4400..f94fa610 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,8 +9,6 @@ cached-property==1.4.2 # via pre-commit certifi==2018.1.18 # via requests cfgv==1.0.0 # via pre-commit chardet==3.0.4 # via requests -configparser==3.5.0 # via flake8 -enum34==1.1.6 # via flake8 flake8==3.5.0 identify==1.0.11 # via pre-commit idna==2.6 # via requests diff --git a/requirements-docs.txt b/requirements-docs.txt index a11881c2..a8b93db2 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -24,5 +24,4 @@ sphinx-rtd-theme==0.3.0 sphinx==1.7.2 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx -typing==3.6.4 # via sphinx urllib3==1.22 # via requests diff --git a/requirements-tests.txt b/requirements-tests.txt index b7b79fd4..4113ad2a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,9 +7,6 @@ asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -backports.ssl-match-hostname==3.5.0.1 # via docker -backports.tempfile==1.0 # via moto -backports.weakref==1.0.post1 # via backports.tempfile boto3==1.7.4 # via moto boto==2.48.0 # via moto botocore==1.10.4 # via boto3, moto, s3transfer @@ -23,22 +20,18 @@ cryptography==2.2.2 # via moto docker-pycreds==0.2.2 # via docker docker==3.2.1 # via moto docutils==0.14 # via botocore -enum34==1.1.6 # via cryptography factory-boy==2.10.0 faker==0.8.13 flask==0.12.2 # via pytest-flask freezegun==0.3.10 -funcsigs==1.0.2 # via mock, pytest -futures==3.2.0 # via s3transfer idna==2.6 # via cryptography, requests -ipaddress==1.0.19 # via cryptography, docker, faker itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==0.9.6 # via aws-xray-sdk markupsafe==1.0 # via jinja2 -mock==2.0.0 # via moto, pytest-mock +mock==2.0.0 # via moto more-itertools==4.1.0 # via pytest moto==1.3.1 nose==1.3.7 diff --git a/requirements.txt b/requirements.txt index 2f7c9458..5e8ef031 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,17 +10,14 @@ alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography -backports.functools-lru-cache==1.5 # via arrow bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.7.4 botocore==1.10.4 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask -contextlib2==0.5.5 # via raven cryptography==2.2.2 docutils==0.14 # via botocore -enum34==1.1.6 # via cryptography flask-bcrypt==0.7.1 flask-cors==3.0.3 flask-mail==0.9.1 @@ -30,13 +27,10 @@ flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 -funcsigs==1.0.2 # via mock future==0.16.0 -futures==3.2.0 # via s3transfer gunicorn==19.7.1 idna==2.6 # via cryptography inflection==0.3.1 -ipaddress==1.0.19 # via cryptography itsdangerous==0.24 # via flask jinja2==2.10 jmespath==0.9.3 # via boto3, botocore From 8303cfbd2bf7799b6636bf1e9d5a018f3b719888 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 13 Apr 2018 14:53:45 -0700 Subject: [PATCH 13/27] Fix datetime --- lemur/plugins/lemur_digicert/plugin.py | 2 +- lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 00fe9519..0e93f487 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -157,7 +157,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get('signing_algorithm')), "validity": { - "valid_to": options['validity_end'].format('YYYY-MM-DD') + "valid_to": options['validity_end'].format('YYYY-MM-DDTHH:MM:SS' + Z) }, "organization": { "name": options['organization'], diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index dca24892..7b19f318 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -103,7 +103,7 @@ def test_map_cis_fields(app): 'signature_hash': 'sha256', 'organization': {'name': 'Example, Inc.', 'units': ['Example Org']}, 'validity': { - 'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DD') + 'valid_to': arrow.get(2017, 5, 7).format('YYYY-MM-DDTHH:MM:SS' + Z) }, 'profile_name': None } @@ -132,7 +132,7 @@ def test_map_cis_fields(app): 'signature_hash': 'sha256', 'organization': {'name': 'Example, Inc.', 'units': ['Example Org']}, 'validity': { - 'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DD') + 'valid_to': arrow.get(2018, 11, 3).format('YYYY-MM-DDTHH:MM:SS' + Z) }, 'profile_name': None } From 4392657a7181456e0a57ac23a82f8a37b76e97e1 Mon Sep 17 00:00:00 2001 From: Titouan Christophe Date: Mon, 16 Apr 2018 19:41:28 +0200 Subject: [PATCH 14/27] [fix] Pip imports for pip 10 --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9f5e2990..3cb731d2 100644 --- a/setup.py +++ b/setup.py @@ -16,14 +16,20 @@ import datetime from distutils import log from distutils.core import Command -from pip.download import PipSession -from pip.req import parse_requirements from setuptools.command.develop import develop from setuptools.command.install import install from setuptools.command.sdist import sdist from setuptools import setup, find_packages from subprocess import check_output +import pip +if tuple(map(int, pip.__version__.split('.'))) >= (10, 0, 0): + from pip._internal.download import PipSession + from pip._internal.req import parse_requirements +else: + from pip.download import PipSession + from pip.req import parse_requirements + ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__))) # When executing the setup.py, we need to be able to import ourselves, this From 4e94e512181759291dab571b9b39a4b3586b756b Mon Sep 17 00:00:00 2001 From: Titouan Christophe Date: Mon, 16 Apr 2018 20:15:25 +0200 Subject: [PATCH 15/27] [add] Reference lemur-influxdb as 3rd party plugin --- docs/administration.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 1d50f280..b83c745b 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1186,6 +1186,19 @@ Digicert https://github.com/opendns/lemur-digicert +InfluxDB +-------- + +:Authors: + Titouan Christophe +:Type: + Metric +:Description: + Sends key metrics to InfluxDB +:Links: + https://github.com/titouanc/lemur-influxdb + + Have an extension that should be listed here? Submit a `pull request `_ and we'll get it added. From b9e93065f71713efdaeef6f3850d4c10b39fbd56 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Thu, 19 Apr 2018 13:26:49 -0700 Subject: [PATCH 16/27] Removing the need for a separate requirements txt (#1214) --- docs/requirements.txt | 36 --------------------- requirements-docs.in | 1 + requirements-docs.txt | 73 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 64 insertions(+), 46 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 5cb8b8b6..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,36 +0,0 @@ -CloudFlare==1.7.5 -Flask==0.12 -Flask-RESTful==0.3.6 -Flask-SQLAlchemy==2.1 -Flask-Script==2.0.5 -Flask-Migrate==2.1.1 -Flask-Bcrypt==0.7.1 -Flask-Principal==0.4.0 -Flask-Mail==0.9.1 -SQLAlchemy-Utils==0.32.14 -requests==2.11.1 -ndg-httpsclient==0.4.2 -psycopg2==2.7.3 -arrow==0.10.0 -six==1.10.0 -marshmallow-sqlalchemy==0.13.1 -gunicorn==19.7.1 -marshmallow==2.13.6 -cryptography==1.9 -xmltodict==0.11.0 -pyjwt==1.5.2 -lockfile==0.12.2 -inflection==0.3.1 -future==0.16.0 -boto3==1.4.6 -acme==0.18.1 -retrying==1.3.3 -tabulate==0.7.7 -pem==17.1.0 -raven[flask]==6.1.0 -jinja2==2.9.6 -# pyldap==2.4.37 # cannot be installed on rtd - required by ldap auth provider -paramiko==2.4.1 # required for the SFTP destination plugin -sphinx -sphinxcontrib-httpdomain -sphinx-rtd-theme diff --git a/requirements-docs.in b/requirements-docs.in index 8f1c7fe9..3dc923ba 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -1,3 +1,4 @@ +-r requirements.txt sphinx sphinxcontrib-httpdomain sphinx-rtd-theme \ No newline at end of file diff --git a/requirements-docs.txt b/requirements-docs.txt index a8b93db2..e62ea38f 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,24 +4,77 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # +acme==0.23.0 alabaster==0.7.10 # via sphinx +alembic-autogenerate-enums==0.0.2 +alembic==0.9.9 +aniso8601==3.0.0 +arrow==0.12.1 +asn1crypto==0.24.0 babel==2.5.3 # via sphinx -certifi==2018.1.18 # via requests -chardet==3.0.4 # via requests -docutils==0.14 # via sphinx -idna==2.6 # via requests +bcrypt==3.1.4 +blinker==1.4 +boto3==1.7.4 +botocore==1.10.4 +cffi==1.11.5 +click==6.7 +cryptography==2.2.2 +docutils==0.14 +flask-bcrypt==0.7.1 +flask-cors==3.0.3 +flask-mail==0.9.1 +flask-migrate==2.1.1 +flask-principal==0.4.0 +flask-restful==0.3.6 +flask-script==2.0.6 +flask-sqlalchemy==2.3.2 +flask==0.12 +future==0.16.0 +gunicorn==19.7.1 +idna==2.6 imagesize==1.0.0 # via sphinx -jinja2==2.10 # via sphinx -markupsafe==1.0 # via jinja2 +inflection==0.3.1 +itsdangerous==0.24 +jinja2==2.10 +jmespath==0.9.3 +josepy==1.0.1 +lockfile==0.12.2 +mako==1.0.7 +markupsafe==1.0 +marshmallow-sqlalchemy==0.13.2 +marshmallow==2.15.0 +mock==2.0.0 +ndg-httpsclient==0.4.4 packaging==17.1 # via sphinx +paramiko==2.4.1 +pbr==4.0.2 +pem==17.1.0 +psycopg2==2.7.4 +pyasn1-modules==0.2.1 +pyasn1==0.4.2 +pycparser==2.18 pygments==2.2.0 # via sphinx +pyjwt==1.6.1 +pynacl==1.2.1 +pyopenssl==17.2.0 pyparsing==2.2.0 # via packaging -pytz==2018.4 # via babel -requests==2.18.4 # via sphinx -six==1.11.0 # via packaging, sphinx, sphinxcontrib-httpdomain +pyrfc3339==1.0 +python-dateutil==2.6.1 +python-editor==1.0.3 +python-ldap==3.0.0 +pytz==2018.4 +raven[flask]==6.6.0 +requests[security]==2.11.1 +retrying==1.3.3 +s3transfer==0.1.13 +six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.3.0 sphinx==1.7.2 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx -urllib3==1.22 # via requests +sqlalchemy-utils==0.33.2 +sqlalchemy==1.2.6 +tabulate==0.8.2 +werkzeug==0.14.1 +xmltodict==0.11.0 From a8ce2190160089fb7cd8b652d02807f75be9301a Mon Sep 17 00:00:00 2001 From: kevgliss Date: Thu, 19 Apr 2018 13:57:35 -0700 Subject: [PATCH 17/27] Readthedocs doesn't have the necessary c header files to build python-ldap. (#1215) --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index e62ea38f..63d5ebe8 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -61,7 +61,7 @@ pyparsing==2.2.0 # via packaging pyrfc3339==1.0 python-dateutil==2.6.1 python-editor==1.0.3 -python-ldap==3.0.0 +# python-ldap==3.0.0 can install due to readthedocs.io not having the correct header files pytz==2018.4 raven[flask]==6.6.0 requests[security]==2.11.1 From 552c07e93276afadd3b6003028308245981710b4 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 23 Apr 2018 09:23:23 -0700 Subject: [PATCH 18/27] reqs update --- Makefile | 3 +++ requirements-dev.in | 2 ++ requirements-dev.txt | 6 +++--- requirements-docs.in | 3 +++ requirements-docs.txt | 15 +++++++-------- requirements-tests.in | 2 ++ requirements-tests.txt | 15 ++++++++------- requirements.in | 2 ++ requirements.txt | 12 ++++++------ 9 files changed, 36 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 92181461..f740faab 100644 --- a/Makefile +++ b/Makefile @@ -109,12 +109,15 @@ ifndef VIRTUAL_ENV $(error Please activate virtualenv first) endif @echo "--> Updating Python requirements" + pip install --upgrade pip pip install --upgrade pip-tools pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index pip-compile --output-file requirements.txt requirements.in -U --no-index @echo "--> Done updating Python requirements" + @echo "--> Removing python-ldap from requirements-docs.txt" + grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt @echo "--> Installing new dependencies" pip install -e . @echo "--> Done installing new dependencies" diff --git a/requirements-dev.in b/requirements-dev.in index 4e560267..de8b60d3 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,3 +1,5 @@ +# Run `make up-reqs` to update pinned dependencies in requirement text files + flake8>=3.2,<4.0 pre-commit invoke diff --git a/requirements-dev.txt b/requirements-dev.txt index f94fa610..356db421 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,11 +6,11 @@ # aspy.yaml==1.1.0 # via pre-commit cached-property==1.4.2 # via pre-commit -certifi==2018.1.18 # via requests +certifi==2018.4.16 # via requests cfgv==1.0.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.11 # via pre-commit +identify==1.0.13 # via pre-commit idna==2.6 # via requests invoke==0.22.1 mccabe==0.6.1 # via flake8 @@ -23,7 +23,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.18.4 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -tqdm==4.22.0 # via twine +tqdm==4.23.0 # via twine twine==1.11.0 urllib3==1.22 # via requests virtualenv==15.2.0 # via pre-commit diff --git a/requirements-docs.in b/requirements-docs.in index 3dc923ba..4111ae2d 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -1,3 +1,6 @@ +# Note: python-ldap from requirements breaks docs because it requires compiled binary. +# The `make up-reqs` will update all requirement text files, and forcibly remove python-ldap +# from requirements-docs.txt -r requirements.txt sphinx sphinxcontrib-httpdomain diff --git a/requirements-docs.txt b/requirements-docs.txt index 63d5ebe8..9d388183 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -14,8 +14,8 @@ asn1crypto==0.24.0 babel==2.5.3 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.4 -botocore==1.10.4 +boto3==1.7.6 +botocore==1.10.6 cffi==1.11.5 click==6.7 cryptography==2.2.2 @@ -37,7 +37,7 @@ inflection==0.3.1 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 -josepy==1.0.1 +josepy==1.1.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.0 @@ -59,22 +59,21 @@ pynacl==1.2.1 pyopenssl==17.2.0 pyparsing==2.2.0 # via packaging pyrfc3339==1.0 -python-dateutil==2.6.1 +python-dateutil==2.7.2 python-editor==1.0.3 -# python-ldap==3.0.0 can install due to readthedocs.io not having the correct header files pytz==2018.4 -raven[flask]==6.6.0 +raven[flask]==6.7.0 requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.3.0 -sphinx==1.7.2 +sphinx==1.7.3 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx sqlalchemy-utils==0.33.2 -sqlalchemy==1.2.6 +sqlalchemy==1.2.7 tabulate==0.8.2 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.in b/requirements-tests.in index 44a5c59f..02a2b0ae 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -1,3 +1,5 @@ +# Run `make up-reqs` to update pinned dependencies in requirement text files + coverage factory-boy Faker diff --git a/requirements-tests.txt b/requirements-tests.txt index 4113ad2a..0c74eacb 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,14 +7,14 @@ asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.4 # via moto +boto3==1.7.6 # via moto boto==2.48.0 # via moto -botocore==1.10.4 # via boto3, moto, s3transfer -certifi==2018.1.18 # via requests +botocore==1.10.6 # via boto3, moto, s3transfer +certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==6.7 # via flask -cookies==2.2.1 # via moto +cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto docker-pycreds==0.2.2 # via docker @@ -33,7 +33,7 @@ jsonpickle==0.9.6 # via aws-xray-sdk markupsafe==1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.1.0 # via pytest -moto==1.3.1 +moto==1.3.3 nose==1.3.7 pbr==4.0.2 # via mock pluggy==0.6.0 # via pytest @@ -48,9 +48,10 @@ python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.4 # via moto pyyaml==3.12 # via pyaml requests-mock==1.4.0 -requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock +requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses +responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 -six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, websocket-client +six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.22 # via requests websocket-client==0.47.0 # via docker diff --git a/requirements.in b/requirements.in index 90a48170..a5fe2317 100644 --- a/requirements.in +++ b/requirements.in @@ -1,3 +1,5 @@ +# Run `make up-reqs` to update pinned dependencies in requirement text files + acme alembic-autogenerate-enums arrow diff --git a/requirements.txt b/requirements.txt index 5e8ef031..23f1f28f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,8 +12,8 @@ arrow==0.12.1 asn1crypto==0.24.0 # via cryptography bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.4 -botocore==1.10.4 # via boto3, s3transfer +boto3==1.7.6 +botocore==1.10.6 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask cryptography==2.2.2 @@ -34,7 +34,7 @@ inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 jmespath==0.9.3 # via boto3, botocore -josepy==1.0.1 # via acme +josepy==1.1.0 # via acme lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.0 # via jinja2, mako @@ -53,17 +53,17 @@ pyjwt==1.6.1 pynacl==1.2.1 # via paramiko pyopenssl==17.2.0 pyrfc3339==1.0 # via acme -python-dateutil==2.6.1 # via alembic, arrow, botocore +python-dateutil==2.7.2 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.0.0 pytz==2018.4 # via acme, flask-restful, pyrfc3339 -raven[flask]==6.6.0 +raven[flask]==6.7.0 requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 sqlalchemy-utils==0.33.2 -sqlalchemy==1.2.6 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.7 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 werkzeug==0.14.1 # via flask xmltodict==0.11.0 From 7e36b0e8fd25e798e29f86f36074b873a1f7e985 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 23 Apr 2018 09:26:36 -0700 Subject: [PATCH 19/27] comment --- requirements-docs.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.in b/requirements-docs.in index 4111ae2d..cf598240 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -1,4 +1,4 @@ -# Note: python-ldap from requirements breaks docs because it requires compiled binary. +# Note: python-ldap from requirements breaks due to readthedocs.io not having the correct header files # The `make up-reqs` will update all requirement text files, and forcibly remove python-ldap # from requirements-docs.txt -r requirements.txt From ae638086782fa346d85b602310d19d39d82209e1 Mon Sep 17 00:00:00 2001 From: Ron Cohen Date: Mon, 23 Apr 2018 22:15:56 +0300 Subject: [PATCH 20/27] Update administration.rst (#1221) --- docs/administration.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index b83c745b..aab7cd58 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1198,6 +1198,18 @@ InfluxDB :Links: https://github.com/titouanc/lemur-influxdb +Hashicorp Vault +--------------- + +:Authors: + Ron Cohen +:Type: + Issuer +:Description: + Adds support for basic Vault PKI secret backend. +:Links: + https://github.com/RcRonco/lemur_vault + Have an extension that should be listed here? Submit a `pull request `_ and we'll get it added. From 91500d10221963330fcee82f4212264e7475e2e7 Mon Sep 17 00:00:00 2001 From: Zach Seils Date: Tue, 24 Apr 2018 19:53:51 -0400 Subject: [PATCH 21/27] Minor comment & stdout corrections (#1225) --- lemur/manage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/manage.py b/lemur/manage.py index 8f45cfb4..0079adb4 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -208,16 +208,16 @@ class InitializeApp(Command): if operator_role: sys.stdout.write("[-] Operator role already created, skipping...!\n") else: - # we create an admin role + # we create an operator role operator_role = role_service.create('operator', description='This is the Lemur operator role.') sys.stdout.write("[+] Created 'operator' role\n") read_only_role = role_service.get_by_name('read-only') if read_only_role: - sys.stdout.write("[-] Operator role already created, skipping...!\n") + sys.stdout.write("[-] Read only role already created, skipping...!\n") else: - # we create an admin role + # we create an read only role read_only_role = role_service.create('read-only', description='This is the Lemur read only role.') sys.stdout.write("[+] Created 'read-only' role\n") From 3e5db9eedbe1f1e58589a61226d1f013546bf750 Mon Sep 17 00:00:00 2001 From: Zach Seils Date: Tue, 24 Apr 2018 19:55:26 -0400 Subject: [PATCH 22/27] Check for default rotation policy before updating db (#1223) --- lemur/manage.py | 15 ++++++++++----- lemur/policies/service.py | 9 +++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/lemur/manage.py b/lemur/manage.py index 0079adb4..655b1827 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -251,12 +251,17 @@ class InitializeApp(Command): recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL') notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients) - days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30) - sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format( - days=days - )) + _DEFAULT_ROTATION_INTERVAL = 'default' + default_rotation_interval = policy_service.get_by_name(_DEFAULT_ROTATION_INTERVAL) + + if default_rotation_interval: + sys.stdout.write("[-] Default rotation interval policy already created, skipping...!\n") + else: + days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30) + sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format( + days=days)) + policy_service.create(days=days, name=_DEFAULT_ROTATION_INTERVAL) - policy_service.create(days=days, name='default') sys.stdout.write("[/] Done!\n") diff --git a/lemur/policies/service.py b/lemur/policies/service.py index 603fb3af..c6719a03 100644 --- a/lemur/policies/service.py +++ b/lemur/policies/service.py @@ -18,6 +18,15 @@ def get(policy_id): return database.get(RotationPolicy, policy_id) +def get_by_name(policy_name): + """ + Retrieves policy by its name. + :param policy_name: + :return: + """ + return database.get_all(RotationPolicy, policy_name, field='name').all() + + def delete(policy_id): """ Delete a rotation policy. From e09b7eb97860b69e40493b3eb528e57e823dba30 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Tue, 24 Apr 2018 17:10:38 -0700 Subject: [PATCH 23/27] Selectively enable CORS. (#1220) --- lemur/factory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lemur/factory.py b/lemur/factory.py index 93c18e71..97f7f6ca 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -124,8 +124,10 @@ def configure_extensions(app): smtp_mail.init_app(app) metrics.init_app(app) sentry.init_app(app) - app.config['CORS_HEADERS'] = 'Content-Type' - cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True) + + if app.config['CORS']: + app.config['CORS_HEADERS'] = 'Content-Type' + cors.init_app(app, resources=r'/api/*', headers='Content-Type', origin='*', supports_credentials=True) def configure_blueprints(app, blueprints): From 0579b2935c555a9763b3d93be2ce2f0f8ccb7423 Mon Sep 17 00:00:00 2001 From: Zach Seils Date: Thu, 26 Apr 2018 12:39:42 -0400 Subject: [PATCH 24/27] Print variable value instead of name (#1227) * Print variable value instead of name * Fixed ordering and variable name for stdout string --- lemur/manage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lemur/manage.py b/lemur/manage.py index 655b1827..919f70e4 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -237,9 +237,6 @@ class InitializeApp(Command): else: sys.stdout.write("[-] Default user has already been created, skipping...!\n") - sys.stdout.write("[+] Creating expiration email notifications!\n") - sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format("LEMUR_SECURITY_TEAM_EMAIL")) - intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", []) sys.stdout.write( "[!] Creating {num} notifications for {intervals} days as specified by LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS\n".format( @@ -249,6 +246,8 @@ class InitializeApp(Command): ) recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL') + sys.stdout.write("[+] Creating expiration email notifications!\n") + sys.stdout.write("[!] Using {0} as specified by LEMUR_SECURITY_TEAM_EMAIL for notifications\n".format(recipients)) notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients) _DEFAULT_ROTATION_INTERVAL = 'default' From 3ffeb8ab003d18ac3ad240eeb96796194d2c7b3f Mon Sep 17 00:00:00 2001 From: "Zach Seils (seils)" Date: Thu, 26 Apr 2018 19:45:00 -0400 Subject: [PATCH 25/27] Fix URL for Latest Docs image --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9582f756..9d878b5f 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ Lemur :alt: Join the chat at https://gitter.im/Netflix/lemur :target: https://gitter.im/Netflix/lemur?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge -.. image:: https://readthedocs.io/projects/lemur/badge/?version=latest +.. image:: https://readthedocs.org/projects/lemur/badge/?version=latest :target: https://lemur.readthedocs.io :alt: Latest Docs From 858c4eb8089236895f397b1d94cfa30f1f7fbcc6 Mon Sep 17 00:00:00 2001 From: "Zach Seils (seils)" Date: Fri, 27 Apr 2018 08:46:11 -0400 Subject: [PATCH 26/27] Add code coverage badge to README --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 9582f756..91f244b3 100644 --- a/README.rst +++ b/README.rst @@ -14,6 +14,10 @@ Lemur .. image:: https://travis-ci.org/Netflix/lemur.svg :target: https://travis-ci.org/Netflix/lemur +.. image:: https://coveralls.io/repos/github/Netflix/lemur/badge.svg?branch=master + :target: https://coveralls.io/github/Netflix/lemur?branch=master + + Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults. From c26ae160603eec3ea0a66d49438ce08b4a7d2995 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Tue, 8 May 2018 10:58:48 -0700 Subject: [PATCH 27/27] fixing docs (#1231) --- lemur/certificates/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 80355950..59f96522 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -300,12 +300,14 @@ class CertificatesUpload(AuthenticatedResource): { "owner": "joe@example.com", - "publicCert": "-----BEGIN CERTIFICATE-----...", - "intermediateCert": "-----BEGIN CERTIFICATE-----...", + "body": "-----BEGIN CERTIFICATE-----...", + "chain": "-----BEGIN CERTIFICATE-----...", "privateKey": "-----BEGIN RSA PRIVATE KEY-----..." "destinations": [], "notifications": [], "replacements": [], + "roles": [], + "notify": true, "name": "cert1" }