From 6d1ef933c45e643643329d8b60dcb6b67bc2e920 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 5 Feb 2019 10:48:52 -0800 Subject: [PATCH 01/15] creating a new celery task to sync sources with destinations. This is as a measure to make sure important new destinations are also present as sources. --- lemur/common/celery.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f2a2f826..308adced 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -20,6 +20,9 @@ from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources +from lemur.destinations import service as destinations_service +from lemur.sources import service as sources_service + if current_app: flask_app = current_app @@ -226,3 +229,19 @@ def sync_source(source): """ current_app.logger.debug("Syncing source {}".format(source)) sync([source]) + + +@celery.task() +def sync_source_destination(): + """ + This celery task will sync destination and source, to make sure all new destinations are also present in source. + Some destinations do not qualify as sources, and hence should be excluded from being added as sources + """ + current_app.logger.debug("Syncing source and destination") + for dst in destinations_service.get_all(): + if dst.plugin_name == 'aws-destination' and not sources_service.get_by_label(dst.label): + sources_service.create(label=dst.label, + plugin_name='aws-source', + options=dst.options, + description=dst.description) + current_app.logger.info("Source: %s added", dst.label) From f7452e837974dd39bf7452a8ee059ec7738f18ef Mon Sep 17 00:00:00 2001 From: Javier Ramos Date: Fri, 15 Mar 2019 09:18:33 +0100 Subject: [PATCH 02/15] Parse DNSNames from CSR into Lemur Certificate --- lemur/certificates/schemas.py | 6 +++++ lemur/certificates/utils.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 lemur/certificates/utils.py diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index d20fd5a7..e9b61539 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -10,6 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError from lemur.authorities.schemas import AuthorityNestedOutputSchema +from lemur.certificates import utils as cert_utils from lemur.common import missing, utils, validators from lemur.common.fields import ArrowDateTime, Hex from lemur.common.schema import LemurInputSchema, LemurOutputSchema @@ -107,6 +108,11 @@ class CertificateInputSchema(CertificateCreationSchema): def load_data(self, data): if data.get('replacements'): data['replaces'] = data['replacements'] # TODO remove when field is deprecated + if data['csr']: + dns_names = cert_utils.get_dns_names_from_csr(data['csr']) + if not data['extensions']['subAltNames']['names']: + data['extensions']['subAltNames']['names'] = [] + data['extensions']['subAltNames']['names'] += dns_names return missing.convert_validity_years(data) diff --git a/lemur/certificates/utils.py b/lemur/certificates/utils.py new file mode 100644 index 00000000..933fe45e --- /dev/null +++ b/lemur/certificates/utils.py @@ -0,0 +1,42 @@ +""" +Utils to parse certificate data. + +.. module: lemur.certificates.hooks + :platform: Unix + :copyright: (c) 2019 by Javier Ramos, see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Javier Ramos +""" + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from marshmallow.exceptions import ValidationError + + +def get_dns_names_from_csr(data): + """ + Fetches DNSNames from CSR. + Potentially extendable to any kind of SubjectAlternativeName + :param data: PEM-encoded string with CSR + :return: + """ + dns_names = [] + try: + request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend()) + except Exception: + raise ValidationError('CSR presented is not valid.') + + try: + alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName) + + for name in alt_names.value.get_values_for_type(x509.DNSName): + dns_name = { + 'nameType': 'DNSName', + 'value': name + } + dns_names.append(dns_name) + except x509.ExtensionNotFound: + pass + + return dns_names From 9e5496b484fd5dcc87120938beabaf7b28031dcf Mon Sep 17 00:00:00 2001 From: Javier Ramos Date: Fri, 15 Mar 2019 10:19:25 +0100 Subject: [PATCH 03/15] Update schemas.py --- lemur/certificates/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index e9b61539..25fc2c46 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -108,7 +108,7 @@ class CertificateInputSchema(CertificateCreationSchema): def load_data(self, data): if data.get('replacements'): data['replaces'] = data['replacements'] # TODO remove when field is deprecated - if data['csr']: + if data.get('csr'): dns_names = cert_utils.get_dns_names_from_csr(data['csr']) if not data['extensions']['subAltNames']['names']: data['extensions']['subAltNames']['names'] = [] From c2158ff8fb284062afb70a2fef40fbbbc94092d9 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 25 Mar 2019 08:28:23 -0700 Subject: [PATCH 04/15] Add order URI during LE cert creation failure; Fail properly when invalid CA passed; Update reqs --- lemur/certificates/schemas.py | 3 +++ lemur/plugins/lemur_acme/plugin.py | 5 ++++- requirements-dev.txt | 4 ++-- requirements-docs.txt | 31 +++++++++++++++--------------- requirements-tests.txt | 26 ++++++++++++------------- requirements.txt | 31 +++++++++++++++--------------- 6 files changed, 52 insertions(+), 48 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index d20fd5a7..f790d92f 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -96,6 +96,9 @@ class CertificateInputSchema(CertificateCreationSchema): @validates_schema def validate_authority(self, data): + if isinstance(data['authority'], str): + raise ValidationError("Authority not found.") + if not data['authority'].active: raise ValidationError("The authority is inactive.", ['authority']) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 66295ed2..59cde380 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -459,7 +459,10 @@ class ACMEIssuerPlugin(IssuerPlugin): "pending_cert": entry["pending_cert"], }) except (PollError, AcmeError, Exception) as e: - current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) + order_url = order.uri + current_app.logger.error( + "Unable to resolve pending cert: {}. " + "Check out {} for more information.".format(pending_cert, order_url), exc_info=True) certs.append({ "cert": False, "pending_cert": entry["pending_cert"], diff --git a/requirements-dev.txt b/requirements-dev.txt index e67aea64..37202d97 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # aspy.yaml==1.2.0 # via pre-commit bleach==3.1.0 # via readme-renderer -certifi==2018.11.29 # via requests +certifi==2019.3.9 # via requests cfgv==1.5.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer @@ -23,7 +23,7 @@ pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer -pyyaml==5.1b3 +pyyaml==5.1 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine diff --git a/requirements-docs.txt b/requirements-docs.txt index e936c197..40cd73de 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,12 +4,12 @@ # # pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # -acme==0.31.0 +acme==0.32.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.8 amqp==2.4.2 -aniso8601==5.1.0 +aniso8601==6.0.0 arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 @@ -17,10 +17,10 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.107 -botocore==1.12.107 -celery[redis]==4.2.1 -certifi==2018.11.29 +boto3==1.9.120 +botocore==1.12.120 +celery[redis]==4.2.2 +certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 chardet==3.0.4 @@ -52,16 +52,16 @@ josepy==1.1.0 jsonlines==1.2.0 kombu==4.3.0 lockfile==0.12.2 -mako==1.0.7 +mako==1.0.8 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.1 +marshmallow-sqlalchemy==0.16.1 +marshmallow==2.19.1 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 pbr==5.1.3 -pem==18.2.0 +pem==19.1.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 pyasn1==0.4.5 @@ -75,10 +75,9 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 pytz==2018.9 -pyyaml==5.1b3 +pyyaml==5.1 raven[flask]==6.10.0 redis==2.10.6 -relativetimebuilder==0.2.0 requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 @@ -86,13 +85,13 @@ s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==1.8.4 +sphinx==1.8.5 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.0 +sqlalchemy==1.3.1 tabulate==0.8.3 urllib3==1.24.1 -vine==1.2.0 -werkzeug==0.14.1 +vine==1.3.0 +werkzeug==0.15.1 xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 55e38cbf..ed48cfdd 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,21 +8,21 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.107 # via moto +boto3==1.9.120 # via moto boto==2.49.0 # via moto -botocore==1.12.107 # via boto3, moto, s3transfer -certifi==2018.11.29 # via requests +botocore==1.12.120 # via boto3, moto, s3transfer +certifi==2019.3.9 # via requests cffi==1.12.2 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask -coverage==4.5.2 +coverage==4.5.3 cryptography==2.6.1 # via moto docker-pycreds==0.4.0 # via docker -docker==3.7.0 # via moto +docker==3.7.1 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==1.0.2 +faker==1.0.4 flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.17.1 # via python-jose @@ -42,23 +42,23 @@ pluggy==0.9.0 # via pytest py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi -pycryptodome==3.7.3 # via python-jose +pycryptodome==3.8.0 # via python-jose pyflakes==2.1.1 pytest-flask==0.14.0 -pytest-mock==1.10.1 -pytest==4.3.0 +pytest-mock==1.10.2 +pytest==4.3.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto -pyyaml==5.1b3 +pyyaml==5.1 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.5 # via moto +responses==0.10.6 # via moto s3transfer==0.2.0 # via boto3 six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests -websocket-client==0.55.0 # via docker -werkzeug==0.14.1 # via flask, moto, pytest-flask +websocket-client==0.56.0 # via docker +werkzeug==0.15.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements.txt b/requirements.txt index 2aa5f157..9adbdf37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,21 +4,21 @@ # # pip-compile --output-file requirements.txt requirements.in -U --no-index # -acme==0.31.0 +acme==0.32.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.8 # via flask-migrate amqp==2.4.2 # via kombu -aniso8601==5.1.0 # via flask-restful, relativetimebuilder +aniso8601==6.0.0 # via flask-restful arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.107 -botocore==1.12.107 -celery[redis]==4.2.1 -certifi==2018.11.29 +boto3==1.9.120 +botocore==1.12.120 +celery[redis]==4.2.2 +certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -49,15 +49,15 @@ josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.3.0 lockfile==0.12.2 -mako==1.0.7 # via alembic +mako==1.0.8 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.1 +marshmallow-sqlalchemy==0.16.1 +marshmallow==2.19.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.3 # via mock -pem==18.2.0 +pem==19.1.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 # via python-ldap pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap @@ -68,21 +68,20 @@ pyopenssl==19.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic -python-ldap==3.1.0 +python-ldap==3.2.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1b3 +pyyaml==5.1 raven[flask]==6.10.0 redis==2.10.6 -relativetimebuilder==0.2.0 # via aniso8601 requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.0 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests -vine==1.2.0 # via amqp -werkzeug==0.14.1 # via flask +vine==1.3.0 # via amqp +werkzeug==0.15.1 # via flask xmltodict==0.12.0 From d2e969b83648fc8d8317f0822522ab192dcd8983 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 21 Feb 2019 19:38:50 -0800 Subject: [PATCH 05/15] better synching of source and destinations --- lemur/plugins/bases/destination.py | 1 + lemur/plugins/lemur_aws/plugin.py | 1 + 2 files changed, 2 insertions(+) diff --git a/lemur/plugins/bases/destination.py b/lemur/plugins/bases/destination.py index 1e7e4ed2..04b01235 100644 --- a/lemur/plugins/bases/destination.py +++ b/lemur/plugins/bases/destination.py @@ -12,6 +12,7 @@ from lemur.plugins.base import Plugin, plugins class DestinationPlugin(Plugin): type = 'destination' requires_key = True + sync_as_source = False def upload(self, name, body, private_key, cert_chain, options, **kwargs): raise NotImplementedError diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 1c2607a5..d3c58464 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -195,6 +195,7 @@ class AWSSourcePlugin(SourcePlugin): slug = 'aws-source' description = 'Discovers all SSL certificates and ELB endpoints in an AWS account' version = aws.VERSION + sync_as_source = True author = 'Kevin Glisson' author_url = 'https://github.com/netflix/lemur' From e10007ef7b135eff08dea58b6b542dbb36ec72e3 Mon Sep 17 00:00:00 2001 From: Ryan DeShone Date: Fri, 29 Mar 2019 10:32:49 -0400 Subject: [PATCH 06/15] Add support for Vault KV API v2 This adds the ability to target KV API v1 or v2. --- lemur/plugins/lemur_vault_dest/plugin.py | 29 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 91f6a07a..6868b7b0 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -37,6 +37,17 @@ class VaultDestinationPlugin(DestinationPlugin): 'validation': '^https?://[a-zA-Z0-9.:-]+$', 'helpMessage': 'Valid URL to Hashi Vault instance' }, + { + 'name': 'vaultKvApiVersion', + 'type': 'select', + 'value': '2', + 'available': [ + '1', + '2' + ], + 'required': True, + 'helpMessage': 'Version of the Vault KV API to use' + }, { 'name': 'vaultAuthTokenFile', 'type': 'str', @@ -98,17 +109,20 @@ class VaultDestinationPlugin(DestinationPlugin): path = self.get_option('vaultPath', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) + api_version = self.get_option('vaultKvApiVersion', options) with open(token_file, 'r') as file: token = file.readline().rstrip('\n') client = hvac.Client(url=url, token=token) + client.secrets.kv.default_kv_version = api_version + if obj_name: path = '{0}/{1}'.format(path, obj_name) else: path = '{0}/{1}'.format(path, cname) - secret = get_secret(url, token, mount, path) + secret = get_secret(client, mount, path) secret['data'][cname] = {} if bundle == 'Nginx' and cert_chain: @@ -123,8 +137,9 @@ class VaultDestinationPlugin(DestinationPlugin): if isinstance(san_list, list): secret['data'][cname]['san'] = san_list try: - client.secrets.kv.v1.create_or_update_secret( - path=path, mount_point=mount, secret=secret['data']) + client.secrets.kv.create_or_update_secret( + path=path, mount_point=mount, secret=secret['data'] + ) except ConnectionError as err: current_app.logger.exception( "Exception uploading secret to vault: {0}".format(err), exc_info=True) @@ -144,12 +159,14 @@ def get_san_list(body): return san_list -def get_secret(url, token, mount, path): +def get_secret(client, mount, path): """ retreiive existing data from mount path and return dictionary """ result = {'data': {}} try: - client = hvac.Client(url=url, token=token) - result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) + if client.secrets.kv.default_kv_version == '1': + result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) + else: + result = client.secrets.kv.v2.read_secret_version(path=path, mount_point=mount) except ConnectionError: pass finally: From f3d0536800d7db899c38e3bc32ee86fee931c0d3 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Apr 2019 20:49:07 -0700 Subject: [PATCH 07/15] removing hardcoded rules, to give more flexibility into defining new source-destinations --- lemur/common/celery.py | 7 ++++--- lemur/plugins/bases/destination.py | 1 + lemur/plugins/lemur_aws/plugin.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 308adced..c926b390 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -234,14 +234,15 @@ def sync_source(source): @celery.task() def sync_source_destination(): """ - This celery task will sync destination and source, to make sure all new destinations are also present in source. + This celery task will sync destination and source, to make sure all new destinations are also present as source. Some destinations do not qualify as sources, and hence should be excluded from being added as sources """ current_app.logger.debug("Syncing source and destination") for dst in destinations_service.get_all(): - if dst.plugin_name == 'aws-destination' and not sources_service.get_by_label(dst.label): + destination_plugin = plugins.get(dst.plugin_name) + if destination_plugin.sync_as_source and not sources_service.get_by_label(dst.label): sources_service.create(label=dst.label, - plugin_name='aws-source', + plugin_name=destination_plugin.sync_as_source_name, options=dst.options, description=dst.description) current_app.logger.info("Source: %s added", dst.label) diff --git a/lemur/plugins/bases/destination.py b/lemur/plugins/bases/destination.py index 04b01235..fc73ebcb 100644 --- a/lemur/plugins/bases/destination.py +++ b/lemur/plugins/bases/destination.py @@ -13,6 +13,7 @@ class DestinationPlugin(Plugin): type = 'destination' requires_key = True sync_as_source = False + sync_as_source_name = '' def upload(self, name, body, private_key, cert_chain, options, **kwargs): raise NotImplementedError diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index d3c58464..2f271296 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -154,6 +154,8 @@ class AWSDestinationPlugin(DestinationPlugin): slug = 'aws-destination' description = 'Allow the uploading of certificates to AWS IAM' version = aws.VERSION + sync_as_source = True + sync_as_source_name = 'aws-source' author = 'Kevin Glisson' author_url = 'https://github.com/netflix/lemur' @@ -195,7 +197,6 @@ class AWSSourcePlugin(SourcePlugin): slug = 'aws-source' description = 'Discovers all SSL certificates and ELB endpoints in an AWS account' version = aws.VERSION - sync_as_source = True author = 'Kevin Glisson' author_url = 'https://github.com/netflix/lemur' From 2ff57e932c0686c769327abc15bd0382dbc21429 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 10 Apr 2019 15:40:48 -0700 Subject: [PATCH 08/15] Update requirements - upgrade to py37 --- docker-compose.yml | 5 ++++- lemur/manage.py | 3 +++ lemur/tests/conftest.py | 2 ++ requirements-dev.txt | 8 ++++---- requirements-docs.txt | 39 ++++++++++++++++++++++----------------- requirements-tests.txt | 20 ++++++++++---------- requirements.in | 4 ++-- requirements.txt | 30 +++++++++++++++--------------- tox.ini | 2 +- 9 files changed, 63 insertions(+), 50 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 66f2f0b1..ee0d8396 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,10 +13,13 @@ services: VIRTUAL_ENV: 'true' postgres: - image: postgres:9.4 + image: postgres + restart: always environment: POSTGRES_USER: lemur POSTGRES_PASSWORD: lemur + ports: + - "5432:5432" redis: image: "redis:alpine" diff --git a/lemur/manage.py b/lemur/manage.py index 9161109b..c9ce4240 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy # noqa from lemur.pending_certificates.models import PendingCertificate # noqa from lemur.dns_providers.models import DnsProvider # noqa +from sqlalchemy.sql import text + manager = Manager(create_app) manager.add_option('-c', '--config', dest='config_path', required=False) @@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur' @MigrateCommand.command def create(): + database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm')) database.db.create_all() stamp(revision='head') diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index 43fa7163..e65b9440 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from flask import current_app from flask_principal import identity_changed, Identity +from sqlalchemy.sql import text from lemur import create_app from lemur.common.utils import parse_private_key @@ -55,6 +56,7 @@ def app(request): @pytest.yield_fixture(scope="session") def db(app, request): _db.drop_all() + _db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm')) _db.create_all() _db.app = app diff --git a/requirements-dev.txt b/requirements-dev.txt index 36e2c9a4..e62d1ee6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,18 +7,18 @@ aspy.yaml==1.2.0 # via pre-commit bleach==3.1.0 # via readme-renderer certifi==2019.3.9 # via requests -cfgv==1.5.0 # via pre-commit +cfgv==1.6.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.4.0 # via pre-commit +identify==1.4.1 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.8 # via pre-commit +importlib-metadata==0.9 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.4 +pre-commit==1.15.1 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer diff --git a/requirements-docs.txt b/requirements-docs.txt index e99c9cdc..e4233960 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # -acme==0.32.0 +acme==0.33.1 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.8 @@ -15,11 +15,11 @@ asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.6 -billiard==3.5.0.5 +billiard==3.6.0.0 blinker==1.4 -boto3==1.9.120 -botocore==1.12.120 -celery[redis]==4.2.2 +boto3==1.9.130 +botocore==1.12.130 +celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 @@ -42,28 +42,28 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -hvac==0.7.2 +hvac==0.8.2 idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 -jinja2==2.10 +jinja2==2.10.1 jmespath==0.9.4 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.3.0 +kombu==4.5.0 lockfile==0.12.2 mako==1.0.8 markupsafe==1.1.1 marshmallow-sqlalchemy==0.16.1 -marshmallow==2.19.1 +marshmallow==2.19.2 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 pbr==5.1.3 pem==19.1.0 -psycopg2==2.7.7 +psycopg2==2.8.1 pyasn1-modules==0.2.4 pyasn1==0.4.5 pycparser==2.19 @@ -71,14 +71,14 @@ pygments==2.3.1 # via sphinx pyjwt==1.7.1 pynacl==1.3.0 pyopenssl==19.0.0 -pyparsing==2.3.1 # via packaging +pyparsing==2.4.0 # via packaging pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 -pytz==2018.9 +pytz==2019.1 pyyaml==5.1 raven[flask]==6.10.0 -redis==2.10.6 +redis==3.2.1 requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 @@ -86,13 +86,18 @@ s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==1.8.5 +sphinx==2.0.1 +sphinxcontrib-applehelp==1.0.1 # via sphinx +sphinxcontrib-devhelp==1.0.1 # via sphinx +sphinxcontrib-htmlhelp==1.0.1 # via sphinx sphinxcontrib-httpdomain==1.7.0 -sphinxcontrib-websupport==1.1.0 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-qthelp==1.0.2 # via sphinx +sphinxcontrib-serializinghtml==1.1.3 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.1 +sqlalchemy==1.3.2 tabulate==0.8.3 urllib3==1.24.1 vine==1.3.0 -werkzeug==0.15.1 +werkzeug==0.15.2 xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index ed48cfdd..87fc5b66 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.120 # via moto +boto3==1.9.130 # via moto boto==2.49.0 # via moto -botocore==1.12.120 # via boto3, moto, s3transfer +botocore==1.12.130 # via boto3, moto, s3transfer certifi==2019.3.9 # via requests cffi==1.12.2 # via cryptography chardet==3.0.4 # via requests @@ -18,7 +18,7 @@ click==7.0 # via flask coverage==4.5.3 cryptography==2.6.1 # via moto docker-pycreds==0.4.0 # via docker -docker==3.7.1 # via moto +docker==3.7.2 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 @@ -28,13 +28,13 @@ freezegun==0.3.11 future==0.17.1 # via python-jose idna==2.8 # via requests itsdangerous==1.1.0 # via flask -jinja2==2.10 # via flask, moto +jinja2==2.10.1 # via flask, moto jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk markupsafe==1.1.1 # via jinja2 mock==2.0.0 # via moto -more-itertools==6.0.0 # via pytest +more-itertools==7.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.3 # via mock @@ -42,14 +42,14 @@ pluggy==0.9.0 # via pytest py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi -pycryptodome==3.8.0 # via python-jose +pycryptodome==3.8.1 # via python-jose pyflakes==2.1.1 pytest-flask==0.14.0 -pytest-mock==1.10.2 -pytest==4.3.1 +pytest-mock==1.10.3 +pytest==4.4.0 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.9 # via moto +pytz==2019.1 # via moto pyyaml==5.1 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses @@ -59,6 +59,6 @@ six==1.12.0 # via cryptography, docker, docker-pycreds, faker, fre text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.56.0 # via docker -werkzeug==0.15.1 # via flask, moto, pytest-flask +werkzeug==0.15.2 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements.in b/requirements.in index 9b27f604..526f1b88 100644 --- a/requirements.in +++ b/requirements.in @@ -27,7 +27,7 @@ gunicorn hvac # required for the vault destination plugin inflection jinja2 -kombu==4.3.0 # kombu 4.4.0 requires redis 3 +kombu lockfile marshmallow-sqlalchemy marshmallow @@ -39,7 +39,7 @@ pyjwt pyOpenSSL python_ldap raven[flask] -redis<3 # redis>=3 is not compatible with celery +redis requests retrying six diff --git a/requirements.txt b/requirements.txt index c0e69fb4..b7cda309 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file requirements.txt requirements.in -U --no-index # -acme==0.32.0 +acme==0.33.1 alembic-autogenerate-enums==0.0.2 alembic==1.0.8 # via flask-migrate amqp==2.4.2 # via kombu @@ -13,11 +13,11 @@ arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko -billiard==3.5.0.5 # via celery +billiard==3.6.0.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.120 -botocore==1.12.120 -celery[redis]==4.2.2 +boto3==1.9.130 +botocore==1.12.130 +celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 # via bcrypt, cryptography, pynacl @@ -40,26 +40,26 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -hvac==0.7.2 +hvac==0.8.2 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask -jinja2==2.10 +jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.3.0 +kombu==4.5.0 lockfile==0.12.2 mako==1.0.8 # via alembic markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.16.1 -marshmallow==2.19.1 +marshmallow==2.19.2 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.3 # via mock pem==19.1.0 -psycopg2==2.7.7 +psycopg2==2.8.1 pyasn1-modules==0.2.4 # via python-ldap pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi @@ -70,19 +70,19 @@ pyrfc3339==1.1 # via acme python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-ldap==3.2.0 -pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 +pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.1 raven[flask]==6.10.0 -redis==2.10.6 +redis==3.2.1 requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.3.2 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests -vine==1.3.0 # via amqp -werkzeug==0.15.1 # via flask +vine==1.3.0 # via amqp, celery +werkzeug==0.15.2 # via flask xmltodict==0.12.0 diff --git a/tox.ini b/tox.ini index fdd2585b..d3ad8944 100644 --- a/tox.ini +++ b/tox.ini @@ -1,2 +1,2 @@ [tox] -envlist = py35 +envlist = py37 From d3fbf46f7a07aa70e6062cb7a922c97b9c0965df Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 10 Apr 2019 16:09:55 -0700 Subject: [PATCH 09/15] Upgrade travis deps --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b540937d..8765fed3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,8 @@ addons: matrix: include: - - python: "3.5" - env: TOXENV=py35 + - python: "3.7" + env: TOXENV=py37 cache: directories: From 142aadffef0f1318e94ab62aba078215cac39340 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 10 Apr 2019 16:18:49 -0700 Subject: [PATCH 10/15] Upgrade travis to xenial --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8765fed3..cf693a8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python sudo: required -dist: trusty +dist: xenial node_js: - "6.2.0" From f185df4f1e38e6e7b682bd11b7d1184382ea6c45 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 13:28:58 -0700 Subject: [PATCH 11/15] bringing class AWSDestinationPlugin(DestinationPlugin) after AWSSourcePlugin.slug, such that we can do: sync_as_source_name = AWSSourcePlugin.slug --- lemur/plugins/lemur_aws/plugin.py | 86 +++++++++++++++---------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 2f271296..57cc831c 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -149,49 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict): return endpoints -class AWSDestinationPlugin(DestinationPlugin): - title = 'AWS' - slug = 'aws-destination' - description = 'Allow the uploading of certificates to AWS IAM' - version = aws.VERSION - sync_as_source = True - sync_as_source_name = 'aws-source' - - author = 'Kevin Glisson' - author_url = 'https://github.com/netflix/lemur' - - options = [ - { - 'name': 'accountNumber', - 'type': 'str', - 'required': True, - 'validation': '[0-9]{12}', - 'helpMessage': 'Must be a valid AWS account number!', - }, - { - 'name': 'path', - 'type': 'str', - 'default': '/', - 'helpMessage': 'Path to upload certificate.' - } - ] - - # 'elb': { - # 'name': {'type': 'name'}, - # 'region': {'type': 'str'}, - # 'port': {'type': 'int'} - # } - - def upload(self, name, body, private_key, cert_chain, options, **kwargs): - iam.upload_cert(name, body, private_key, - self.get_option('path', options), - cert_chain=cert_chain, - account_number=self.get_option('accountNumber', options)) - - def deploy(self, elb_name, account, region, certificate): - pass - - class AWSSourcePlugin(SourcePlugin): title = 'AWS' slug = 'aws-source' @@ -268,6 +225,49 @@ class AWSSourcePlugin(SourcePlugin): iam.delete_cert(certificate.name, account_number=account_number) +class AWSDestinationPlugin(DestinationPlugin): + title = 'AWS' + slug = 'aws-destination' + description = 'Allow the uploading of certificates to AWS IAM' + version = aws.VERSION + sync_as_source = True + sync_as_source_name = AWSSourcePlugin.slug + + author = 'Kevin Glisson' + author_url = 'https://github.com/netflix/lemur' + + options = [ + { + 'name': 'accountNumber', + 'type': 'str', + 'required': True, + 'validation': '[0-9]{12}', + 'helpMessage': 'Must be a valid AWS account number!', + }, + { + 'name': 'path', + 'type': 'str', + 'default': '/', + 'helpMessage': 'Path to upload certificate.' + } + ] + + # 'elb': { + # 'name': {'type': 'name'}, + # 'region': {'type': 'str'}, + # 'port': {'type': 'int'} + # } + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + iam.upload_cert(name, body, private_key, + self.get_option('path', options), + cert_chain=cert_chain, + account_number=self.get_option('accountNumber', options)) + + def deploy(self, elb_name, account, region, certificate): + pass + + class S3DestinationPlugin(ExportDestinationPlugin): title = 'AWS-S3' slug = 'aws-s3' From 266c83367d81e563fa4984206ca1b5bd472527c4 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 13:29:37 -0700 Subject: [PATCH 12/15] avoiding hard-coded plugin names --- lemur/common/celery.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index ed751d9b..65114e01 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -18,7 +18,7 @@ from lemur.authorities.service import get as get_authority from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service -from lemur.plugins.base import plugins +from lemur.plugins.base import plugins, IPlugin from lemur.sources.cli import clean, sync, validate_sources from lemur.destinations import service as destinations_service from lemur.sources import service as sources_service @@ -265,13 +265,31 @@ def sync_source_destination(): """ This celery task will sync destination and source, to make sure all new destinations are also present as source. Some destinations do not qualify as sources, and hence should be excluded from being added as sources + We identify qualified destinations based on the sync_as_source attributed of the plugin. + The destination sync_as_source_name reviels the name of the suitable source-plugin. + We rely on account numbers to avoid duplicates. """ current_app.logger.debug("Syncing source and destination") + + # a set of all accounts numbers available as sources + src_accounts = set() + sources = validate_sources("all") + for src in sources: + src_accounts.add(IPlugin.get_option('accountNumber' ,src.options)) + for dst in destinations_service.get_all(): destination_plugin = plugins.get(dst.plugin_name) - if destination_plugin.sync_as_source and not sources_service.get_by_label(dst.label): + account_number = IPlugin.get_option('accountNumber', src.options) + if destination_plugin.sync_as_source and (account_number not in src_accounts): + src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) + for o in src_options: + if o.get('name') == 'accountNumber': + o.update({'value': account_number}) + sources_service.create(label=dst.label, plugin_name=destination_plugin.sync_as_source_name, - options=dst.options, + options=src_options, description=dst.description) current_app.logger.info("Source: %s added", dst.label) + + From ec3d2d73162b8b84de0de5b6fb1d72a85c210904 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 13:51:21 -0700 Subject: [PATCH 13/15] fixing typo --- lemur/common/celery.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 65114e01..fdac27eb 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -275,17 +275,16 @@ def sync_source_destination(): src_accounts = set() sources = validate_sources("all") for src in sources: - src_accounts.add(IPlugin.get_option('accountNumber' ,src.options)) + src_accounts.add(IPlugin.get_option('accountNumber', src.options)) for dst in destinations_service.get_all(): destination_plugin = plugins.get(dst.plugin_name) - account_number = IPlugin.get_option('accountNumber', src.options) + account_number = IPlugin.get_option('accountNumber', dst.options) if destination_plugin.sync_as_source and (account_number not in src_accounts): src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) for o in src_options: if o.get('name') == 'accountNumber': o.update({'value': account_number}) - sources_service.create(label=dst.label, plugin_name=destination_plugin.sync_as_source_name, options=src_options, From 60edab9f6db11861c059afd6fc2535b758c87cf5 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 14:12:31 -0700 Subject: [PATCH 14/15] cleaning up --- lemur/plugins/lemur_aws/plugin.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 57cc831c..41bec31c 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -252,12 +252,6 @@ class AWSDestinationPlugin(DestinationPlugin): } ] - # 'elb': { - # 'name': {'type': 'name'}, - # 'region': {'type': 'str'}, - # 'port': {'type': 'int'} - # } - def upload(self, name, body, private_key, cert_chain, options, **kwargs): iam.upload_cert(name, body, private_key, self.get_option('path', options), From 245923414741772d16d2ae1c79ef26fa401f75c3 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 14:34:26 -0700 Subject: [PATCH 15/15] removing lines --- lemur/common/celery.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index fdac27eb..61dde28e 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -290,5 +290,3 @@ def sync_source_destination(): options=src_options, description=dst.description) current_app.logger.info("Source: %s added", dst.label) - -