diff --git a/.travis.yml b/.travis.yml index b540937d..cf693a8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python sudo: required -dist: trusty +dist: xenial node_js: - "6.2.0" @@ -10,8 +10,8 @@ addons: matrix: include: - - python: "3.5" - env: TOXENV=py35 + - python: "3.7" + env: TOXENV=py37 cache: directories: diff --git a/docker-compose.yml b/docker-compose.yml index 66f2f0b1..ee0d8396 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,10 +13,13 @@ services: VIRTUAL_ENV: 'true' postgres: - image: postgres:9.4 + image: postgres + restart: always environment: POSTGRES_USER: lemur POSTGRES_PASSWORD: lemur + ports: + - "5432:5432" redis: image: "redis:alpine" diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 991dac2c..10747d31 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -20,6 +20,8 @@ from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources +from lemur.destinations import service as destinations_service +from lemur.sources.service import add_aws_destination_to_sources if current_app: flask_app = current_app @@ -255,3 +257,21 @@ def sync_source(source): sync([source]) log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) + + +@celery.task() +def sync_source_destination(): + """ + This celery task will sync destination and source, to make sure all new destinations are also present as source. + Some destinations do not qualify as sources, and hence should be excluded from being added as sources + We identify qualified destinations based on the sync_as_source attributed of the plugin. + The destination sync_as_source_name reveals the name of the suitable source-plugin. + We rely on account numbers to avoid duplicates. + """ + current_app.logger.debug("Syncing AWS destinations and sources") + + for dst in destinations_service.get_all(): + if add_aws_destination_to_sources(dst): + current_app.logger.debug("Source: %s added", dst.label) + + current_app.logger.debug("Completed Syncing AWS destinations and sources") diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index ed6fcb0f..8e505fce 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -6,11 +6,13 @@ .. moduleauthor:: Kevin Glisson """ from sqlalchemy import func +from flask import current_app from lemur import database from lemur.models import certificate_destination_associations from lemur.destinations.models import Destination from lemur.certificates.models import Certificate +from lemur.sources.service import add_aws_destination_to_sources def create(label, plugin_name, options, description=None): @@ -28,6 +30,12 @@ def create(label, plugin_name, options, description=None): del option['value']['plugin_object'] destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description) + current_app.logger.info("Destination: %s created", label) + + # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination + if add_aws_destination_to_sources(destination): + current_app.logger.info("Source: %s created", label) + return database.create(destination) diff --git a/lemur/manage.py b/lemur/manage.py index 9161109b..c9ce4240 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy # noqa from lemur.pending_certificates.models import PendingCertificate # noqa from lemur.dns_providers.models import DnsProvider # noqa +from sqlalchemy.sql import text + manager = Manager(create_app) manager.add_option('-c', '--config', dest='config_path', required=False) @@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur' @MigrateCommand.command def create(): + database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm')) database.db.create_all() stamp(revision='head') diff --git a/lemur/plugins/bases/destination.py b/lemur/plugins/bases/destination.py index 1e7e4ed2..fc73ebcb 100644 --- a/lemur/plugins/bases/destination.py +++ b/lemur/plugins/bases/destination.py @@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins class DestinationPlugin(Plugin): type = 'destination' requires_key = True + sync_as_source = False + sync_as_source_name = '' def upload(self, name, body, private_key, cert_chain, options, **kwargs): raise NotImplementedError diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 1c2607a5..41bec31c 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict): return endpoints -class AWSDestinationPlugin(DestinationPlugin): - title = 'AWS' - slug = 'aws-destination' - description = 'Allow the uploading of certificates to AWS IAM' - version = aws.VERSION - - author = 'Kevin Glisson' - author_url = 'https://github.com/netflix/lemur' - - options = [ - { - 'name': 'accountNumber', - 'type': 'str', - 'required': True, - 'validation': '[0-9]{12}', - 'helpMessage': 'Must be a valid AWS account number!', - }, - { - 'name': 'path', - 'type': 'str', - 'default': '/', - 'helpMessage': 'Path to upload certificate.' - } - ] - - # 'elb': { - # 'name': {'type': 'name'}, - # 'region': {'type': 'str'}, - # 'port': {'type': 'int'} - # } - - def upload(self, name, body, private_key, cert_chain, options, **kwargs): - iam.upload_cert(name, body, private_key, - self.get_option('path', options), - cert_chain=cert_chain, - account_number=self.get_option('accountNumber', options)) - - def deploy(self, elb_name, account, region, certificate): - pass - - class AWSSourcePlugin(SourcePlugin): title = 'AWS' slug = 'aws-source' @@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin): iam.delete_cert(certificate.name, account_number=account_number) +class AWSDestinationPlugin(DestinationPlugin): + title = 'AWS' + slug = 'aws-destination' + description = 'Allow the uploading of certificates to AWS IAM' + version = aws.VERSION + sync_as_source = True + sync_as_source_name = AWSSourcePlugin.slug + + author = 'Kevin Glisson' + author_url = 'https://github.com/netflix/lemur' + + options = [ + { + 'name': 'accountNumber', + 'type': 'str', + 'required': True, + 'validation': '[0-9]{12}', + 'helpMessage': 'Must be a valid AWS account number!', + }, + { + 'name': 'path', + 'type': 'str', + 'default': '/', + 'helpMessage': 'Path to upload certificate.' + } + ] + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + iam.upload_cert(name, body, private_key, + self.get_option('path', options), + cert_chain=cert_chain, + account_number=self.get_option('accountNumber', options)) + + def deploy(self, elb_name, account, region, certificate): + pass + + class S3DestinationPlugin(ExportDestinationPlugin): title = 'AWS-S3' slug = 'aws-s3' diff --git a/lemur/plugins/utils.py b/lemur/plugins/utils.py index a1914dd7..e057d071 100644 --- a/lemur/plugins/utils.py +++ b/lemur/plugins/utils.py @@ -18,4 +18,14 @@ def get_plugin_option(name, options): """ for o in options: if o.get('name') == name: - return o['value'] + return o.get('value', o.get('default')) + + +def set_plugin_option(name, value, options): + """ + Set value for option name for options dict. + :param options: + """ + for o in options: + if o.get('name') == name: + o.update({'value': value}) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 47b7f02c..a4d373ab 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -6,6 +6,7 @@ .. moduleauthor:: Kevin Glisson """ import arrow +import copy from flask import current_app @@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi from lemur.common.defaults import serial from lemur.plugins.base import plugins +from lemur.plugins.utils import get_plugin_option, set_plugin_option def certificate_create(certificate, source): @@ -256,3 +258,35 @@ def render(args): query = database.filter(query, Source, terms) return database.sort_and_page(query, Source, args) + + +def add_aws_destination_to_sources(dst): + """ + Given a destination check, if it can be added as sources, and included it if not already a source + We identify qualified destinations based on the sync_as_source attributed of the plugin. + The destination sync_as_source_name reveals the name of the suitable source-plugin. + We rely on account numbers to avoid duplicates. + :return: true for success and false for not adding the destination as source + """ + # a set of all accounts numbers available as sources + src_accounts = set() + sources = get_all() + for src in sources: + src_accounts.add(get_plugin_option('accountNumber', src.options)) + + # check + destination_plugin = plugins.get(dst.plugin_name) + account_number = get_plugin_option('accountNumber', dst.options) + if account_number is not None and \ + destination_plugin.sync_as_source is not None and \ + destination_plugin.sync_as_source and \ + (account_number not in src_accounts): + src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) + set_plugin_option('accountNumber', account_number, src_options) + create(label=dst.label, + plugin_name=destination_plugin.sync_as_source_name, + options=src_options, + description=dst.description) + return True + + return False diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index 43fa7163..e65b9440 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from flask import current_app from flask_principal import identity_changed, Identity +from sqlalchemy.sql import text from lemur import create_app from lemur.common.utils import parse_private_key @@ -55,6 +56,7 @@ def app(request): @pytest.yield_fixture(scope="session") def db(app, request): _db.drop_all() + _db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm')) _db.create_all() _db.app = app diff --git a/requirements-dev.txt b/requirements-dev.txt index 36e2c9a4..e62d1ee6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,18 +7,18 @@ aspy.yaml==1.2.0 # via pre-commit bleach==3.1.0 # via readme-renderer certifi==2019.3.9 # via requests -cfgv==1.5.0 # via pre-commit +cfgv==1.6.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.4.0 # via pre-commit +identify==1.4.1 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.8 # via pre-commit +importlib-metadata==0.9 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.4 +pre-commit==1.15.1 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer diff --git a/requirements-docs.txt b/requirements-docs.txt index e99c9cdc..e4233960 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # -acme==0.32.0 +acme==0.33.1 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.8 @@ -15,11 +15,11 @@ asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.6 -billiard==3.5.0.5 +billiard==3.6.0.0 blinker==1.4 -boto3==1.9.120 -botocore==1.12.120 -celery[redis]==4.2.2 +boto3==1.9.130 +botocore==1.12.130 +celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 @@ -42,28 +42,28 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -hvac==0.7.2 +hvac==0.8.2 idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 -jinja2==2.10 +jinja2==2.10.1 jmespath==0.9.4 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.3.0 +kombu==4.5.0 lockfile==0.12.2 mako==1.0.8 markupsafe==1.1.1 marshmallow-sqlalchemy==0.16.1 -marshmallow==2.19.1 +marshmallow==2.19.2 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 pbr==5.1.3 pem==19.1.0 -psycopg2==2.7.7 +psycopg2==2.8.1 pyasn1-modules==0.2.4 pyasn1==0.4.5 pycparser==2.19 @@ -71,14 +71,14 @@ pygments==2.3.1 # via sphinx pyjwt==1.7.1 pynacl==1.3.0 pyopenssl==19.0.0 -pyparsing==2.3.1 # via packaging +pyparsing==2.4.0 # via packaging pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 -pytz==2018.9 +pytz==2019.1 pyyaml==5.1 raven[flask]==6.10.0 -redis==2.10.6 +redis==3.2.1 requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 @@ -86,13 +86,18 @@ s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==1.8.5 +sphinx==2.0.1 +sphinxcontrib-applehelp==1.0.1 # via sphinx +sphinxcontrib-devhelp==1.0.1 # via sphinx +sphinxcontrib-htmlhelp==1.0.1 # via sphinx sphinxcontrib-httpdomain==1.7.0 -sphinxcontrib-websupport==1.1.0 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-qthelp==1.0.2 # via sphinx +sphinxcontrib-serializinghtml==1.1.3 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.1 +sqlalchemy==1.3.2 tabulate==0.8.3 urllib3==1.24.1 vine==1.3.0 -werkzeug==0.15.1 +werkzeug==0.15.2 xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index ed48cfdd..87fc5b66 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.120 # via moto +boto3==1.9.130 # via moto boto==2.49.0 # via moto -botocore==1.12.120 # via boto3, moto, s3transfer +botocore==1.12.130 # via boto3, moto, s3transfer certifi==2019.3.9 # via requests cffi==1.12.2 # via cryptography chardet==3.0.4 # via requests @@ -18,7 +18,7 @@ click==7.0 # via flask coverage==4.5.3 cryptography==2.6.1 # via moto docker-pycreds==0.4.0 # via docker -docker==3.7.1 # via moto +docker==3.7.2 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 @@ -28,13 +28,13 @@ freezegun==0.3.11 future==0.17.1 # via python-jose idna==2.8 # via requests itsdangerous==1.1.0 # via flask -jinja2==2.10 # via flask, moto +jinja2==2.10.1 # via flask, moto jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk markupsafe==1.1.1 # via jinja2 mock==2.0.0 # via moto -more-itertools==6.0.0 # via pytest +more-itertools==7.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.3 # via mock @@ -42,14 +42,14 @@ pluggy==0.9.0 # via pytest py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi -pycryptodome==3.8.0 # via python-jose +pycryptodome==3.8.1 # via python-jose pyflakes==2.1.1 pytest-flask==0.14.0 -pytest-mock==1.10.2 -pytest==4.3.1 +pytest-mock==1.10.3 +pytest==4.4.0 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.9 # via moto +pytz==2019.1 # via moto pyyaml==5.1 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses @@ -59,6 +59,6 @@ six==1.12.0 # via cryptography, docker, docker-pycreds, faker, fre text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.56.0 # via docker -werkzeug==0.15.1 # via flask, moto, pytest-flask +werkzeug==0.15.2 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements.in b/requirements.in index ecd95284..a3cabc18 100644 --- a/requirements.in +++ b/requirements.in @@ -27,7 +27,7 @@ gunicorn hvac # required for the vault destination plugin inflection jinja2 -kombu==4.3.0 # kombu 4.4.0 requires redis 3 +kombu lockfile marshmallow-sqlalchemy marshmallow @@ -39,7 +39,7 @@ pyjwt pyOpenSSL python_ldap raven[flask] -redis<3 # redis>=3 is not compatible with celery +redis requests retrying six diff --git a/requirements.txt b/requirements.txt index b458fb00..188dc66e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file requirements.txt requirements.in -U --no-index # -acme==0.32.0 +acme==0.33.1 alembic-autogenerate-enums==0.0.2 alembic==1.0.8 # via flask-migrate amqp==2.4.2 # via kombu @@ -13,11 +13,11 @@ arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko -billiard==3.5.0.5 # via celery +billiard==3.6.0.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.120 -botocore==1.12.120 -celery[redis]==4.2.2 +boto3==1.9.130 +botocore==1.12.130 +celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 # via bcrypt, cryptography, pynacl @@ -40,26 +40,26 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -hvac==0.7.2 +hvac==0.8.2 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask -jinja2==2.10 +jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.3.0 +kombu==4.5.0 lockfile==0.12.2 mako==1.0.8 # via alembic markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.16.1 -marshmallow==2.19.1 +marshmallow==2.19.2 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.3 # via mock pem==19.1.0 -psycopg2==2.7.7 +psycopg2==2.8.1 pyasn1-modules==0.2.4 # via python-ldap pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi @@ -70,20 +70,20 @@ pyrfc3339==1.1 # via acme python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-ldap==3.2.0 -pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 +pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.1 raven[flask]==6.10.0 -redis==2.10.6 +redis==3.2.1 requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.3.2 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests -vine==1.3.0 # via amqp -werkzeug==0.15.1 # via flask +vine==1.3.0 # via amqp, celery +werkzeug==0.15.2 # via flask xmltodict==0.12.0 pyjks==18.0.0 diff --git a/tox.ini b/tox.ini index fdd2585b..d3ad8944 100644 --- a/tox.ini +++ b/tox.ini @@ -1,2 +1,2 @@ [tox] -envlist = py35 +envlist = py37