Merge branch 'master' of github.com:Netflix/lemur into re-update-190412

.travis.yml

@@ -1,6 +1,6 @@
 language: python
 sudo: required
-dist: trusty
+dist: xenial
 
 node_js:
   - "6.2.0"
@@ -10,8 +10,8 @@ addons:
 
 matrix:
   include:
-    - python: "3.5"
+    - python: "3.7"
-      env: TOXENV=py35
+      env: TOXENV=py37
 
 cache:
   directories:

docker-compose.yml

@@ -13,10 +13,13 @@ services:
       VIRTUAL_ENV: 'true'
 
   postgres:
-    image: postgres:9.4
+    image: postgres
+    restart: always
     environment:
       POSTGRES_USER: lemur
       POSTGRES_PASSWORD: lemur
+    ports:
+      - "5432:5432"
 
   redis:
     image: "redis:alpine"

lemur/common/celery.py

@@ -20,6 +20,8 @@ from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.sources.cli import clean, sync, validate_sources
+from lemur.destinations import service as destinations_service
+from lemur.sources.service import add_aws_destination_to_sources
 
 if current_app:
     flask_app = current_app
@@ -255,3 +257,21 @@ def sync_source(source):
     sync([source])
     log_data["message"] = "Done syncing source"
     current_app.logger.debug(log_data)
+
+
+@celery.task()
+def sync_source_destination():
+    """
+    This celery task will sync destination and source, to make sure all new destinations are also present as source.
+    Some destinations do not qualify as sources, and hence should be excluded from being added as sources
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    """
+    current_app.logger.debug("Syncing AWS destinations and sources")
+
+    for dst in destinations_service.get_all():
+        if add_aws_destination_to_sources(dst):
+            current_app.logger.debug("Source: %s added", dst.label)
+
+    current_app.logger.debug("Completed Syncing AWS destinations and sources")

lemur/destinations/service.py

@@ -6,11 +6,13 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from sqlalchemy import func
+from flask import current_app
 
 from lemur import database
 from lemur.models import certificate_destination_associations
 from lemur.destinations.models import Destination
 from lemur.certificates.models import Certificate
+from lemur.sources.service import add_aws_destination_to_sources
 
 
 def create(label, plugin_name, options, description=None):
@@ -28,6 +30,12 @@ def create(label, plugin_name, options, description=None):
             del option['value']['plugin_object']
 
     destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
+    current_app.logger.info("Destination: %s created", label)
+
+    # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination
+    if add_aws_destination_to_sources(destination):
+        current_app.logger.info("Source: %s created", label)
+
     return database.create(destination)
 
 

lemur/manage.py

@@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy  # noqa
 from lemur.pending_certificates.models import PendingCertificate  # noqa
 from lemur.dns_providers.models import DnsProvider  # noqa
 
+from sqlalchemy.sql import text
+
 manager = Manager(create_app)
 manager.add_option('-c', '--config', dest='config_path', required=False)
 
@@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
 
 @MigrateCommand.command
 def create():
+    database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
     database.db.create_all()
     stamp(revision='head')
 

lemur/plugins/bases/destination.py

@@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
 class DestinationPlugin(Plugin):
     type = 'destination'
     requires_key = True
+    sync_as_source = False
+    sync_as_source_name = ''
 
     def upload(self, name, body, private_key, cert_chain, options, **kwargs):
         raise NotImplementedError

lemur/plugins/lemur_aws/plugin.py

@@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
     return endpoints
 
 
-class AWSDestinationPlugin(DestinationPlugin):
-    title = 'AWS'
-    slug = 'aws-destination'
-    description = 'Allow the uploading of certificates to AWS IAM'
-    version = aws.VERSION
-
-    author = 'Kevin Glisson'
-    author_url = 'https://github.com/netflix/lemur'
-
-    options = [
-        {
-            'name': 'accountNumber',
-            'type': 'str',
-            'required': True,
-            'validation': '[0-9]{12}',
-            'helpMessage': 'Must be a valid AWS account number!',
-        },
-        {
-            'name': 'path',
-            'type': 'str',
-            'default': '/',
-            'helpMessage': 'Path to upload certificate.'
-        }
-    ]
-
-    # 'elb': {
-    #    'name': {'type': 'name'},
-    #    'region': {'type': 'str'},
-    #    'port': {'type': 'int'}
-    # }
-
-    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
-        iam.upload_cert(name, body, private_key,
-                        self.get_option('path', options),
-                        cert_chain=cert_chain,
-                        account_number=self.get_option('accountNumber', options))
-
-    def deploy(self, elb_name, account, region, certificate):
-        pass
-
-
 class AWSSourcePlugin(SourcePlugin):
     title = 'AWS'
     slug = 'aws-source'
@@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
         iam.delete_cert(certificate.name, account_number=account_number)
 
 
+class AWSDestinationPlugin(DestinationPlugin):
+    title = 'AWS'
+    slug = 'aws-destination'
+    description = 'Allow the uploading of certificates to AWS IAM'
+    version = aws.VERSION
+    sync_as_source = True
+    sync_as_source_name = AWSSourcePlugin.slug
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur'
+
+    options = [
+        {
+            'name': 'accountNumber',
+            'type': 'str',
+            'required': True,
+            'validation': '[0-9]{12}',
+            'helpMessage': 'Must be a valid AWS account number!',
+        },
+        {
+            'name': 'path',
+            'type': 'str',
+            'default': '/',
+            'helpMessage': 'Path to upload certificate.'
+        }
+    ]
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        iam.upload_cert(name, body, private_key,
+                        self.get_option('path', options),
+                        cert_chain=cert_chain,
+                        account_number=self.get_option('accountNumber', options))
+
+    def deploy(self, elb_name, account, region, certificate):
+        pass
+
+
 class S3DestinationPlugin(ExportDestinationPlugin):
     title = 'AWS-S3'
     slug = 'aws-s3'

lemur/plugins/lemur_vault_dest/plugin.py

@@ -37,6 +37,17 @@ class VaultDestinationPlugin(DestinationPlugin):
             'validation': '^https?://[a-zA-Z0-9.:-]+$',
             'helpMessage': 'Valid URL to Hashi Vault instance'
         },
+        {
+            'name': 'vaultKvApiVersion',
+            'type': 'select',
+            'value': '2',
+            'available': [
+                '1',
+                '2'
+            ],
+            'required': True,
+            'helpMessage': 'Version of the Vault KV API to use'
+        },
         {
             'name': 'vaultAuthTokenFile',
             'type': 'str',
@@ -98,17 +109,20 @@ class VaultDestinationPlugin(DestinationPlugin):
         path = self.get_option('vaultPath', options)
         bundle = self.get_option('bundleChain', options)
         obj_name = self.get_option('objectName', options)
+        api_version = self.get_option('vaultKvApiVersion', options)
 
         with open(token_file, 'r') as file:
             token = file.readline().rstrip('\n')
 
         client = hvac.Client(url=url, token=token)
+        client.secrets.kv.default_kv_version = api_version
+
         if obj_name:
             path = '{0}/{1}'.format(path, obj_name)
         else:
             path = '{0}/{1}'.format(path, cname)
 
-        secret = get_secret(url, token, mount, path)
+        secret = get_secret(client, mount, path)
         secret['data'][cname] = {}
 
         if bundle == 'Nginx' and cert_chain:
@@ -123,8 +137,9 @@ class VaultDestinationPlugin(DestinationPlugin):
         if isinstance(san_list, list):
             secret['data'][cname]['san'] = san_list
         try:
-            client.secrets.kv.v1.create_or_update_secret(
+            client.secrets.kv.create_or_update_secret(
-                path=path, mount_point=mount, secret=secret['data'])
+                path=path, mount_point=mount, secret=secret['data']
+            )
         except ConnectionError as err:
             current_app.logger.exception(
                 "Exception uploading secret to vault: {0}".format(err), exc_info=True)
@@ -144,12 +159,14 @@ def get_san_list(body):
         return san_list
 
 
-def get_secret(url, token, mount, path):
+def get_secret(client, mount, path):
     """ retreiive existing data from mount path and return dictionary """
     result = {'data': {}}
     try:
-        client = hvac.Client(url=url, token=token)
+        if client.secrets.kv.default_kv_version == '1':
-        result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount)
+            result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount)
+        else:
+            result = client.secrets.kv.v2.read_secret_version(path=path, mount_point=mount)
     except ConnectionError:
         pass
     finally:

lemur/plugins/utils.py

@@ -18,4 +18,14 @@ def get_plugin_option(name, options):
     """
     for o in options:
         if o.get('name') == name:
-            return o['value']
+            return o.get('value', o.get('default'))
+
+
+def set_plugin_option(name, value, options):
+    """
+    Set value for option name for options dict.
+    :param options:
+    """
+    for o in options:
+        if o.get('name') == name:
+            o.update({'value': value})

lemur/sources/service.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+import copy
 
 from flask import current_app
 
@@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi
 from lemur.common.defaults import serial
 
 from lemur.plugins.base import plugins
+from lemur.plugins.utils import get_plugin_option, set_plugin_option
 
 
 def certificate_create(certificate, source):
@@ -256,3 +258,35 @@ def render(args):
         query = database.filter(query, Source, terms)
 
     return database.sort_and_page(query, Source, args)
+
+
+def add_aws_destination_to_sources(dst):
+    """
+    Given a destination check, if it can be added as sources, and included it if not already a source
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    :return: true for success and false for not adding the destination as source
+    """
+    # a set of all accounts numbers available as sources
+    src_accounts = set()
+    sources = get_all()
+    for src in sources:
+        src_accounts.add(get_plugin_option('accountNumber', src.options))
+
+    # check
+    destination_plugin = plugins.get(dst.plugin_name)
+    account_number = get_plugin_option('accountNumber', dst.options)
+    if account_number is not None and \
+            destination_plugin.sync_as_source is not None and \
+            destination_plugin.sync_as_source and \
+            (account_number not in src_accounts):
+        src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
+        set_plugin_option('accountNumber', account_number, src_options)
+        create(label=dst.label,
+               plugin_name=destination_plugin.sync_as_source_name,
+               options=src_options,
+               description=dst.description)
+        return True
+
+    return False

lemur/tests/conftest.py

@@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from flask import current_app
 from flask_principal import identity_changed, Identity
+from sqlalchemy.sql import text
 
 from lemur import create_app
 from lemur.common.utils import parse_private_key
@@ -55,6 +56,7 @@ def app(request):
 @pytest.yield_fixture(scope="session")
 def db(app, request):
     _db.drop_all()
+    _db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
     _db.create_all()
 
     _db.app = app

requirements-dev.txt

@@ -7,18 +7,18 @@
 aspy.yaml==1.2.0          # via pre-commit
 bleach==3.1.0             # via readme-renderer
 certifi==2019.3.9         # via requests
-cfgv==1.5.0               # via pre-commit
+cfgv==1.6.0               # via pre-commit
 chardet==3.0.4            # via requests
 docutils==0.14            # via readme-renderer
 flake8==3.5.0
-identify==1.4.0           # via pre-commit
+identify==1.4.1           # via pre-commit
 idna==2.8                 # via requests
-importlib-metadata==0.8   # via pre-commit
+importlib-metadata==0.9   # via pre-commit
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.3
 pkginfo==1.5.0.1          # via twine
-pre-commit==1.14.4
+pre-commit==1.15.1
 pycodestyle==2.3.1        # via flake8
 pyflakes==1.6.0           # via flake8
 pygments==2.3.1           # via readme-renderer

requirements-docs.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
 #
-acme==0.32.0
+acme==0.33.1
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.8
@@ -15,11 +15,11 @@ asn1crypto==0.24.0
 asyncpool==1.0
 babel==2.6.0              # via sphinx
 bcrypt==3.1.6
-billiard==3.5.0.5
+billiard==3.6.0.0
 blinker==1.4
-boto3==1.9.120
+boto3==1.9.130
-botocore==1.12.120
+botocore==1.12.130
-celery[redis]==4.2.2
+celery[redis]==4.3.0
 certifi==2019.3.9
 certsrv==2.1.1
 cffi==1.12.2
@@ -42,28 +42,28 @@ flask-sqlalchemy==2.3.2
 flask==1.0.2
 future==0.17.1
 gunicorn==19.9.0
-hvac==0.7.2
+hvac==0.8.2
 idna==2.8
 imagesize==1.1.0          # via sphinx
 inflection==0.3.1
 itsdangerous==1.1.0
-jinja2==2.10
+jinja2==2.10.1
 jmespath==0.9.4
 josepy==1.1.0
 jsonlines==1.2.0
-kombu==4.3.0
+kombu==4.5.0
 lockfile==0.12.2
 mako==1.0.8
 markupsafe==1.1.1
 marshmallow-sqlalchemy==0.16.1
-marshmallow==2.19.1
+marshmallow==2.19.2
 mock==2.0.0
 ndg-httpsclient==0.5.1
 packaging==19.0           # via sphinx
 paramiko==2.4.2
 pbr==5.1.3
 pem==19.1.0
-psycopg2==2.7.7
+psycopg2==2.8.1
 pyasn1-modules==0.2.4
 pyasn1==0.4.5
 pycparser==2.19
@@ -71,14 +71,14 @@ pygments==2.3.1           # via sphinx
 pyjwt==1.7.1
 pynacl==1.3.0
 pyopenssl==19.0.0
-pyparsing==2.3.1          # via packaging
+pyparsing==2.4.0          # via packaging
 pyrfc3339==1.1
 python-dateutil==2.8.0
 python-editor==1.0.4
-pytz==2018.9
+pytz==2019.1
 pyyaml==5.1
 raven[flask]==6.10.0
-redis==2.10.6
+redis==3.2.1
 requests-toolbelt==0.9.1
 requests[security]==2.21.0
 retrying==1.3.3
@@ -86,13 +86,18 @@ s3transfer==0.2.0
 six==1.12.0
 snowballstemmer==1.2.1    # via sphinx
 sphinx-rtd-theme==0.4.3
-sphinx==1.8.5
+sphinx==2.0.1
+sphinxcontrib-applehelp==1.0.1  # via sphinx
+sphinxcontrib-devhelp==1.0.1  # via sphinx
+sphinxcontrib-htmlhelp==1.0.1  # via sphinx
 sphinxcontrib-httpdomain==1.7.0
-sphinxcontrib-websupport==1.1.0  # via sphinx
+sphinxcontrib-jsmath==1.0.1  # via sphinx
+sphinxcontrib-qthelp==1.0.2  # via sphinx
+sphinxcontrib-serializinghtml==1.1.3  # via sphinx
 sqlalchemy-utils==0.33.11
-sqlalchemy==1.3.1
+sqlalchemy==1.3.2
 tabulate==0.8.3
 urllib3==1.24.1
 vine==1.3.0
-werkzeug==0.15.1
+werkzeug==0.15.2
 xmltodict==0.12.0

requirements-tests.txt

@@ -8,9 +8,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.9.120            # via moto
+boto3==1.9.130            # via moto
 boto==2.49.0              # via moto
-botocore==1.12.120        # via boto3, moto, s3transfer
+botocore==1.12.130        # via boto3, moto, s3transfer
 certifi==2019.3.9         # via requests
 cffi==1.12.2              # via cryptography
 chardet==3.0.4            # via requests
@@ -18,7 +18,7 @@ click==7.0                # via flask
 coverage==4.5.3
 cryptography==2.6.1       # via moto
 docker-pycreds==0.4.0     # via docker
-docker==3.7.1             # via moto
+docker==3.7.2             # via moto
 docutils==0.14            # via botocore
 ecdsa==0.13               # via python-jose
 factory-boy==2.11.1
@@ -28,13 +28,13 @@ freezegun==0.3.11
 future==0.17.1            # via python-jose
 idna==2.8                 # via requests
 itsdangerous==1.1.0       # via flask
-jinja2==2.10              # via flask, moto
+jinja2==2.10.1            # via flask, moto
 jmespath==0.9.4           # via boto3, botocore
 jsondiff==1.1.1           # via moto
 jsonpickle==1.1           # via aws-xray-sdk
 markupsafe==1.1.1         # via jinja2
 mock==2.0.0               # via moto
-more-itertools==6.0.0     # via pytest
+more-itertools==7.0.0     # via pytest
 moto==1.3.7
 nose==1.3.7
 pbr==5.1.3                # via mock
@@ -42,14 +42,14 @@ pluggy==0.9.0             # via pytest
 py==1.8.0                 # via pytest
 pyaml==18.11.0            # via moto
 pycparser==2.19           # via cffi
-pycryptodome==3.8.0       # via python-jose
+pycryptodome==3.8.1       # via python-jose
 pyflakes==2.1.1
 pytest-flask==0.14.0
-pytest-mock==1.10.2
+pytest-mock==1.10.3
-pytest==4.3.1
+pytest==4.4.0
 python-dateutil==2.8.0    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
-pytz==2018.9              # via moto
+pytz==2019.1              # via moto
 pyyaml==5.1
 requests-mock==1.5.2
 requests==2.21.0          # via aws-xray-sdk, docker, moto, requests-mock, responses
@@ -59,6 +59,6 @@ six==1.12.0               # via cryptography, docker, docker-pycreds, faker, fre
 text-unidecode==1.2       # via faker
 urllib3==1.24.1           # via botocore, requests
 websocket-client==0.56.0  # via docker
-werkzeug==0.15.1          # via flask, moto, pytest-flask
+werkzeug==0.15.2          # via flask, moto, pytest-flask
 wrapt==1.11.1             # via aws-xray-sdk
 xmltodict==0.12.0         # via moto

requirements.in

@@ -27,7 +27,7 @@ gunicorn
 hvac # required for the vault destination plugin
 inflection
 jinja2
-kombu==4.3.0 # kombu 4.4.0 requires redis 3
+kombu
 lockfile
 marshmallow-sqlalchemy
 marshmallow
@@ -39,7 +39,7 @@ pyjwt
 pyOpenSSL
 python_ldap
 raven[flask]
-redis<3 # redis>=3 is not compatible with celery
+redis
 requests
 retrying
 six

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in -U --no-index
 #
-acme==0.32.0
+acme==0.33.1
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.8            # via flask-migrate
 amqp==2.4.2               # via kombu
@@ -13,11 +13,11 @@ arrow==0.13.1
 asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.6             # via flask-bcrypt, paramiko
-billiard==3.5.0.5         # via celery
+billiard==3.6.0.0         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.9.120
+boto3==1.9.130
-botocore==1.12.120
+botocore==1.12.130
-celery[redis]==4.2.2
+celery[redis]==4.3.0
 certifi==2019.3.9
 certsrv==2.1.1
 cffi==1.12.2              # via bcrypt, cryptography, pynacl
@@ -40,26 +40,26 @@ flask-sqlalchemy==2.3.2
 flask==1.0.2
 future==0.17.1
 gunicorn==19.9.0
-hvac==0.7.2
+hvac==0.8.2
 idna==2.8                 # via requests
 inflection==0.3.1
 itsdangerous==1.1.0       # via flask
-jinja2==2.10
+jinja2==2.10.1
 jmespath==0.9.4           # via boto3, botocore
 josepy==1.1.0             # via acme
 jsonlines==1.2.0          # via cloudflare
-kombu==4.3.0
+kombu==4.5.0
 lockfile==0.12.2
 mako==1.0.8               # via alembic
 markupsafe==1.1.1         # via jinja2, mako
 marshmallow-sqlalchemy==0.16.1
-marshmallow==2.19.1
+marshmallow==2.19.2
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
 paramiko==2.4.2
 pbr==5.1.3                # via mock
 pem==19.1.0
-psycopg2==2.7.7
+psycopg2==2.8.1
 pyasn1-modules==0.2.4     # via python-ldap
 pyasn1==0.4.5             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
 pycparser==2.19           # via cffi
@@ -70,19 +70,19 @@ pyrfc3339==1.1            # via acme
 python-dateutil==2.8.0    # via alembic, arrow, botocore
 python-editor==1.0.4      # via alembic
 python-ldap==3.2.0
-pytz==2018.9              # via acme, celery, flask-restful, pyrfc3339
+pytz==2019.1              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==5.1
 raven[flask]==6.10.0
-redis==2.10.6
+redis==3.2.1
 requests-toolbelt==0.9.1  # via acme
 requests[security]==2.21.0
 retrying==1.3.3
 s3transfer==0.2.0         # via boto3
 six==1.12.0
 sqlalchemy-utils==0.33.11
-sqlalchemy==1.3.1         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.3.2         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.3
 urllib3==1.24.1           # via botocore, requests
-vine==1.3.0               # via amqp
+vine==1.3.0               # via amqp, celery
-werkzeug==0.15.1          # via flask
+werkzeug==0.15.2          # via flask
 xmltodict==0.12.0

tox.ini

@@ -1,2 +1,2 @@
 [tox]
-envlist = py35
+envlist = py37