Merge branch 'master' into master

lemur/common/celery.py

@@ -20,6 +20,8 @@ from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.sources.cli import clean, sync, validate_sources
+from lemur.destinations import service as destinations_service
+from lemur.sources.service import add_aws_destination_to_sources
 
 if current_app:
     flask_app = current_app
@@ -255,3 +257,21 @@ def sync_source(source):
     sync([source])
     log_data["message"] = "Done syncing source"
     current_app.logger.debug(log_data)
+
+
+@celery.task()
+def sync_source_destination():
+    """
+    This celery task will sync destination and source, to make sure all new destinations are also present as source.
+    Some destinations do not qualify as sources, and hence should be excluded from being added as sources
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    """
+    current_app.logger.debug("Syncing AWS destinations and sources")
+
+    for dst in destinations_service.get_all():
+        if add_aws_destination_to_sources(dst):
+            current_app.logger.debug("Source: %s added", dst.label)
+
+    current_app.logger.debug("Completed Syncing AWS destinations and sources")

lemur/destinations/service.py

@@ -6,11 +6,13 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from sqlalchemy import func
+from flask import current_app
 
 from lemur import database
 from lemur.models import certificate_destination_associations
 from lemur.destinations.models import Destination
 from lemur.certificates.models import Certificate
+from lemur.sources.service import add_aws_destination_to_sources
 
 
 def create(label, plugin_name, options, description=None):
@@ -28,6 +30,12 @@ def create(label, plugin_name, options, description=None):
             del option['value']['plugin_object']
 
     destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
+    current_app.logger.info("Destination: %s created", label)
+
+    # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination
+    if add_aws_destination_to_sources(destination):
+        current_app.logger.info("Source: %s created", label)
+
     return database.create(destination)
 
 

lemur/manage.py

@@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy  # noqa
 from lemur.pending_certificates.models import PendingCertificate  # noqa
 from lemur.dns_providers.models import DnsProvider  # noqa
 
+from sqlalchemy.sql import text
+
 manager = Manager(create_app)
 manager.add_option('-c', '--config', dest='config_path', required=False)
 
@@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
 
 @MigrateCommand.command
 def create():
+    database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
     database.db.create_all()
     stamp(revision='head')
 

lemur/plugins/bases/destination.py

@@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
 class DestinationPlugin(Plugin):
     type = 'destination'
     requires_key = True
+    sync_as_source = False
+    sync_as_source_name = ''
 
     def upload(self, name, body, private_key, cert_chain, options, **kwargs):
         raise NotImplementedError

lemur/plugins/lemur_aws/plugin.py

@@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
     return endpoints
 
 
-class AWSDestinationPlugin(DestinationPlugin):
-    title = 'AWS'
-    slug = 'aws-destination'
-    description = 'Allow the uploading of certificates to AWS IAM'
-    version = aws.VERSION
-
-    author = 'Kevin Glisson'
-    author_url = 'https://github.com/netflix/lemur'
-
-    options = [
-        {
-            'name': 'accountNumber',
-            'type': 'str',
-            'required': True,
-            'validation': '[0-9]{12}',
-            'helpMessage': 'Must be a valid AWS account number!',
-        },
-        {
-            'name': 'path',
-            'type': 'str',
-            'default': '/',
-            'helpMessage': 'Path to upload certificate.'
-        }
-    ]
-
-    # 'elb': {
-    #    'name': {'type': 'name'},
-    #    'region': {'type': 'str'},
-    #    'port': {'type': 'int'}
-    # }
-
-    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
-        iam.upload_cert(name, body, private_key,
-                        self.get_option('path', options),
-                        cert_chain=cert_chain,
-                        account_number=self.get_option('accountNumber', options))
-
-    def deploy(self, elb_name, account, region, certificate):
-        pass
-
-
 class AWSSourcePlugin(SourcePlugin):
     title = 'AWS'
     slug = 'aws-source'
@@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
         iam.delete_cert(certificate.name, account_number=account_number)
 
 
+class AWSDestinationPlugin(DestinationPlugin):
+    title = 'AWS'
+    slug = 'aws-destination'
+    description = 'Allow the uploading of certificates to AWS IAM'
+    version = aws.VERSION
+    sync_as_source = True
+    sync_as_source_name = AWSSourcePlugin.slug
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur'
+
+    options = [
+        {
+            'name': 'accountNumber',
+            'type': 'str',
+            'required': True,
+            'validation': '[0-9]{12}',
+            'helpMessage': 'Must be a valid AWS account number!',
+        },
+        {
+            'name': 'path',
+            'type': 'str',
+            'default': '/',
+            'helpMessage': 'Path to upload certificate.'
+        }
+    ]
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        iam.upload_cert(name, body, private_key,
+                        self.get_option('path', options),
+                        cert_chain=cert_chain,
+                        account_number=self.get_option('accountNumber', options))
+
+    def deploy(self, elb_name, account, region, certificate):
+        pass
+
+
 class S3DestinationPlugin(ExportDestinationPlugin):
     title = 'AWS-S3'
     slug = 'aws-s3'

lemur/plugins/utils.py

@@ -18,4 +18,14 @@ def get_plugin_option(name, options):
     """
     for o in options:
         if o.get('name') == name:
-            return o['value']
+            return o.get('value', o.get('default'))
+
+
+def set_plugin_option(name, value, options):
+    """
+    Set value for option name for options dict.
+    :param options:
+    """
+    for o in options:
+        if o.get('name') == name:
+            o.update({'value': value})

lemur/sources/service.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+import copy
 
 from flask import current_app
 
@@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi
 from lemur.common.defaults import serial
 
 from lemur.plugins.base import plugins
+from lemur.plugins.utils import get_plugin_option, set_plugin_option
 
 
 def certificate_create(certificate, source):
@@ -256,3 +258,35 @@ def render(args):
         query = database.filter(query, Source, terms)
 
     return database.sort_and_page(query, Source, args)
+
+
+def add_aws_destination_to_sources(dst):
+    """
+    Given a destination check, if it can be added as sources, and included it if not already a source
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reveals the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    :return: true for success and false for not adding the destination as source
+    """
+    # a set of all accounts numbers available as sources
+    src_accounts = set()
+    sources = get_all()
+    for src in sources:
+        src_accounts.add(get_plugin_option('accountNumber', src.options))
+
+    # check
+    destination_plugin = plugins.get(dst.plugin_name)
+    account_number = get_plugin_option('accountNumber', dst.options)
+    if account_number is not None and \
+            destination_plugin.sync_as_source is not None and \
+            destination_plugin.sync_as_source and \
+            (account_number not in src_accounts):
+        src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
+        set_plugin_option('accountNumber', account_number, src_options)
+        create(label=dst.label,
+               plugin_name=destination_plugin.sync_as_source_name,
+               options=src_options,
+               description=dst.description)
+        return True
+
+    return False

lemur/tests/conftest.py

@@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from flask import current_app
 from flask_principal import identity_changed, Identity
+from sqlalchemy.sql import text
 
 from lemur import create_app
 from lemur.common.utils import parse_private_key
@@ -55,6 +56,7 @@ def app(request):
 @pytest.yield_fixture(scope="session")
 def db(app, request):
     _db.drop_all()
+    _db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
     _db.create_all()
 
     _db.app = app