diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 61dde28e..10747d31 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -18,11 +18,10 @@ from lemur.authorities.service import get as get_authority from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service -from lemur.plugins.base import plugins, IPlugin +from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources from lemur.destinations import service as destinations_service -from lemur.sources import service as sources_service - +from lemur.sources.service import add_aws_destination_to_sources if current_app: flask_app = current_app @@ -266,27 +265,13 @@ def sync_source_destination(): This celery task will sync destination and source, to make sure all new destinations are also present as source. Some destinations do not qualify as sources, and hence should be excluded from being added as sources We identify qualified destinations based on the sync_as_source attributed of the plugin. - The destination sync_as_source_name reviels the name of the suitable source-plugin. + The destination sync_as_source_name reveals the name of the suitable source-plugin. We rely on account numbers to avoid duplicates. """ - current_app.logger.debug("Syncing source and destination") - - # a set of all accounts numbers available as sources - src_accounts = set() - sources = validate_sources("all") - for src in sources: - src_accounts.add(IPlugin.get_option('accountNumber', src.options)) + current_app.logger.debug("Syncing AWS destinations and sources") for dst in destinations_service.get_all(): - destination_plugin = plugins.get(dst.plugin_name) - account_number = IPlugin.get_option('accountNumber', dst.options) - if destination_plugin.sync_as_source and (account_number not in src_accounts): - src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) - for o in src_options: - if o.get('name') == 'accountNumber': - o.update({'value': account_number}) - sources_service.create(label=dst.label, - plugin_name=destination_plugin.sync_as_source_name, - options=src_options, - description=dst.description) - current_app.logger.info("Source: %s added", dst.label) + if add_aws_destination_to_sources(dst): + current_app.logger.debug("Source: %s added", dst.label) + + current_app.logger.debug("Completed Syncing AWS destinations and sources") diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index ed6fcb0f..8e505fce 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -6,11 +6,13 @@ .. moduleauthor:: Kevin Glisson """ from sqlalchemy import func +from flask import current_app from lemur import database from lemur.models import certificate_destination_associations from lemur.destinations.models import Destination from lemur.certificates.models import Certificate +from lemur.sources.service import add_aws_destination_to_sources def create(label, plugin_name, options, description=None): @@ -28,6 +30,12 @@ def create(label, plugin_name, options, description=None): del option['value']['plugin_object'] destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description) + current_app.logger.info("Destination: %s created", label) + + # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination + if add_aws_destination_to_sources(destination): + current_app.logger.info("Source: %s created", label) + return database.create(destination) diff --git a/lemur/plugins/utils.py b/lemur/plugins/utils.py index a1914dd7..e057d071 100644 --- a/lemur/plugins/utils.py +++ b/lemur/plugins/utils.py @@ -18,4 +18,14 @@ def get_plugin_option(name, options): """ for o in options: if o.get('name') == name: - return o['value'] + return o.get('value', o.get('default')) + + +def set_plugin_option(name, value, options): + """ + Set value for option name for options dict. + :param options: + """ + for o in options: + if o.get('name') == name: + o.update({'value': value}) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 47b7f02c..a4d373ab 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -6,6 +6,7 @@ .. moduleauthor:: Kevin Glisson """ import arrow +import copy from flask import current_app @@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi from lemur.common.defaults import serial from lemur.plugins.base import plugins +from lemur.plugins.utils import get_plugin_option, set_plugin_option def certificate_create(certificate, source): @@ -256,3 +258,35 @@ def render(args): query = database.filter(query, Source, terms) return database.sort_and_page(query, Source, args) + + +def add_aws_destination_to_sources(dst): + """ + Given a destination check, if it can be added as sources, and included it if not already a source + We identify qualified destinations based on the sync_as_source attributed of the plugin. + The destination sync_as_source_name reveals the name of the suitable source-plugin. + We rely on account numbers to avoid duplicates. + :return: true for success and false for not adding the destination as source + """ + # a set of all accounts numbers available as sources + src_accounts = set() + sources = get_all() + for src in sources: + src_accounts.add(get_plugin_option('accountNumber', src.options)) + + # check + destination_plugin = plugins.get(dst.plugin_name) + account_number = get_plugin_option('accountNumber', dst.options) + if account_number is not None and \ + destination_plugin.sync_as_source is not None and \ + destination_plugin.sync_as_source and \ + (account_number not in src_accounts): + src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) + set_plugin_option('accountNumber', account_number, src_options) + create(label=dst.label, + plugin_name=destination_plugin.sync_as_source_name, + options=src_options, + description=dst.description) + return True + + return False