From 44a060b15932a160b1ee8a676a8af8ae40e976b1 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 4 Feb 2019 15:36:39 -0800 Subject: [PATCH 01/16] adding support for creating a source while creating a new dst, while the destination is from AWS --- lemur/destinations/service.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index ed6fcb0f..705f45e7 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -6,11 +6,13 @@ .. moduleauthor:: Kevin Glisson """ from sqlalchemy import func +from flask import current_app from lemur import database from lemur.models import certificate_destination_associations from lemur.destinations.models import Destination from lemur.certificates.models import Certificate +from lemur.sources import service as sources_service def create(label, plugin_name, options, description=None): @@ -28,6 +30,13 @@ def create(label, plugin_name, options, description=None): del option['value']['plugin_object'] destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description) + current_app.logger.info("Destination: %s created", label) + + # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination + if plugin_name == 'aws-destination': + sources_service.create(label=label, plugin_name=plugin_name, options=options, description=description) + current_app.logger.info("Source: %s created", label) + return database.create(destination) From f249a82d71ac70f146958fa9c1da1d93a911e43d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 4 Feb 2019 16:10:48 -0800 Subject: [PATCH 02/16] renaming destination to source. --- lemur/destinations/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 705f45e7..94ca3977 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -34,7 +34,7 @@ def create(label, plugin_name, options, description=None): # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination if plugin_name == 'aws-destination': - sources_service.create(label=label, plugin_name=plugin_name, options=options, description=description) + sources_service.create(label=label, plugin_name='aws-source', options=options, description=description) current_app.logger.info("Source: %s created", label) return database.create(destination) From 1bda246df2ae678cd04ba504a89c7a84e534ac65 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 12 Mar 2019 14:17:53 -0700 Subject: [PATCH 03/16] simple hardcoded announcement --- lemur/static/app/index.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index 466cfe9e..fcc54069 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,6 +89,11 @@ +
+ × + Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! +
+
From b66fac049445d5dfab67953343e20149ed5911fb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Apr 2019 10:08:25 -0700 Subject: [PATCH 04/16] removing the announcement --- lemur/static/app/index.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index fcc54069..466cfe9e 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,11 +89,6 @@
-
- × - Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! -
-
From ba691a26d48e503029e058a8bee69b80f06391fd Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 12 Mar 2019 14:17:53 -0700 Subject: [PATCH 05/16] simple hardcoded announcement --- lemur/static/app/index.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index 466cfe9e..fcc54069 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,6 +89,11 @@
+
+ × + Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! +
+
From 84dfdd0600368b81e740d7e2a55afa32bf2bafd7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Apr 2019 10:08:25 -0700 Subject: [PATCH 06/16] removing the announcement --- lemur/static/app/index.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index fcc54069..466cfe9e 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,11 +89,6 @@
-
- × - Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! -
-
From e1a67e9b4eb90bc8e26b29f20709a5c6d88525da Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 12 Mar 2019 14:17:53 -0700 Subject: [PATCH 07/16] simple hardcoded announcement --- lemur/static/app/index.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index 466cfe9e..fcc54069 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,6 +89,11 @@
+
+ × + Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! +
+
From 818da6653d5a0105bb8528ca08c882672ffd0501 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Apr 2019 10:08:25 -0700 Subject: [PATCH 08/16] removing the announcement --- lemur/static/app/index.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index fcc54069..466cfe9e 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,11 +89,6 @@
-
- × - Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! -
-
From 5900828051298656fc8cd2de419de89788deec18 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 12 Mar 2019 14:17:53 -0700 Subject: [PATCH 09/16] simple hardcoded announcement --- lemur/static/app/index.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index 466cfe9e..fcc54069 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,6 +89,11 @@
+
+ × + Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! +
+
From d1ead4b79c62df629970bdb9f0d107e7e79c8097 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Apr 2019 10:08:25 -0700 Subject: [PATCH 10/16] removing the announcement --- lemur/static/app/index.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index fcc54069..466cfe9e 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -89,11 +89,6 @@
-
- × - Info: Digicert maintenance and downtime scheduled for 6 April 2019 from 8:30 AM to 8:30 PM Pacific Time! -
-
From 557fac39b58e565e6485aaa68fb1e66962a63b7e Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 16:37:31 -0700 Subject: [PATCH 11/16] refactoring the sync job into a service method that we can also call when adding a new destination --- lemur/common/celery.py | 27 ++++++--------------------- lemur/sources/service.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 61dde28e..aa160b15 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -21,8 +21,7 @@ from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins, IPlugin from lemur.sources.cli import clean, sync, validate_sources from lemur.destinations import service as destinations_service -from lemur.sources import service as sources_service - +from lemur.sources.service import add_aws_destination_to_sources if current_app: flask_app = current_app @@ -269,24 +268,10 @@ def sync_source_destination(): The destination sync_as_source_name reviels the name of the suitable source-plugin. We rely on account numbers to avoid duplicates. """ - current_app.logger.debug("Syncing source and destination") - - # a set of all accounts numbers available as sources - src_accounts = set() - sources = validate_sources("all") - for src in sources: - src_accounts.add(IPlugin.get_option('accountNumber', src.options)) + current_app.logger.debug("Syncing AWWS destinations and sources") for dst in destinations_service.get_all(): - destination_plugin = plugins.get(dst.plugin_name) - account_number = IPlugin.get_option('accountNumber', dst.options) - if destination_plugin.sync_as_source and (account_number not in src_accounts): - src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) - for o in src_options: - if o.get('name') == 'accountNumber': - o.update({'value': account_number}) - sources_service.create(label=dst.label, - plugin_name=destination_plugin.sync_as_source_name, - options=src_options, - description=dst.description) - current_app.logger.info("Source: %s added", dst.label) + if add_aws_destination_to_sources(dst): + current_app.logger.debug("Source: %s added", dst.label) + + current_app.logger.debug("Completed Syncing AWS destinations and sources") diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 47b7f02c..31886b5f 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -6,6 +6,7 @@ .. moduleauthor:: Kevin Glisson """ import arrow +import copy from flask import current_app @@ -21,6 +22,7 @@ from lemur.common.utils import find_matching_certificates_by_hash, parse_certifi from lemur.common.defaults import serial from lemur.plugins.base import plugins +from lemur.plugins.utils import get_plugin_option, set_plugin_option def certificate_create(certificate, source): @@ -256,3 +258,32 @@ def render(args): query = database.filter(query, Source, terms) return database.sort_and_page(query, Source, args) + + +def add_aws_destination_to_sources(dst): + """ + Given a destination check, if it can be added as sources, and included it if not already a source + We identify qualified destinations based on the sync_as_source attributed of the plugin. + The destination sync_as_source_name reveals the name of the suitable source-plugin. + We rely on account numbers to avoid duplicates. + :return: true for success and false for not adding the destination as source + """ + # a set of all accounts numbers available as sources + src_accounts = set() + sources = get_all() + for src in sources: + src_accounts.add(get_plugin_option('accountNumber', src.options)) + + # check + destination_plugin = plugins.get(dst.plugin_name) + account_number = get_plugin_option('accountNumber', dst.options) + if destination_plugin.sync_as_source and (account_number not in src_accounts): + src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) + set_plugin_option('accountNumber', account_number, src_options) + create(label=dst.label, + plugin_name=destination_plugin.sync_as_source_name, + options=src_options, + description=dst.description) + return True + + return False From d7abf2ec18093d566d9c18edef57995309272da2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 16:38:00 -0700 Subject: [PATCH 12/16] adding a new util method for setting options --- lemur/plugins/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/utils.py b/lemur/plugins/utils.py index a1914dd7..e057d071 100644 --- a/lemur/plugins/utils.py +++ b/lemur/plugins/utils.py @@ -18,4 +18,14 @@ def get_plugin_option(name, options): """ for o in options: if o.get('name') == name: - return o['value'] + return o.get('value', o.get('default')) + + +def set_plugin_option(name, value, options): + """ + Set value for option name for options dict. + :param options: + """ + for o in options: + if o.get('name') == name: + o.update({'value': value}) From 69c00c4db57efa54d14ba182dd00309d132cac1e Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 16:39:47 -0700 Subject: [PATCH 13/16] upon creating a new destination, we also add it as source, if the plugin defines this as an option --- lemur/destinations/service.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 94ca3977..8e505fce 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -12,7 +12,7 @@ from lemur import database from lemur.models import certificate_destination_associations from lemur.destinations.models import Destination from lemur.certificates.models import Certificate -from lemur.sources import service as sources_service +from lemur.sources.service import add_aws_destination_to_sources def create(label, plugin_name, options, description=None): @@ -33,8 +33,7 @@ def create(label, plugin_name, options, description=None): current_app.logger.info("Destination: %s created", label) # add the destination as source, to avoid new destinations that are not in source, as long as an AWS destination - if plugin_name == 'aws-destination': - sources_service.create(label=label, plugin_name='aws-source', options=options, description=description) + if add_aws_destination_to_sources(destination): current_app.logger.info("Source: %s created", label) return database.create(destination) From 6ec84a398c16787771d2c5547a30269e28755900 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 17:13:37 -0700 Subject: [PATCH 14/16] checking for None --- lemur/sources/service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 31886b5f..a4d373ab 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -277,7 +277,10 @@ def add_aws_destination_to_sources(dst): # check destination_plugin = plugins.get(dst.plugin_name) account_number = get_plugin_option('accountNumber', dst.options) - if destination_plugin.sync_as_source and (account_number not in src_accounts): + if account_number is not None and \ + destination_plugin.sync_as_source is not None and \ + destination_plugin.sync_as_source and \ + (account_number not in src_accounts): src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) set_plugin_option('accountNumber', account_number, src_options) create(label=dst.label, From 512e1a0bdda0b60974f8826f930979a4e00856f7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 17:17:28 -0700 Subject: [PATCH 15/16] fixing typos --- lemur/common/celery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index aa160b15..4192eb10 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -265,10 +265,10 @@ def sync_source_destination(): This celery task will sync destination and source, to make sure all new destinations are also present as source. Some destinations do not qualify as sources, and hence should be excluded from being added as sources We identify qualified destinations based on the sync_as_source attributed of the plugin. - The destination sync_as_source_name reviels the name of the suitable source-plugin. + The destination sync_as_source_name reveals the name of the suitable source-plugin. We rely on account numbers to avoid duplicates. """ - current_app.logger.debug("Syncing AWWS destinations and sources") + current_app.logger.debug("Syncing AWS destinations and sources") for dst in destinations_service.get_all(): if add_aws_destination_to_sources(dst): From 6d67ec7e3462b415af1d1ad1b99c94b27ac0a7c8 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 17:34:02 -0700 Subject: [PATCH 16/16] removing unused import --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 4192eb10..10747d31 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -18,7 +18,7 @@ from lemur.authorities.service import get as get_authority from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service -from lemur.plugins.base import plugins, IPlugin +from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources from lemur.destinations import service as destinations_service from lemur.sources.service import add_aws_destination_to_sources