From 6d1ef933c45e643643329d8b60dcb6b67bc2e920 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 5 Feb 2019 10:48:52 -0800 Subject: [PATCH 1/8] creating a new celery task to sync sources with destinations. This is as a measure to make sure important new destinations are also present as sources. --- lemur/common/celery.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f2a2f826..308adced 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -20,6 +20,9 @@ from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources +from lemur.destinations import service as destinations_service +from lemur.sources import service as sources_service + if current_app: flask_app = current_app @@ -226,3 +229,19 @@ def sync_source(source): """ current_app.logger.debug("Syncing source {}".format(source)) sync([source]) + + +@celery.task() +def sync_source_destination(): + """ + This celery task will sync destination and source, to make sure all new destinations are also present in source. + Some destinations do not qualify as sources, and hence should be excluded from being added as sources + """ + current_app.logger.debug("Syncing source and destination") + for dst in destinations_service.get_all(): + if dst.plugin_name == 'aws-destination' and not sources_service.get_by_label(dst.label): + sources_service.create(label=dst.label, + plugin_name='aws-source', + options=dst.options, + description=dst.description) + current_app.logger.info("Source: %s added", dst.label) From d2e969b83648fc8d8317f0822522ab192dcd8983 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 21 Feb 2019 19:38:50 -0800 Subject: [PATCH 2/8] better synching of source and destinations --- lemur/plugins/bases/destination.py | 1 + lemur/plugins/lemur_aws/plugin.py | 1 + 2 files changed, 2 insertions(+) diff --git a/lemur/plugins/bases/destination.py b/lemur/plugins/bases/destination.py index 1e7e4ed2..04b01235 100644 --- a/lemur/plugins/bases/destination.py +++ b/lemur/plugins/bases/destination.py @@ -12,6 +12,7 @@ from lemur.plugins.base import Plugin, plugins class DestinationPlugin(Plugin): type = 'destination' requires_key = True + sync_as_source = False def upload(self, name, body, private_key, cert_chain, options, **kwargs): raise NotImplementedError diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 1c2607a5..d3c58464 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -195,6 +195,7 @@ class AWSSourcePlugin(SourcePlugin): slug = 'aws-source' description = 'Discovers all SSL certificates and ELB endpoints in an AWS account' version = aws.VERSION + sync_as_source = True author = 'Kevin Glisson' author_url = 'https://github.com/netflix/lemur' From f3d0536800d7db899c38e3bc32ee86fee931c0d3 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Apr 2019 20:49:07 -0700 Subject: [PATCH 3/8] removing hardcoded rules, to give more flexibility into defining new source-destinations --- lemur/common/celery.py | 7 ++++--- lemur/plugins/bases/destination.py | 1 + lemur/plugins/lemur_aws/plugin.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 308adced..c926b390 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -234,14 +234,15 @@ def sync_source(source): @celery.task() def sync_source_destination(): """ - This celery task will sync destination and source, to make sure all new destinations are also present in source. + This celery task will sync destination and source, to make sure all new destinations are also present as source. Some destinations do not qualify as sources, and hence should be excluded from being added as sources """ current_app.logger.debug("Syncing source and destination") for dst in destinations_service.get_all(): - if dst.plugin_name == 'aws-destination' and not sources_service.get_by_label(dst.label): + destination_plugin = plugins.get(dst.plugin_name) + if destination_plugin.sync_as_source and not sources_service.get_by_label(dst.label): sources_service.create(label=dst.label, - plugin_name='aws-source', + plugin_name=destination_plugin.sync_as_source_name, options=dst.options, description=dst.description) current_app.logger.info("Source: %s added", dst.label) diff --git a/lemur/plugins/bases/destination.py b/lemur/plugins/bases/destination.py index 04b01235..fc73ebcb 100644 --- a/lemur/plugins/bases/destination.py +++ b/lemur/plugins/bases/destination.py @@ -13,6 +13,7 @@ class DestinationPlugin(Plugin): type = 'destination' requires_key = True sync_as_source = False + sync_as_source_name = '' def upload(self, name, body, private_key, cert_chain, options, **kwargs): raise NotImplementedError diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index d3c58464..2f271296 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -154,6 +154,8 @@ class AWSDestinationPlugin(DestinationPlugin): slug = 'aws-destination' description = 'Allow the uploading of certificates to AWS IAM' version = aws.VERSION + sync_as_source = True + sync_as_source_name = 'aws-source' author = 'Kevin Glisson' author_url = 'https://github.com/netflix/lemur' @@ -195,7 +197,6 @@ class AWSSourcePlugin(SourcePlugin): slug = 'aws-source' description = 'Discovers all SSL certificates and ELB endpoints in an AWS account' version = aws.VERSION - sync_as_source = True author = 'Kevin Glisson' author_url = 'https://github.com/netflix/lemur' From f185df4f1e38e6e7b682bd11b7d1184382ea6c45 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 13:28:58 -0700 Subject: [PATCH 4/8] bringing class AWSDestinationPlugin(DestinationPlugin) after AWSSourcePlugin.slug, such that we can do: sync_as_source_name = AWSSourcePlugin.slug --- lemur/plugins/lemur_aws/plugin.py | 86 +++++++++++++++---------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 2f271296..57cc831c 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -149,49 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict): return endpoints -class AWSDestinationPlugin(DestinationPlugin): - title = 'AWS' - slug = 'aws-destination' - description = 'Allow the uploading of certificates to AWS IAM' - version = aws.VERSION - sync_as_source = True - sync_as_source_name = 'aws-source' - - author = 'Kevin Glisson' - author_url = 'https://github.com/netflix/lemur' - - options = [ - { - 'name': 'accountNumber', - 'type': 'str', - 'required': True, - 'validation': '[0-9]{12}', - 'helpMessage': 'Must be a valid AWS account number!', - }, - { - 'name': 'path', - 'type': 'str', - 'default': '/', - 'helpMessage': 'Path to upload certificate.' - } - ] - - # 'elb': { - # 'name': {'type': 'name'}, - # 'region': {'type': 'str'}, - # 'port': {'type': 'int'} - # } - - def upload(self, name, body, private_key, cert_chain, options, **kwargs): - iam.upload_cert(name, body, private_key, - self.get_option('path', options), - cert_chain=cert_chain, - account_number=self.get_option('accountNumber', options)) - - def deploy(self, elb_name, account, region, certificate): - pass - - class AWSSourcePlugin(SourcePlugin): title = 'AWS' slug = 'aws-source' @@ -268,6 +225,49 @@ class AWSSourcePlugin(SourcePlugin): iam.delete_cert(certificate.name, account_number=account_number) +class AWSDestinationPlugin(DestinationPlugin): + title = 'AWS' + slug = 'aws-destination' + description = 'Allow the uploading of certificates to AWS IAM' + version = aws.VERSION + sync_as_source = True + sync_as_source_name = AWSSourcePlugin.slug + + author = 'Kevin Glisson' + author_url = 'https://github.com/netflix/lemur' + + options = [ + { + 'name': 'accountNumber', + 'type': 'str', + 'required': True, + 'validation': '[0-9]{12}', + 'helpMessage': 'Must be a valid AWS account number!', + }, + { + 'name': 'path', + 'type': 'str', + 'default': '/', + 'helpMessage': 'Path to upload certificate.' + } + ] + + # 'elb': { + # 'name': {'type': 'name'}, + # 'region': {'type': 'str'}, + # 'port': {'type': 'int'} + # } + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + iam.upload_cert(name, body, private_key, + self.get_option('path', options), + cert_chain=cert_chain, + account_number=self.get_option('accountNumber', options)) + + def deploy(self, elb_name, account, region, certificate): + pass + + class S3DestinationPlugin(ExportDestinationPlugin): title = 'AWS-S3' slug = 'aws-s3' From 266c83367d81e563fa4984206ca1b5bd472527c4 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 13:29:37 -0700 Subject: [PATCH 5/8] avoiding hard-coded plugin names --- lemur/common/celery.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index ed751d9b..65114e01 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -18,7 +18,7 @@ from lemur.authorities.service import get as get_authority from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service -from lemur.plugins.base import plugins +from lemur.plugins.base import plugins, IPlugin from lemur.sources.cli import clean, sync, validate_sources from lemur.destinations import service as destinations_service from lemur.sources import service as sources_service @@ -265,13 +265,31 @@ def sync_source_destination(): """ This celery task will sync destination and source, to make sure all new destinations are also present as source. Some destinations do not qualify as sources, and hence should be excluded from being added as sources + We identify qualified destinations based on the sync_as_source attributed of the plugin. + The destination sync_as_source_name reviels the name of the suitable source-plugin. + We rely on account numbers to avoid duplicates. """ current_app.logger.debug("Syncing source and destination") + + # a set of all accounts numbers available as sources + src_accounts = set() + sources = validate_sources("all") + for src in sources: + src_accounts.add(IPlugin.get_option('accountNumber' ,src.options)) + for dst in destinations_service.get_all(): destination_plugin = plugins.get(dst.plugin_name) - if destination_plugin.sync_as_source and not sources_service.get_by_label(dst.label): + account_number = IPlugin.get_option('accountNumber', src.options) + if destination_plugin.sync_as_source and (account_number not in src_accounts): + src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) + for o in src_options: + if o.get('name') == 'accountNumber': + o.update({'value': account_number}) + sources_service.create(label=dst.label, plugin_name=destination_plugin.sync_as_source_name, - options=dst.options, + options=src_options, description=dst.description) current_app.logger.info("Source: %s added", dst.label) + + From ec3d2d73162b8b84de0de5b6fb1d72a85c210904 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 13:51:21 -0700 Subject: [PATCH 6/8] fixing typo --- lemur/common/celery.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 65114e01..fdac27eb 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -275,17 +275,16 @@ def sync_source_destination(): src_accounts = set() sources = validate_sources("all") for src in sources: - src_accounts.add(IPlugin.get_option('accountNumber' ,src.options)) + src_accounts.add(IPlugin.get_option('accountNumber', src.options)) for dst in destinations_service.get_all(): destination_plugin = plugins.get(dst.plugin_name) - account_number = IPlugin.get_option('accountNumber', src.options) + account_number = IPlugin.get_option('accountNumber', dst.options) if destination_plugin.sync_as_source and (account_number not in src_accounts): src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options) for o in src_options: if o.get('name') == 'accountNumber': o.update({'value': account_number}) - sources_service.create(label=dst.label, plugin_name=destination_plugin.sync_as_source_name, options=src_options, From 60edab9f6db11861c059afd6fc2535b758c87cf5 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 14:12:31 -0700 Subject: [PATCH 7/8] cleaning up --- lemur/plugins/lemur_aws/plugin.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 57cc831c..41bec31c 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -252,12 +252,6 @@ class AWSDestinationPlugin(DestinationPlugin): } ] - # 'elb': { - # 'name': {'type': 'name'}, - # 'region': {'type': 'str'}, - # 'port': {'type': 'int'} - # } - def upload(self, name, body, private_key, cert_chain, options, **kwargs): iam.upload_cert(name, body, private_key, self.get_option('path', options), From 245923414741772d16d2ae1c79ef26fa401f75c3 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Apr 2019 14:34:26 -0700 Subject: [PATCH 8/8] removing lines --- lemur/common/celery.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index fdac27eb..61dde28e 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -290,5 +290,3 @@ def sync_source_destination(): options=src_options, description=dst.description) current_app.logger.info("Source: %s added", dst.label) - -