Merge pull request #2468 from hosseinsh/hosseinsh-celeryjob-sync-src-dst

Celery job for synching destinations and sources

lemur/common/celery.py

@@ -18,8 +18,11 @@ from lemur.authorities.service import get as get_authority
 from lemur.factory import create_app
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
-from lemur.plugins.base import plugins
+from lemur.plugins.base import plugins, IPlugin
 from lemur.sources.cli import clean, sync, validate_sources
+from lemur.destinations import service as destinations_service
+from lemur.sources import service as sources_service
+
 
 if current_app:
     flask_app = current_app
@@ -255,3 +258,35 @@ def sync_source(source):
     sync([source])
     log_data["message"] = "Done syncing source"
     current_app.logger.debug(log_data)
+
+
+@celery.task()
+def sync_source_destination():
+    """
+    This celery task will sync destination and source, to make sure all new destinations are also present as source.
+    Some destinations do not qualify as sources, and hence should be excluded from being added as sources
+    We identify qualified destinations based on the sync_as_source attributed of the plugin.
+    The destination sync_as_source_name reviels the name of the suitable source-plugin.
+    We rely on account numbers to avoid duplicates.
+    """
+    current_app.logger.debug("Syncing source and destination")
+
+    # a set of all accounts numbers available as sources
+    src_accounts = set()
+    sources = validate_sources("all")
+    for src in sources:
+        src_accounts.add(IPlugin.get_option('accountNumber', src.options))
+
+    for dst in destinations_service.get_all():
+        destination_plugin = plugins.get(dst.plugin_name)
+        account_number = IPlugin.get_option('accountNumber', dst.options)
+        if destination_plugin.sync_as_source and (account_number not in src_accounts):
+            src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
+            for o in src_options:
+                if o.get('name') == 'accountNumber':
+                    o.update({'value': account_number})
+            sources_service.create(label=dst.label,
+                                   plugin_name=destination_plugin.sync_as_source_name,
+                                   options=src_options,
+                                   description=dst.description)
+            current_app.logger.info("Source: %s added", dst.label)

lemur/plugins/bases/destination.py

@@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
 class DestinationPlugin(Plugin):
     type = 'destination'
     requires_key = True
+    sync_as_source = False
+    sync_as_source_name = ''
 
     def upload(self, name, body, private_key, cert_chain, options, **kwargs):
         raise NotImplementedError

lemur/plugins/lemur_aws/plugin.py

@@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
     return endpoints
 
 
-class AWSDestinationPlugin(DestinationPlugin):
-    title = 'AWS'
-    slug = 'aws-destination'
-    description = 'Allow the uploading of certificates to AWS IAM'
-    version = aws.VERSION
-
-    author = 'Kevin Glisson'
-    author_url = 'https://github.com/netflix/lemur'
-
-    options = [
-        {
-            'name': 'accountNumber',
-            'type': 'str',
-            'required': True,
-            'validation': '[0-9]{12}',
-            'helpMessage': 'Must be a valid AWS account number!',
-        },
-        {
-            'name': 'path',
-            'type': 'str',
-            'default': '/',
-            'helpMessage': 'Path to upload certificate.'
-        }
-    ]
-
-    # 'elb': {
-    #    'name': {'type': 'name'},
-    #    'region': {'type': 'str'},
-    #    'port': {'type': 'int'}
-    # }
-
-    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
-        iam.upload_cert(name, body, private_key,
-                        self.get_option('path', options),
-                        cert_chain=cert_chain,
-                        account_number=self.get_option('accountNumber', options))
-
-    def deploy(self, elb_name, account, region, certificate):
-        pass
-
-
 class AWSSourcePlugin(SourcePlugin):
     title = 'AWS'
     slug = 'aws-source'
@@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
         iam.delete_cert(certificate.name, account_number=account_number)
 
 
+class AWSDestinationPlugin(DestinationPlugin):
+    title = 'AWS'
+    slug = 'aws-destination'
+    description = 'Allow the uploading of certificates to AWS IAM'
+    version = aws.VERSION
+    sync_as_source = True
+    sync_as_source_name = AWSSourcePlugin.slug
+
+    author = 'Kevin Glisson'
+    author_url = 'https://github.com/netflix/lemur'
+
+    options = [
+        {
+            'name': 'accountNumber',
+            'type': 'str',
+            'required': True,
+            'validation': '[0-9]{12}',
+            'helpMessage': 'Must be a valid AWS account number!',
+        },
+        {
+            'name': 'path',
+            'type': 'str',
+            'default': '/',
+            'helpMessage': 'Path to upload certificate.'
+        }
+    ]
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        iam.upload_cert(name, body, private_key,
+                        self.get_option('path', options),
+                        cert_chain=cert_chain,
+                        account_number=self.get_option('accountNumber', options))
+
+    def deploy(self, elb_name, account, region, certificate):
+        pass
+
+
 class S3DestinationPlugin(ExportDestinationPlugin):
     title = 'AWS-S3'
     slug = 'aws-s3'