Merge pull request #2468 from hosseinsh/hosseinsh-celeryjob-sync-src-dst

Celery job for synching destinations and sources
This commit is contained in:
Hossein Shafagh 2019-04-11 15:17:16 -07:00 committed by GitHub
commit 81e2ff99ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 75 additions and 42 deletions

View File

@ -18,8 +18,11 @@ from lemur.authorities.service import get as get_authority
from lemur.factory import create_app from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins from lemur.plugins.base import plugins, IPlugin
from lemur.sources.cli import clean, sync, validate_sources from lemur.sources.cli import clean, sync, validate_sources
from lemur.destinations import service as destinations_service
from lemur.sources import service as sources_service
if current_app: if current_app:
flask_app = current_app flask_app = current_app
@ -255,3 +258,35 @@ def sync_source(source):
sync([source]) sync([source])
log_data["message"] = "Done syncing source" log_data["message"] = "Done syncing source"
current_app.logger.debug(log_data) current_app.logger.debug(log_data)
@celery.task()
def sync_source_destination():
"""
This celery task will sync destination and source, to make sure all new destinations are also present as source.
Some destinations do not qualify as sources, and hence should be excluded from being added as sources
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reviels the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
"""
current_app.logger.debug("Syncing source and destination")
# a set of all accounts numbers available as sources
src_accounts = set()
sources = validate_sources("all")
for src in sources:
src_accounts.add(IPlugin.get_option('accountNumber', src.options))
for dst in destinations_service.get_all():
destination_plugin = plugins.get(dst.plugin_name)
account_number = IPlugin.get_option('accountNumber', dst.options)
if destination_plugin.sync_as_source and (account_number not in src_accounts):
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
for o in src_options:
if o.get('name') == 'accountNumber':
o.update({'value': account_number})
sources_service.create(label=dst.label,
plugin_name=destination_plugin.sync_as_source_name,
options=src_options,
description=dst.description)
current_app.logger.info("Source: %s added", dst.label)

View File

@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
class DestinationPlugin(Plugin): class DestinationPlugin(Plugin):
type = 'destination' type = 'destination'
requires_key = True requires_key = True
sync_as_source = False
sync_as_source_name = ''
def upload(self, name, body, private_key, cert_chain, options, **kwargs): def upload(self, name, body, private_key, cert_chain, options, **kwargs):
raise NotImplementedError raise NotImplementedError

View File

@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
return endpoints return endpoints
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
# 'elb': {
# 'name': {'type': 'name'},
# 'region': {'type': 'str'},
# 'port': {'type': 'int'}
# }
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class AWSSourcePlugin(SourcePlugin): class AWSSourcePlugin(SourcePlugin):
title = 'AWS' title = 'AWS'
slug = 'aws-source' slug = 'aws-source'
@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
iam.delete_cert(certificate.name, account_number=account_number) iam.delete_cert(certificate.name, account_number=account_number)
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
sync_as_source = True
sync_as_source_name = AWSSourcePlugin.slug
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class S3DestinationPlugin(ExportDestinationPlugin): class S3DestinationPlugin(ExportDestinationPlugin):
title = 'AWS-S3' title = 'AWS-S3'
slug = 'aws-s3' slug = 'aws-s3'