diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 932b84ca..2e4b5ddc 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -68,14 +68,15 @@ def get_all_certs(): return Certificate.query.all() -def get_by_source(source_label): +def get_all_pending_cleaning(source): """ - Retrieves all certificates from a given source. + Retrieves all certificates that are available for cleaning. - :param source_label: + :param source: :return: """ - return Certificate.query.filter(Certificate.sources.any(label=source_label)) + return Certificate.query.filter(Certificate.sources.any(id=source.id))\ + .filter(not_(Certificate.endpoints.any())).all() def get_all_pending_reissue(): diff --git a/lemur/plugins/bases/source.py b/lemur/plugins/bases/source.py index 83814066..e2283fe9 100644 --- a/lemur/plugins/bases/source.py +++ b/lemur/plugins/bases/source.py @@ -28,7 +28,7 @@ class SourcePlugin(Plugin): def get_endpoints(self, options, **kwargs): raise NotImplementedError - def clean(self, options, **kwargs): + def clean(self, certificate, options, **kwargs): raise NotImplementedError @property diff --git a/lemur/plugins/lemur_aws/elb.py b/lemur/plugins/lemur_aws/elb.py index 0e6c78c6..a066ffb4 100644 --- a/lemur/plugins/lemur_aws/elb.py +++ b/lemur/plugins/lemur_aws/elb.py @@ -16,14 +16,14 @@ from lemur.plugins.lemur_aws.sts import sts_client def retry_throttled(exception): """ - Determiens if this exception is due to throttling + Determines if this exception is due to throttling :param exception: :return: """ if isinstance(exception, botocore.exceptions.ClientError): if exception.response['Error']['Code'] == 'LoadBalancerNotFound': - return True - return False + return False + return True def is_valid(listener_tuple): diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 2051c5b2..98433f89 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -6,7 +6,24 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +import botocore + +from retrying import retry + from lemur.plugins.lemur_aws.sts import assume_service +from lemur.plugins.lemur_aws.sts import sts_client + + +def retry_throttled(exception): + """ + Determines if this exception is due to throttling + :param exception: + :return: + """ + if isinstance(exception, botocore.exceptions.ClientError): + if exception.response['Error']['Code'] == 'NoSuchEntity': + return False + return True def get_name_from_arn(arn): @@ -33,15 +50,17 @@ def upload_cert(account_number, name, body, private_key, cert_chain=None): cert_chain=str(cert_chain)) -def delete_cert(account_number, cert_name): +@sts_client('iam') +@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +def delete_cert(cert_name, **kwargs): """ Delete a certificate from AWS - :param account_number: :param cert_name: :return: """ - return assume_service(account_number, 'iam').delete_server_cert(cert_name) + client = kwargs.pop('client') + client.delete_server_certificate(ServerCertificateName=cert_name) def get_all_server_certs(account_number): diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 9d719e3d..96652945 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -261,21 +261,9 @@ class AWSSourcePlugin(SourcePlugin): else: elb.attach_certificate(endpoint.name, endpoint.port, arn, account_number=account_number, region=region) - def clean(self, options, **kwargs): + def clean(self, certificate, options, **kwargs): account_number = self.get_option('accountNumber', options) - certificates = self.get_certificates(options) - endpoints = self.get_endpoints(options) - - orphaned = [] - for certificate in certificates: - for endpoint in endpoints: - if certificate['name'] == endpoint['certificate_name']: - break - else: - orphaned.append(certificate['name']) - iam.delete_cert(account_number, certificate) - - return orphaned + iam.delete_cert(certificate.name, account_number=account_number) class S3DestinationPlugin(DestinationPlugin): diff --git a/lemur/plugins/lemur_aws/sts.py b/lemur/plugins/lemur_aws/sts.py index 046d6481..79c055cb 100644 --- a/lemur/plugins/lemur_aws/sts.py +++ b/lemur/plugins/lemur_aws/sts.py @@ -56,6 +56,7 @@ def sts_client(service, service_type='client'): kwargs.pop('account_number'), current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur') ) + # TODO add user specific information to RoleSessionName role = sts.assume_role(RoleArn=arn, RoleSessionName='lemur') diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index 77b9231f..78c5a0b1 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -15,8 +15,12 @@ from flask_script import Manager from flask import current_app from lemur.extensions import metrics +from lemur.plugins.base import plugins + from lemur.sources import service as source_service from lemur.users import service as user_service +from lemur.certificates import service as certificate_service + manager = Manager(usage="Handles all source related tasks.") @@ -48,8 +52,8 @@ def validate_sources(source_strings): @manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') def sync(source_strings): - source_objs = validate_sources(source_strings) - for source in source_objs: + sources = validate_sources(source_strings) + for source in sources: start_time = time.time() print("[+] Staring to sync source: {label}!\n".format(label=source.label)) @@ -86,15 +90,45 @@ def sync(source_strings): @manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') -def clean(source_strings): - source_objs = validate_sources(source_strings) - for source in source_objs: +@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.') +def clean(source_strings, commit): + sources = validate_sources(source_strings) + for source in sources: + s = plugins.get(source.plugin_name) + + if not hasattr(s, 'clean'): + print("Cannot clean source: {0}, source plugin does not implement 'clean()'".format( + source.label + )) + continue + start_time = time.time() + print("[+] Staring to clean source: {label}!\n".format(label=source.label)) - source_service.clean(source) + + cleaned = 0 + for certificate in certificate_service.get_all_pending_cleaning(source): + if commit: + try: + s.clean(certificate, source.options) + certificate.sources.remove(source) + certificate_service.database.update(certificate) + metrics.send('clean_success', 'counter', 1, metric_tags={'source': source.label}) + except Exception as e: + current_app.logger.exception(e) + metrics.send('clean_failed', 'counter', 1, metric_tags={'source': source.label}) + + current_app.logger.warning("Removed {0} from source {1} during cleaning".format( + certificate.name, + source.label + )) + + cleaned += 1 + print( - "[+] Finished cleaning source: {label}. Run Time: {time}\n".format( + "[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format( label=source.label, - time=(time.time() - start_time) + time=(time.time() - start_time), + cleaned=cleaned ) ) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index d58af8c7..6f1f4dbe 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -12,7 +12,7 @@ from flask import current_app from lemur import database from lemur.sources.models import Source from lemur.certificates.models import Certificate -from lemur.certificates import service as cert_service +from lemur.certificates import service as certificate_service from lemur.endpoints import service as endpoint_service from lemur.destinations import service as destination_service @@ -21,29 +21,6 @@ from lemur.certificates.schemas import CertificateUploadInputSchema from lemur.plugins.base import plugins -# TODO optimize via sql query -def _disassociate_certs_from_source(certificates, source): - current_certificates = cert_service.get_by_source(source_label=source.label) - missing = [] - for cc in current_certificates: - for fc in certificates: - if fc['body'] == cc.body: - break - else: - missing.append(cc) - - for c in missing: - for s in c.sources: - if s.label == source: - current_app.logger.info( - "Certificate {name} is no longer associated with {source}.".format( - name=c.name, - source=source.label - ) - ) - c.sources.delete(s) - - def certificate_create(certificate, source): data, errors = CertificateUploadInputSchema().load(certificate) @@ -52,7 +29,7 @@ def certificate_create(certificate, source): data['creator'] = certificate['creator'] - cert = cert_service.import_certificate(**data) + cert = certificate_service.import_certificate(**data) cert.description = "This certificate was automatically discovered by Lemur" cert.sources.append(source) sync_update_destination(cert, source) @@ -99,12 +76,12 @@ def sync_endpoints(source): certificate = endpoint.pop('certificate', None) if certificate_name: - cert = cert_service.get_by_name(certificate_name) + cert = certificate_service.get_by_name(certificate_name) elif certificate: - cert = cert_service.find_duplicates(certificate) + cert = certificate_service.find_duplicates(certificate) if not cert: - cert = cert_service.import_certificate(**certificate) + cert = certificate_service.import_certificate(**certificate) if not cert: current_app.logger.error( @@ -142,7 +119,7 @@ def sync_certificates(source, user): certificates = s.get_certificates(source.options) for certificate in certificates: - exists = cert_service.find_duplicates(certificate) + exists = certificate_service.find_duplicates(certificate) certificate['owner'] = user.email certificate['creator'] = user @@ -162,9 +139,6 @@ def sync_certificates(source, user): ) ) - # we need to try and find the absent of certificates so we can properly disassociate them when they are deleted - _disassociate_certs_from_source(certificates, source) - return new, updated @@ -178,33 +152,13 @@ def sync(source, user): return {'endpoints': (new_endpoints, updated_endpoints), 'certificates': (new_certs, updated_certs)} -def clean(source): - s = plugins.get(source.plugin_name) - - try: - certificates = s.clean(source.options) - except NotImplementedError: - current_app.logger.warning("Cannot clean source: {0}, source plugin does not implement 'clean()'".format( - source.label - )) - return - - for certificate in certificates: - cert = cert_service.get_by_name(certificate) - - if cert: - current_app.logger.warning("Removed {0} from source {1} during cleaning".format( - cert.name, - source.label - )) - cert.sources.remove(source) - - def create(label, plugin_name, options, description=None): """ Creates a new source, that can then be used as a source for certificates. :param label: Source common name + :param plugin_name: + :param options: :param description: :rtype : Source :return: New source @@ -219,6 +173,8 @@ def update(source_id, label, options, description): :param source_id: Lemur assigned ID :param label: Source common name + :param options: + :param description: :rtype : Source :return: """