Clean refactor (#635)

* Adding rotation to the UI.

* Removing spinkit dependency.

* refactoring source cleaning

lemur/certificates/service.py

@@ -68,14 +68,15 @@ def get_all_certs():
     return Certificate.query.all()
 
 
-def get_by_source(source_label):
+def get_all_pending_cleaning(source):
     """
-    Retrieves all certificates from a given source.
+    Retrieves all certificates that are available for cleaning.
 
-    :param source_label:
+    :param source:
     :return:
     """
-    return Certificate.query.filter(Certificate.sources.any(label=source_label))
+    return Certificate.query.filter(Certificate.sources.any(id=source.id))\
+                            .filter(not_(Certificate.endpoints.any())).all()
 
 
 def get_all_pending_reissue():

lemur/plugins/bases/source.py

@@ -28,7 +28,7 @@ class SourcePlugin(Plugin):
     def get_endpoints(self, options, **kwargs):
         raise NotImplementedError
 
-    def clean(self, options, **kwargs):
+    def clean(self, certificate, options, **kwargs):
         raise NotImplementedError
 
     @property

lemur/plugins/lemur_aws/elb.py

@@ -16,14 +16,14 @@ from lemur.plugins.lemur_aws.sts import sts_client
 
 def retry_throttled(exception):
     """
-    Determiens if this exception is due to throttling
+    Determines if this exception is due to throttling
     :param exception:
     :return:
     """
     if isinstance(exception, botocore.exceptions.ClientError):
         if exception.response['Error']['Code'] == 'LoadBalancerNotFound':
-            return True
+            return False
-    return False
+    return True
 
 
 def is_valid(listener_tuple):

lemur/plugins/lemur_aws/iam.py

@@ -6,7 +6,24 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import botocore
+
+from retrying import retry
+
 from lemur.plugins.lemur_aws.sts import assume_service
+from lemur.plugins.lemur_aws.sts import sts_client
+
+
+def retry_throttled(exception):
+    """
+    Determines if this exception is due to throttling
+    :param exception:
+    :return:
+    """
+    if isinstance(exception, botocore.exceptions.ClientError):
+        if exception.response['Error']['Code'] == 'NoSuchEntity':
+            return False
+    return True
 
 
 def get_name_from_arn(arn):
@@ -33,15 +50,17 @@ def upload_cert(account_number, name, body, private_key, cert_chain=None):
                                                                     cert_chain=str(cert_chain))
 
 
-def delete_cert(account_number, cert_name):
+@sts_client('iam')
+@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+def delete_cert(cert_name, **kwargs):
     """
     Delete a certificate from AWS
 
-    :param account_number:
     :param cert_name:
     :return:
     """
-    return assume_service(account_number, 'iam').delete_server_cert(cert_name)
+    client = kwargs.pop('client')
+    client.delete_server_certificate(ServerCertificateName=cert_name)
 
 
 def get_all_server_certs(account_number):

lemur/plugins/lemur_aws/plugin.py

@@ -261,21 +261,9 @@ class AWSSourcePlugin(SourcePlugin):
         else:
             elb.attach_certificate(endpoint.name, endpoint.port, arn, account_number=account_number, region=region)
 
-    def clean(self, options, **kwargs):
+    def clean(self, certificate, options, **kwargs):
         account_number = self.get_option('accountNumber', options)
-        certificates = self.get_certificates(options)
+        iam.delete_cert(certificate.name, account_number=account_number)
-        endpoints = self.get_endpoints(options)
-
-        orphaned = []
-        for certificate in certificates:
-            for endpoint in endpoints:
-                if certificate['name'] == endpoint['certificate_name']:
-                    break
-            else:
-                orphaned.append(certificate['name'])
-                iam.delete_cert(account_number, certificate)
-
-        return orphaned
 
 
 class S3DestinationPlugin(DestinationPlugin):

lemur/plugins/lemur_aws/sts.py

@@ -56,6 +56,7 @@ def sts_client(service, service_type='client'):
                 kwargs.pop('account_number'),
                 current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur')
             )
+
             # TODO add user specific information to RoleSessionName
             role = sts.assume_role(RoleArn=arn, RoleSessionName='lemur')
 

lemur/sources/cli.py

@@ -15,8 +15,12 @@ from flask_script import Manager
 from flask import current_app
 
 from lemur.extensions import metrics
+from lemur.plugins.base import plugins
+
 from lemur.sources import service as source_service
 from lemur.users import service as user_service
+from lemur.certificates import service as certificate_service
+
 
 manager = Manager(usage="Handles all source related tasks.")
 
@@ -48,8 +52,8 @@ def validate_sources(source_strings):
 
 @manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
 def sync(source_strings):
-    source_objs = validate_sources(source_strings)
+    sources = validate_sources(source_strings)
-    for source in source_objs:
+    for source in sources:
         start_time = time.time()
         print("[+] Staring to sync source: {label}!\n".format(label=source.label))
 
@@ -86,15 +90,45 @@ def sync(source_strings):
 
 
 @manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
-def clean(source_strings):
+@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
-    source_objs = validate_sources(source_strings)
+def clean(source_strings, commit):
-    for source in source_objs:
+    sources = validate_sources(source_strings)
+    for source in sources:
+        s = plugins.get(source.plugin_name)
+
+        if not hasattr(s, 'clean'):
+            print("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
+                source.label
+            ))
+            continue
+
         start_time = time.time()
+
         print("[+] Staring to clean source: {label}!\n".format(label=source.label))
-        source_service.clean(source)
+
+        cleaned = 0
+        for certificate in certificate_service.get_all_pending_cleaning(source):
+                if commit:
+                    try:
+                        s.clean(certificate, source.options)
+                        certificate.sources.remove(source)
+                        certificate_service.database.update(certificate)
+                        metrics.send('clean_success', 'counter', 1, metric_tags={'source': source.label})
+                    except Exception as e:
+                        current_app.logger.exception(e)
+                        metrics.send('clean_failed', 'counter', 1, metric_tags={'source': source.label})
+
+                current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
+                    certificate.name,
+                    source.label
+                ))
+
+                cleaned += 1
+
         print(
-            "[+] Finished cleaning source: {label}. Run Time: {time}\n".format(
+            "[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format(
                 label=source.label,
-                time=(time.time() - start_time)
+                time=(time.time() - start_time),
+                cleaned=cleaned
             )
         )

lemur/sources/service.py

@@ -12,7 +12,7 @@ from flask import current_app
 from lemur import database
 from lemur.sources.models import Source
 from lemur.certificates.models import Certificate
-from lemur.certificates import service as cert_service
+from lemur.certificates import service as certificate_service
 from lemur.endpoints import service as endpoint_service
 from lemur.destinations import service as destination_service
 
@@ -21,29 +21,6 @@ from lemur.certificates.schemas import CertificateUploadInputSchema
 from lemur.plugins.base import plugins
 
 
-# TODO optimize via sql query
-def _disassociate_certs_from_source(certificates, source):
-    current_certificates = cert_service.get_by_source(source_label=source.label)
-    missing = []
-    for cc in current_certificates:
-        for fc in certificates:
-            if fc['body'] == cc.body:
-                break
-        else:
-            missing.append(cc)
-
-    for c in missing:
-        for s in c.sources:
-            if s.label == source:
-                current_app.logger.info(
-                    "Certificate {name} is no longer associated with {source}.".format(
-                        name=c.name,
-                        source=source.label
-                    )
-                )
-                c.sources.delete(s)
-
-
 def certificate_create(certificate, source):
     data, errors = CertificateUploadInputSchema().load(certificate)
 
@@ -52,7 +29,7 @@ def certificate_create(certificate, source):
 
     data['creator'] = certificate['creator']
 
-    cert = cert_service.import_certificate(**data)
+    cert = certificate_service.import_certificate(**data)
     cert.description = "This certificate was automatically discovered by Lemur"
     cert.sources.append(source)
     sync_update_destination(cert, source)
@@ -99,12 +76,12 @@ def sync_endpoints(source):
         certificate = endpoint.pop('certificate', None)
 
         if certificate_name:
-            cert = cert_service.get_by_name(certificate_name)
+            cert = certificate_service.get_by_name(certificate_name)
 
         elif certificate:
-            cert = cert_service.find_duplicates(certificate)
+            cert = certificate_service.find_duplicates(certificate)
             if not cert:
-                cert = cert_service.import_certificate(**certificate)
+                cert = certificate_service.import_certificate(**certificate)
 
         if not cert:
             current_app.logger.error(
@@ -142,7 +119,7 @@ def sync_certificates(source, user):
     certificates = s.get_certificates(source.options)
 
     for certificate in certificates:
-        exists = cert_service.find_duplicates(certificate)
+        exists = certificate_service.find_duplicates(certificate)
 
         certificate['owner'] = user.email
         certificate['creator'] = user
@@ -162,9 +139,6 @@ def sync_certificates(source, user):
                 )
             )
 
-    # we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
-    _disassociate_certs_from_source(certificates, source)
-
     return new, updated
 
 
@@ -178,33 +152,13 @@ def sync(source, user):
     return {'endpoints': (new_endpoints, updated_endpoints), 'certificates': (new_certs, updated_certs)}
 
 
-def clean(source):
-    s = plugins.get(source.plugin_name)
-
-    try:
-        certificates = s.clean(source.options)
-    except NotImplementedError:
-        current_app.logger.warning("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
-            source.label
-        ))
-        return
-
-    for certificate in certificates:
-        cert = cert_service.get_by_name(certificate)
-
-        if cert:
-            current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
-                cert.name,
-                source.label
-            ))
-            cert.sources.remove(source)
-
-
 def create(label, plugin_name, options, description=None):
     """
     Creates a new source, that can then be used as a source for certificates.
 
     :param label: Source common name
+    :param plugin_name:
+    :param options:
     :param description:
     :rtype : Source
     :return: New source
@@ -219,6 +173,8 @@ def update(source_id, label, options, description):
 
     :param source_id:  Lemur assigned ID
     :param label: Source common name
+    :param options:
+    :param description:
     :rtype : Source
     :return:
     """