Black lint all the things

lemur/sources/cli.py

@@ -35,24 +35,32 @@ def validate_sources(source_strings):
             table.append([source.label, source.active, source.description])
 
         print("No source specified choose from below:")
-        print(tabulate(table, headers=['Label', 'Active', 'Description']))
+        print(tabulate(table, headers=["Label", "Active", "Description"]))
         sys.exit(1)
 
-    if 'all' in source_strings:
+    if "all" in source_strings:
         sources = source_service.get_all()
     else:
         for source_str in source_strings:
             source = source_service.get_by_label(source_str)
 
             if not source:
-                print("Unable to find specified source with label: {0}".format(source_str))
+                print(
+                    "Unable to find specified source with label: {0}".format(source_str)
+                )
                 sys.exit(1)
 
             sources.append(source)
     return sources
 
 
-@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
+@manager.option(
+    "-s",
+    "--sources",
+    dest="source_strings",
+    action="append",
+    help="Sources to operate on.",
+)
 def sync(source_strings):
     sources = validate_sources(source_strings)
     for source in sources:
@@ -61,26 +69,23 @@ def sync(source_strings):
         start_time = time.time()
         print("[+] Staring to sync source: {label}!\n".format(label=source.label))
 
-        user = user_service.get_by_username('lemur')
+        user = user_service.get_by_username("lemur")
 
         try:
             data = source_service.sync(source, user)
             print(
                 "[+] Certificates: New: {new} Updated: {updated}".format(
-                    new=data['certificates'][0],
-                    updated=data['certificates'][1]
+                    new=data["certificates"][0], updated=data["certificates"][1]
                 )
             )
             print(
                 "[+] Endpoints: New: {new} Updated: {updated}".format(
-                    new=data['endpoints'][0],
-                    updated=data['endpoints'][1]
+                    new=data["endpoints"][0], updated=data["endpoints"][1]
                 )
             )
             print(
                 "[+] Finished syncing source: {label}. Run Time: {time}".format(
-                    label=source.label,
-                    time=(time.time() - start_time)
+                    label=source.label, time=(time.time() - start_time)
                 )
             )
             status = SUCCESS_METRIC_STATUS
@@ -88,27 +93,50 @@ def sync(source_strings):
         except Exception as e:
             current_app.logger.exception(e)
 
-            print(
-                "[X] Failed syncing source {label}!\n".format(label=source.label)
-            )
+            print("[X] Failed syncing source {label}!\n".format(label=source.label))
 
             sentry.captureException()
-            metrics.send('source_sync_fail', 'counter', 1, metric_tags={'source': source.label, 'status': status})
+            metrics.send(
+                "source_sync_fail",
+                "counter",
+                1,
+                metric_tags={"source": source.label, "status": status},
+            )
 
-        metrics.send('source_sync', 'counter', 1, metric_tags={'source': source.label, 'status': status})
+        metrics.send(
+            "source_sync",
+            "counter",
+            1,
+            metric_tags={"source": source.label, "status": status},
+        )
 
 
-@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
-@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
+@manager.option(
+    "-s",
+    "--sources",
+    dest="source_strings",
+    action="append",
+    help="Sources to operate on.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
 def clean(source_strings, commit):
     sources = validate_sources(source_strings)
     for source in sources:
         s = plugins.get(source.plugin_name)
 
-        if not hasattr(s, 'clean'):
-            print("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
-                source.label
-            ))
+        if not hasattr(s, "clean"):
+            print(
+                "Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
+                    source.label
+                )
+            )
             continue
 
         start_time = time.time()
@@ -128,19 +156,23 @@ def clean(source_strings, commit):
                     current_app.logger.exception(e)
                     sentry.captureException()
 
-            metrics.send('clean', 'counter', 1, metric_tags={'source': source.label, 'status': status})
+            metrics.send(
+                "clean",
+                "counter",
+                1,
+                metric_tags={"source": source.label, "status": status},
+            )
 
-            current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
-                certificate.name,
-                source.label
-            ))
+            current_app.logger.warning(
+                "Removed {0} from source {1} during cleaning".format(
+                    certificate.name, source.label
+                )
+            )
 
             cleaned += 1
 
         print(
             "[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format(
-                label=source.label,
-                time=(time.time() - start_time),
-                cleaned=cleaned
+                label=source.label, time=(time.time() - start_time), cleaned=cleaned
             )
         )

lemur/sources/models.py

@@ -15,7 +15,7 @@ from sqlalchemy_utils import ArrowType
 
 
 class Source(db.Model):
-    __tablename__ = 'sources'
+    __tablename__ = "sources"
     id = Column(Integer, primary_key=True)
     label = Column(String(32), unique=True)
     options = Column(JSONType)

lemur/sources/schemas.py

@@ -30,7 +30,7 @@ class SourceOutputSchema(LemurOutputSchema):
     @post_dump
     def fill_object(self, data):
         if data:
-            data['plugin']['pluginOptions'] = data['options']
+            data["plugin"]["pluginOptions"] = data["options"]
         return data
 
 

lemur/sources/service.py

@@ -29,9 +29,11 @@ def certificate_create(certificate, source):
     data, errors = CertificateUploadInputSchema().load(certificate)
 
     if errors:
-        raise Exception("Unable to import certificate: {reasons}".format(reasons=errors))
+        raise Exception(
+            "Unable to import certificate: {reasons}".format(reasons=errors)
+        )
 
-    data['creator'] = certificate['creator']
+    data["creator"] = certificate["creator"]
 
     cert = certificate_service.import_certificate(**data)
     cert.description = "This certificate was automatically discovered by Lemur"
@@ -70,33 +72,44 @@ def sync_endpoints(source):
     try:
         endpoints = s.get_endpoints(source.options)
     except NotImplementedError:
-        current_app.logger.warning("Unable to sync endpoints for source {0} plugin has not implemented 'get_endpoints'".format(source.label))
+        current_app.logger.warning(
+            "Unable to sync endpoints for source {0} plugin has not implemented 'get_endpoints'".format(
+                source.label
+            )
+        )
         return new, updated
 
     for endpoint in endpoints:
-        exists = endpoint_service.get_by_dnsname_and_port(endpoint['dnsname'], endpoint['port'])
+        exists = endpoint_service.get_by_dnsname_and_port(
+            endpoint["dnsname"], endpoint["port"]
+        )
 
-        certificate_name = endpoint.pop('certificate_name')
+        certificate_name = endpoint.pop("certificate_name")
 
-        endpoint['certificate'] = certificate_service.get_by_name(certificate_name)
+        endpoint["certificate"] = certificate_service.get_by_name(certificate_name)
 
-        if not endpoint['certificate']:
+        if not endpoint["certificate"]:
             current_app.logger.error(
-                "Certificate Not Found. Name: {0} Endpoint: {1}".format(certificate_name, endpoint['name']))
+                "Certificate Not Found. Name: {0} Endpoint: {1}".format(
+                    certificate_name, endpoint["name"]
+                )
+            )
             continue
 
-        policy = endpoint.pop('policy')
+        policy = endpoint.pop("policy")
 
         policy_ciphers = []
-        for nc in policy['ciphers']:
+        for nc in policy["ciphers"]:
             policy_ciphers.append(endpoint_service.get_or_create_cipher(name=nc))
 
-        policy['ciphers'] = policy_ciphers
-        endpoint['policy'] = endpoint_service.get_or_create_policy(**policy)
-        endpoint['source'] = source
+        policy["ciphers"] = policy_ciphers
+        endpoint["policy"] = endpoint_service.get_or_create_policy(**policy)
+        endpoint["source"] = source
 
         if not exists:
-            current_app.logger.debug("Endpoint Created: Name: {name}".format(name=endpoint['name']))
+            current_app.logger.debug(
+                "Endpoint Created: Name: {name}".format(name=endpoint["name"])
+            )
             endpoint_service.create(**endpoint)
             new += 1
 
@@ -119,27 +132,27 @@ def sync_certificates(source, user):
     for certificate in certificates:
         exists = False
 
-        if certificate.get('search', None):
-            conditions = certificate.pop('search')
+        if certificate.get("search", None):
+            conditions = certificate.pop("search")
             exists = certificate_service.get_by_attributes(conditions)
 
-        if not exists and certificate.get('name'):
-            result = certificate_service.get_by_name(certificate['name'])
+        if not exists and certificate.get("name"):
+            result = certificate_service.get_by_name(certificate["name"])
             if result:
                 exists = [result]
 
-        if not exists and certificate.get('serial'):
-            exists = certificate_service.get_by_serial(certificate['serial'])
+        if not exists and certificate.get("serial"):
+            exists = certificate_service.get_by_serial(certificate["serial"])
 
         if not exists:
-            cert = parse_certificate(certificate['body'])
+            cert = parse_certificate(certificate["body"])
             matching_serials = certificate_service.get_by_serial(serial(cert))
             exists = find_matching_certificates_by_hash(cert, matching_serials)
 
-        if not certificate.get('owner'):
-            certificate['owner'] = user.email
+        if not certificate.get("owner"):
+            certificate["owner"] = user.email
 
-        certificate['creator'] = user
+        certificate["creator"] = user
         exists = [x for x in exists if x]
 
         if not exists:
@@ -148,10 +161,10 @@ def sync_certificates(source, user):
 
         else:
             for e in exists:
-                if certificate.get('external_id'):
-                    e.external_id = certificate['external_id']
-                if certificate.get('authority_id'):
-                    e.authority_id = certificate['authority_id']
+                if certificate.get("external_id"):
+                    e.external_id = certificate["external_id"]
+                if certificate.get("authority_id"):
+                    e.authority_id = certificate["authority_id"]
                 certificate_update(e, source)
                 updated += 1
 
@@ -165,7 +178,10 @@ def sync(source, user):
     source.last_run = arrow.utcnow()
     database.update(source)
 
-    return {'endpoints': (new_endpoints, updated_endpoints), 'certificates': (new_certs, updated_certs)}
+    return {
+        "endpoints": (new_endpoints, updated_endpoints),
+        "certificates": (new_certs, updated_certs),
+    }
 
 
 def create(label, plugin_name, options, description=None):
@@ -179,7 +195,9 @@ def create(label, plugin_name, options, description=None):
     :rtype : Source
     :return: New source
     """
-    source = Source(label=label, options=options, plugin_name=plugin_name, description=description)
+    source = Source(
+        label=label, options=options, plugin_name=plugin_name, description=description
+    )
     return database.create(source)
 
 
@@ -230,7 +248,7 @@ def get_by_label(label):
     :param label:
     :return:
     """
-    return database.get(Source, label, field='label')
+    return database.get(Source, label, field="label")
 
 
 def get_all():
@@ -244,8 +262,8 @@ def get_all():
 
 
 def render(args):
-    filt = args.pop('filter')
-    certificate_id = args.pop('certificate_id', None)
+    filt = args.pop("filter")
+    certificate_id = args.pop("certificate_id", None)
 
     if certificate_id:
         query = database.session_query(Source).join(Certificate, Source.certificate)
@@ -254,7 +272,7 @@ def render(args):
         query = database.session_query(Source)
 
     if filt:
-        terms = filt.split(';')
+        terms = filt.split(";")
         query = database.filter(query, Source, terms)
 
     return database.sort_and_page(query, Source, args)
@@ -272,21 +290,27 @@ def add_aws_destination_to_sources(dst):
     src_accounts = set()
     sources = get_all()
     for src in sources:
-        src_accounts.add(get_plugin_option('accountNumber', src.options))
+        src_accounts.add(get_plugin_option("accountNumber", src.options))
 
     # check
     destination_plugin = plugins.get(dst.plugin_name)
-    account_number = get_plugin_option('accountNumber', dst.options)
-    if account_number is not None and \
-            destination_plugin.sync_as_source is not None and \
-            destination_plugin.sync_as_source and \
-            (account_number not in src_accounts):
-        src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
-        set_plugin_option('accountNumber', account_number, src_options)
-        create(label=dst.label,
-               plugin_name=destination_plugin.sync_as_source_name,
-               options=src_options,
-               description=dst.description)
+    account_number = get_plugin_option("accountNumber", dst.options)
+    if (
+        account_number is not None
+        and destination_plugin.sync_as_source is not None
+        and destination_plugin.sync_as_source
+        and (account_number not in src_accounts)
+    ):
+        src_options = copy.deepcopy(
+            plugins.get(destination_plugin.sync_as_source_name).options
+        )
+        set_plugin_option("accountNumber", account_number, src_options)
+        create(
+            label=dst.label,
+            plugin_name=destination_plugin.sync_as_source_name,
+            options=src_options,
+            description=dst.description,
+        )
         return True
 
     return False

lemur/sources/views.py

@@ -11,19 +11,24 @@ from flask_restful import Api, reqparse
 from lemur.sources import service
 
 from lemur.common.schema import validate_schema
-from lemur.sources.schemas import source_input_schema, source_output_schema, sources_output_schema
+from lemur.sources.schemas import (
+    source_input_schema,
+    source_output_schema,
+    sources_output_schema,
+)
 
 from lemur.auth.service import AuthenticatedResource
 from lemur.auth.permissions import admin_permission
 from lemur.common.utils import paginated_parser
 
 
-mod = Blueprint('sources', __name__)
+mod = Blueprint("sources", __name__)
 api = Api(mod)
 
 
 class SourcesList(AuthenticatedResource):
     """ Defines the 'sources' endpoint """
+
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(SourcesList, self).__init__()
@@ -151,7 +156,12 @@ class SourcesList(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        return service.create(data['label'], data['plugin']['slug'], data['plugin']['plugin_options'], data['description'])
+        return service.create(
+            data["label"],
+            data["plugin"]["slug"],
+            data["plugin"]["plugin_options"],
+            data["description"],
+        )
 
 
 class Sources(AuthenticatedResource):
@@ -271,16 +281,22 @@ class Sources(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        return service.update(source_id, data['label'], data['plugin']['plugin_options'], data['description'])
+        return service.update(
+            source_id,
+            data["label"],
+            data["plugin"]["plugin_options"],
+            data["description"],
+        )
 
     @admin_permission.require(http_exception=403)
     def delete(self, source_id):
         service.delete(source_id)
-        return {'result': True}
+        return {"result": True}
 
 
 class CertificateSources(AuthenticatedResource):
     """ Defines the 'certificate/<int:certificate_id/sources'' endpoint """
+
     def __init__(self):
         super(CertificateSources, self).__init__()
 
@@ -340,11 +356,14 @@ class CertificateSources(AuthenticatedResource):
         """
         parser = paginated_parser.copy()
         args = parser.parse_args()
-        args['certificate_id'] = certificate_id
+        args["certificate_id"] = certificate_id
         return service.render(args)
 
 
-api.add_resource(SourcesList, '/sources', endpoint='sources')
-api.add_resource(Sources, '/sources/<int:source_id>', endpoint='account')
-api.add_resource(CertificateSources, '/certificates/<int:certificate_id>/sources',
-                 endpoint='certificateSources')
+api.add_resource(SourcesList, "/sources", endpoint="sources")
+api.add_resource(Sources, "/sources/<int:source_id>", endpoint="account")
+api.add_resource(
+    CertificateSources,
+    "/certificates/<int:certificate_id>/sources",
+    endpoint="certificateSources",
+)