Merge pull request #2869 from hosseinsh/cert-sync-endpoint-find-by-hash

Cert sync endpoint find by hash

lemur/plugins/lemur_aws/iam.py

@@ -10,7 +10,7 @@ import botocore
 
 from retrying import retry
 
-from lemur.extensions import metrics
+from lemur.extensions import metrics, sentry
 from lemur.plugins.lemur_aws.sts import sts_client
 
 
@@ -122,9 +122,11 @@ def get_certificate(name, **kwargs):
     """
     client = kwargs.pop("client")
     metrics.send("get_certificate", "counter", 1, metric_tags={"name": name})
-    return client.get_server_certificate(ServerCertificateName=name)[
+    try:
-        "ServerCertificate"
+        return client.get_server_certificate(ServerCertificateName=name)["ServerCertificate"]
-    ]
+    except client.exceptions.NoSuchEntityException:
+        sentry.captureException()
+        return None
 
 
 @sts_client("iam")

lemur/plugins/lemur_aws/plugin.py

@@ -32,7 +32,9 @@
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 .. moduleauthor:: Harm Weites <harm@weites.com>
 """
+from acme.errors import ClientError
 from flask import current_app
+from lemur.extensions import sentry, metrics
 
 from lemur.plugins import lemur_aws as aws
 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
@@ -271,6 +273,29 @@ class AWSSourcePlugin(SourcePlugin):
         account_number = self.get_option("accountNumber", options)
         iam.delete_cert(certificate.name, account_number=account_number)
 
+    def get_certificate_by_name(self, certificate_name, options):
+        account_number = self.get_option("accountNumber", options)
+        # certificate name may contain path, in which case we remove it
+        if "/" in certificate_name:
+            certificate_name = certificate_name.split('/')[-1]
+        try:
+            cert = iam.get_certificate(certificate_name, account_number=account_number)
+            if cert:
+                return dict(
+                    body=cert["CertificateBody"],
+                    chain=cert.get("CertificateChain"),
+                    name=cert["ServerCertificateMetadata"]["ServerCertificateName"],
+                )
+        except ClientError:
+            current_app.logger.warning(
+                "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name))
+            sentry.captureException()
+            metrics.send(
+                "get_elb_certificate_failed", "counter", 1,
+                metric_tags={"certificate_name": certificate_name, "account_number": account_number}
+            )
+        return None
+
 
 class AWSDestinationPlugin(DestinationPlugin):
     title = "AWS"

lemur/sources/service.py

@@ -15,7 +15,7 @@ from lemur.sources.models import Source
 from lemur.certificates.models import Certificate
 from lemur.certificates import service as certificate_service
 from lemur.endpoints import service as endpoint_service
-from lemur.extensions import metrics
+from lemur.extensions import metrics, sentry
 from lemur.destinations import service as destination_service
 
 from lemur.certificates.schemas import CertificateUploadInputSchema
@@ -66,7 +66,7 @@ def sync_update_destination(certificate, source):
 
 
 def sync_endpoints(source):
-    new, updated = 0, 0
+    new, updated, updated_by_hash = 0, 0, 0
     current_app.logger.debug("Retrieving endpoints from {0}".format(source.label))
     s = plugins.get(source.plugin_name)
 
@@ -89,6 +89,39 @@ def sync_endpoints(source):
 
         endpoint["certificate"] = certificate_service.get_by_name(certificate_name)
 
+        # if get cert by name failed, we attempt a search via serial number and hash comparison
+        # and link the endpoint certificate to Lemur certificate
+        if not endpoint["certificate"]:
+            certificate_attached_to_endpoint = None
+            try:
+                certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options)
+            except NotImplementedError:
+                current_app.logger.warning(
+                    "Unable to describe server certificate for endpoints in source {0}:"
+                    " plugin has not implemented 'get_certificate_by_name'".format(
+                        source.label
+                    )
+                )
+                sentry.captureException()
+
+            if certificate_attached_to_endpoint:
+                lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint)
+                updated_by_hash += updated_by_hash_tmp
+
+                if lemur_matching_cert:
+                    endpoint["certificate"] = lemur_matching_cert[0]
+
+                if len(lemur_matching_cert) > 1:
+                    current_app.logger.error(
+                        "Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format(
+                            len(lemur_matching_cert), certificate_name, endpoint["name"]
+                        )
+                    )
+                    metrics.send("endpoint.certificate.conflict",
+                                 "gauge", len(lemur_matching_cert),
+                                 metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
+                                              "acct": s.get_option("accountNumber", source.options)})
+
         if not endpoint["certificate"]:
             current_app.logger.error(
                 "Certificate Not Found. Name: {0} Endpoint: {1}".format(
@@ -97,7 +130,8 @@ def sync_endpoints(source):
             )
             metrics.send("endpoint.certificate.not.found",
                          "counter", 1,
-                         metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)})
+                         metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
+                                      "acct": s.get_option("accountNumber", source.options)})
             continue
 
         policy = endpoint.pop("policy")
@@ -122,42 +156,50 @@ def sync_endpoints(source):
             endpoint_service.update(exists.id, **endpoint)
             updated += 1
 
-    return new, updated
+    return new, updated, updated_by_hash
+
+
+def find_cert(certificate):
+    updated_by_hash = 0
+    exists = False
+
+    if certificate.get("search", None):
+        conditions = certificate.pop("search")
+        exists = certificate_service.get_by_attributes(conditions)
+
+    if not exists and certificate.get("name"):
+        result = certificate_service.get_by_name(certificate["name"])
+        if result:
+            exists = [result]
+
+    if not exists and certificate.get("serial"):
+        exists = certificate_service.get_by_serial(certificate["serial"])
+
+    if not exists:
+        cert = parse_certificate(certificate["body"])
+        matching_serials = certificate_service.get_by_serial(serial(cert))
+        exists = find_matching_certificates_by_hash(cert, matching_serials)
+        updated_by_hash += 1
+
+    exists = [x for x in exists if x]
+    return exists, updated_by_hash
 
 
 # TODO this is very slow as we don't batch update certificates
 def sync_certificates(source, user):
-    new, updated = 0, 0
+    new, updated, updated_by_hash = 0, 0, 0
 
     current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
     s = plugins.get(source.plugin_name)
     certificates = s.get_certificates(source.options)
 
     for certificate in certificates:
-        exists = False
+        exists, updated_by_hash = find_cert(certificate)
-
-        if certificate.get("search", None):
-            conditions = certificate.pop("search")
-            exists = certificate_service.get_by_attributes(conditions)
-
-        if not exists and certificate.get("name"):
-            result = certificate_service.get_by_name(certificate["name"])
-            if result:
-                exists = [result]
-
-        if not exists and certificate.get("serial"):
-            exists = certificate_service.get_by_serial(certificate["serial"])
-
-        if not exists:
-            cert = parse_certificate(certificate["body"])
-            matching_serials = certificate_service.get_by_serial(serial(cert))
-            exists = find_matching_certificates_by_hash(cert, matching_serials)
 
         if not certificate.get("owner"):
             certificate["owner"] = user.email
 
         certificate["creator"] = user
-        exists = [x for x in exists if x]
 
         if not exists:
             certificate_create(certificate, source)
@@ -172,12 +214,20 @@ def sync_certificates(source, user):
                 certificate_update(e, source)
                 updated += 1
 
-    return new, updated
+    return new, updated, updated_by_hash
 
 
 def sync(source, user):
-    new_certs, updated_certs = sync_certificates(source, user)
+    new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user)
-    new_endpoints, updated_endpoints = sync_endpoints(source)
+    new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source)
+
+    metrics.send("sync.updated_certs_by_hash",
+                 "gauge", updated_certs_by_hash,
+                 metric_tags={"source": source.label})
+
+    metrics.send("sync.updated_endpoints_by_hash",
+                 "gauge", updated_endpoints_by_hash,
+                 metric_tags={"source": source.label})
 
     source.last_run = arrow.utcnow()
     database.update(source)