Merge branch 'master' into adding-package-lock

lemur/certificates/cli.py

@@ -23,7 +23,7 @@ from lemur.certificates.service import (
     get_certificate_primitives,
     get_all_pending_reissue,
     get_by_name,
-    get_all_certs,
+    get_all_valid_certs,
     get,
     get_all_certs_attached_to_endpoint_without_autorotate,
 )
@@ -210,6 +210,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
     status = FAILURE_METRIC_STATUS
 
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+    }
+
     try:
         old_cert = validate_certificate(old_certificate_name)
         new_cert = validate_certificate(new_certificate_name)
@@ -219,26 +223,43 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
             print(
                 f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
             )
+            log_data["message"] = "Rotating endpoint"
+            log_data["endpoint"] = endpoint.dnsname
+            log_data["certificate"] = new_cert.name
             request_rotation(endpoint, new_cert, message, commit)
+            current_app.logger.info(log_data)
 
         elif old_cert and new_cert:
             print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
 
+            log_data["message"] = "Rotating all endpoints"
+            log_data["certificate"] = new_cert.name
+            log_data["certificate_old"] = old_cert.name
+            log_data["message"] = "Rotating endpoint from old to new cert"
             for endpoint in old_cert.endpoints:
                 print(f"[+] Rotating {endpoint.name}")
+                log_data["endpoint"] = endpoint.dnsname
                 request_rotation(endpoint, new_cert, message, commit)
+                current_app.logger.info(log_data)
 
         else:
             print("[+] Rotating all endpoints that have new certificates available")
+            log_data["message"] = "Rotating all endpoints that have new certificates available"
             for endpoint in endpoint_service.get_all_pending_rotation():
+                log_data["endpoint"] = endpoint.dnsname
                 if len(endpoint.certificate.replaced) == 1:
                     print(
                         f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
                     )
+                    log_data["certificate"] = endpoint.certificate.replaced[0].name
                     request_rotation(
                         endpoint, endpoint.certificate.replaced[0], message, commit
                     )
+                    current_app.logger.info(log_data)
+
                 else:
+                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
+                    print(log_data)
                     metrics.send(
                         "endpoint_rotation",
                         "counter",
@@ -636,7 +657,14 @@ def check_revoked():
     encounters an issue with verification it marks the certificate status
     as `unknown`.
     """
-    for cert in get_all_certs():
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Checking for revoked Certificates"
+    }
+
+    certs = get_all_valid_certs(current_app.config.get("SUPPORTED_REVOCATION_AUTHORITY_PLUGINS", []))
+    for cert in certs:
         try:
             if cert.chain:
                 status = verify_string(cert.body, cert.chain)
@@ -645,6 +673,20 @@ def check_revoked():
 
             cert.status = "valid" if status else "revoked"
 
+            if cert.status == "revoked":
+                log_data["valid"] = cert.status
+                log_data["certificate_name"] = cert.name
+                log_data["certificate_id"] = cert.id
+                metrics.send(
+                    "certificate_revoked",
+                    "counter",
+                    1,
+                    metric_tags={"status": log_data["valid"],
+                                 "certificate_name": log_data["certificate_name"],
+                                 "certificate_id": log_data["certificate_id"]},
+                )
+                current_app.logger.info(log_data)
+
         except Exception as e:
             sentry.captureException()
             current_app.logger.exception(e)

lemur/certificates/service.py

@@ -102,6 +102,27 @@ def get_all_certs():
     return Certificate.query.all()
 
 
+def get_all_valid_certs(authority_plugin_name):
+    """
+    Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names
+    ignored if no authority_plugin_name provided.
+
+    Note that depending on the DB size retrieving all certificates might an expensive operation
+
+    :return:
+    """
+    if authority_plugin_name:
+        return (
+            Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
+                Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
+                Authority.plugin_name.in_(authority_plugin_name)).all()
+        )
+    else:
+        return (
+            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all()
+        )
+
+
 def get_all_pending_cleaning_expired(source):
     """
     Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not

lemur/certificates/verify.py

@@ -8,6 +8,7 @@
 import requests
 import subprocess
 from flask import current_app
+from lemur.extensions import sentry
 from requests.exceptions import ConnectionError, InvalidSchema
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -152,10 +153,18 @@ def verify(cert_path, issuer_chain_path):
 
     # OCSP is our main source of truth, in a lot of cases CRLs
     # have been deprecated and are no longer updated
-    verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
+    try:
+        verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.exception(e)
 
     if verify_result is None:
-        verify_result = crl_verify(cert, cert_path)
+        try:
+            verify_result = crl_verify(cert, cert_path)
+        except Exception as e:
+            sentry.captureException()
+            current_app.logger.exception(e)
 
     if verify_result is None:
         current_app.logger.debug("Failed to verify {}".format(cert.serial_number))