Merge branch 'master' into dependabot/pip/botocore-1.18.18

2020-10-23 09:08:47 -07:00 · 2020-10-23 09:08:47 -07:00 · 5ccc99bbfa
commit 5ccc99bbfa
parent a3b90c1a6b 2ea39a51e3
10 changed files with 158 additions and 41 deletions
--- a/lemur/certificates/cli.py
+++ b/lemur/certificates/cli.py
@ -735,3 +735,45 @@ def automatically_enable_autorotate():
                                  })
        cert.rotation = True
        database.update(cert)
+
+
+@manager.command
+def deactivate_entrust_certificates():
+    """
+    Attempt to deactivate test certificates issued by Entrust
+    """
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Deactivating Entrust certificates"
+    }
+
+    certificates = get_all_valid_certs(['entrust-issuer'])
+    entrust_plugin = plugins.get('entrust-issuer')
+    for cert in certificates:
+        try:
+            response = entrust_plugin.deactivate_certificate(cert)
+            if response == 200:
+                cert.status = "revoked"
+            else:
+                cert.status = "unknown"
+
+            log_data["valid"] = cert.status
+            log_data["certificate_name"] = cert.name
+            log_data["certificate_id"] = cert.id
+            metrics.send(
+                "certificate_deactivate",
+                "counter",
+                1,
+                metric_tags={"status": log_data["valid"],
+                             "certificate_name": log_data["certificate_name"],
+                             "certificate_id": log_data["certificate_id"]},
+            )
+            current_app.logger.info(log_data)
+
+            database.update(cert)
+
+        except Exception as e:
+            current_app.logger.info(log_data)
+            sentry.captureException()
+            current_app.logger.exception(e)
--- a/lemur/certificates/schemas.py
+++ b/lemur/certificates/schemas.py
@ -340,6 +340,8 @@ class CertificateOutputSchema(LemurOutputSchema):

    @post_dump
    def handle_subject_details(self, data):
+        subject_details = ["country", "state", "location", "organization", "organizational_unit"]
+
        # Remove subject details if authority is CA/Browser Forum compliant. The code will use default set of values in that case.
        # If CA/Browser Forum compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below
        # condition checks for 'not False' ==> 'True or None'
@ -347,11 +349,13 @@ class CertificateOutputSchema(LemurOutputSchema):
            is_cab_compliant = data.get("authority").get("isCabCompliant")

            if is_cab_compliant is not False:
-                data.pop("country", None)
-                data.pop("state", None)
-                data.pop("location", None)
-                data.pop("organization", None)
-                data.pop("organizational_unit", None)
+                for field in subject_details:
+                    data.pop(field, None)
+
+        # Removing subject fields if None, else it complains in de-serialization
+        for field in subject_details:
+            if field in data and data[field] is None:
+                data.pop(field)


 class CertificateShortOutputSchema(LemurOutputSchema):
--- a/lemur/certificates/service.py
+++ b/lemur/certificates/service.py
@ -105,7 +105,7 @@ def get_all_certs():

 def get_all_valid_certs(authority_plugin_name):
    """
-    Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names
+    Retrieves all valid (not expired & not revoked) certificates within Lemur, for the given authority plugin names
    ignored if no authority_plugin_name provided.

    Note that depending on the DB size retrieving all certificates might an expensive operation
@ -116,11 +116,12 @@ def get_all_valid_certs(authority_plugin_name):
        return (
            Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
                Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
-                Authority.plugin_name.in_(authority_plugin_name)).all()
+                Authority.plugin_name.in_(authority_plugin_name)).filter(Certificate.revoked.is_(False)).all()
        )
    else:
        return (
-            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all()
+            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
+                Certificate.revoked.is_(False)).all()
        )


@ -359,7 +360,12 @@ def create(**kwargs):
    try:
        cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs)
    except Exception:
-        current_app.logger.error("Exception minting certificate", exc_info=True)
+        log_data = {
+            "message": "Exception minting certificate",
+            "issuer": kwargs["authority"].name,
+            "cn": kwargs["common_name"],
+        }
+        current_app.logger.error(log_data, exc_info=True)
        sentry.captureException()
        raise
    kwargs["body"] = cert_body
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@ -759,7 +759,7 @@ def check_revoked():

    log_data = {
        "function": function,
-        "message": "check if any certificates are revoked revoked",
+        "message": "check if any valid certificate is revoked",
        "task_id": task_id,
    }

@ -842,3 +842,39 @@ def enable_autorotate_for_certs_attached_to_endpoint():
    cli_certificate.automatically_enable_autorotate()
    metrics.send(f"{function}.success", "counter", 1)
    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def deactivate_entrust_test_certificates():
+    """
+    This celery task attempts to deactivate all not yet deactivated Entrust certificates, and should only run in TEST
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "deactivate entrust certificates",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.deactivate_entrust_certificates()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
--- a/lemur/common/defaults.py
+++ b/lemur/common/defaults.py
@ -95,9 +95,11 @@ def organization(cert):
    :return:
    """
    try:
-        return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[
-            0
-        ].value.strip()
+        o = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
+        if not o:
+            return None
+
+        return o[0].value.strip()
    except Exception as e:
        sentry.captureException()
        current_app.logger.error("Unable to get organization! {0}".format(e))
@ -110,9 +112,11 @@ def organizational_unit(cert):
    :return:
    """
    try:
-        return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)[
-            0
-        ].value.strip()
+        ou = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)
+        if not ou:
+            return None
+
+        return ou[0].value.strip()
    except Exception as e:
        sentry.captureException()
        current_app.logger.error("Unable to get organizational unit! {0}".format(e))
@ -125,9 +129,11 @@ def country(cert):
    :return:
    """
    try:
-        return cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)[
-            0
-        ].value.strip()
+        c = cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)
+        if not c:
+            return None
+
+        return c[0].value.strip()
    except Exception as e:
        sentry.captureException()
        current_app.logger.error("Unable to get country! {0}".format(e))
@ -140,9 +146,11 @@ def state(cert):
    :return:
    """
    try:
-        return cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)[
-            0
-        ].value.strip()
+        s = cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)
+        if not s:
+            return None
+
+        return s[0].value.strip()
    except Exception as e:
        sentry.captureException()
        current_app.logger.error("Unable to get state! {0}".format(e))
@ -155,9 +163,11 @@ def location(cert):
    :return:
    """
    try:
-        return cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)[
-            0
-        ].value.strip()
+        loc = cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)
+        if not loc:
+            return None
+
+        return loc[0].value.strip()
    except Exception as e:
        sentry.captureException()
        current_app.logger.error("Unable to get location! {0}".format(e))
--- a/lemur/plugins/lemur_digicert/plugin.py
+++ b/lemur/plugins/lemur_digicert/plugin.py
@ -37,7 +37,13 @@ def log_status_code(r, *args, **kwargs):
    :param kwargs:
    :return:
    """
+    log_data = {
+        "reason": (r.reason if r.reason else ""),
+        "status_code": r.status_code,
+        "url": (r.url if r.url else ""),
+    }
    metrics.send("digicert_status_code_{}".format(r.status_code), "counter", 1)
+    current_app.logger.info(log_data)


 def signature_hash(signing_algorithm):
@ -171,7 +177,7 @@ def map_cis_fields(options, csr):
        "csr": csr,
        "signature_hash": signature_hash(options.get("signing_algorithm")),
        "validity": {
-            "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z"
+            "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:SS") + "Z"
        },
        "organization": {
            "name": options["organization"],
@ -204,7 +210,7 @@ def handle_response(response):
    :return:
    """
    if response.status_code > 399:
-        raise Exception(response.json()["errors"][0]["message"])
+        raise Exception("DigiCert rejected request with the error:" + response.json()["errors"][0]["message"])

    return response.json()

@ -215,10 +221,17 @@ def handle_cis_response(response):
    :param response:
    :return:
    """
-    if response.status_code > 399:
-        raise Exception(response.text)
+    if response.status_code == 404:
+        raise Exception("DigiCert: order not in issued state")
+    elif response.status_code == 406:
+        raise Exception("DigiCert: wrong header request format")
+    elif response.status_code > 399:
+        raise Exception("DigiCert rejected request with the error:" + response.text)

-    return response.json()
+    if response.url.endswith("download"):
+        return response.content
+    else:
+        return response.json()


@retry(stop_max_attempt_number=10, wait_fixed=10000)
@ -238,11 +251,9 @@ def get_cis_certificate(session, base_url, order_id):
    certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id)
    session.headers.update({"Accept": "application/x-pkcs7-certificates"})
    response = session.get(certificate_url)
+    response_content = handle_cis_response(response)

-    if response.status_code == 404:
-        raise Exception("Order not in issued state.")
-
-    cert_chain_pem = convert_pkcs7_bytes_to_pem(response.content)
+    cert_chain_pem = convert_pkcs7_bytes_to_pem(response_content)
    if len(cert_chain_pem) < 3:
        raise Exception("Missing the certificate chain")
    return cert_chain_pem
--- a/lemur/plugins/lemur_digicert/tests/test_digicert.py
+++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py
@ -123,7 +123,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority):
            "signature_hash": "sha256",
            "organization": {"name": "Example, Inc."},
            "validity": {
-                "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM") + "Z"
+                "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM:SS") + "Z"
            },
            "profile_name": None,
        }
@ -159,7 +159,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho
            "signature_hash": "sha256",
            "organization": {"name": "Example, Inc."},
            "validity": {
-                "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z"
+                "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM:SS") + "Z"
            },
            "profile_name": None,
        }
--- a/lemur/plugins/lemur_entrust/plugin.py
+++ b/lemur/plugins/lemur_entrust/plugin.py
@ -20,7 +20,13 @@ def log_status_code(r, *args, **kwargs):
    :param kwargs:
    :return:
    """
+    log_data = {
+        "reason": (r.reason if r.reason else ""),
+        "status_code": r.status_code,
+        "url": (r.url if r.url else ""),
+    }
    metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1)
+    current_app.logger.info(log_data)


 def determine_end_date(end_date):
@ -109,7 +115,12 @@ def handle_response(my_response):
        "response": d
    }
    current_app.logger.info(log_data)
-    return d
+    if d == {'response': 'No detailed message'}:
+        # status if no data
+        return s
+    else:
+        #  return data from the response
+        return d


 class EntrustIssuerPlugin(IssuerPlugin):
--- a/lemur/static/app/angular/authorities/authority/options.tpl.html
+++ b/lemur/static/app/angular/authorities/authority/options.tpl.html
@ -24,7 +24,6 @@
              ng-options="option.value as option.name for option in [
                      {'name': 'RSA-2048', 'value': 'RSA2048'},
                      {'name': 'RSA-4096', 'value': 'RSA4096'},
-                      {'name': 'ECC-PRIME192V1', 'value': 'ECCPRIME192V1'},
                      {'name': 'ECC-PRIME256V1', 'value': 'ECCPRIME256V1'},
                      {'name': 'ECC-SECP384R1', 'value': 'ECCSECP384R1'},
                      {'name': 'ECC-SECP521R1', 'value': 'ECCSECP521R1'}]"
--- a/lemur/static/app/angular/certificates/certificate/options.tpl.html
+++ b/lemur/static/app/angular/certificates/certificate/options.tpl.html
@ -35,10 +35,8 @@
                  ng-options="option.value as option.name for option in [
                      {'name': 'RSA-2048', 'value': 'RSA2048'},
                      {'name': 'RSA-4096', 'value': 'RSA4096'},
-                      {'name': 'ECC-PRIME192V1', 'value': 'ECCPRIME192V1'},
                      {'name': 'ECC-PRIME256V1', 'value': 'ECCPRIME256V1'},
-                      {'name': 'ECC-SECP384R1', 'value': 'ECCSECP384R1'},
-                      {'name': 'ECC-SECP521R1', 'value': 'ECCSECP521R1'}]"
+                      {'name': 'ECC-SECP384R1', 'value': 'ECCSECP384R1'}]"

                  ng-init="certificate.keyType = 'RSA2048'"></select>
        </div>