Merge branch 'master' into remove-test-secrets

docs/administration.rst

@@ -1171,6 +1171,23 @@ The following configuration properties are required to use the PowerDNS ACME Plu
 
             File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle.
 
+ACME Plugin
+~~~~~~~~~~~~
+
+The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
+account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
+
+
+.. data:: ACME_PRIVATE_KEY
+    :noindex:
+
+            This is the private key, the account was registered with (in JWK format)
+
+.. data:: ACME_REGR
+    :noindex:
+
+            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
+
 .. _CommandLineInterface:
 
 Command Line Interface

docs/production/index.rst

@@ -511,3 +511,47 @@ The following must be added to the config file to activate the pinning (the pinn
     KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
     -----END CERTIFICATE-----
     """
+
+
+.. _AcmeAccountReuse:
+
+LetsEncrypt: Using a pre-existing ACME account
+-----------------------------------------------
+
+Let's Encrypt allows reusing an existing ACME account, to create and especially revoke certificates. The current
+implementation in the acme plugin, only allows for a single account for all ACME authorities, which might be an issue,
+when you try to use Let's Encrypt together with another certificate authority that uses the ACME protocol.
+
+To use an existing account, you need to configure the `ACME_PRIVATE_KEY` and `ACME_REGR` variables in the lemur
+configuration.
+
+`ACME_PRIVATE_KEY` needs to be in the JWK format::
+
+    {
+        "kty": "RSA",
+        "n": "yr1qBwHizA7ME_iV32bY10ILp.....",
+        "e": "AQAB",
+        "d": "llBlYhil3I.....",
+        "p": "-5LW2Lewogo.........",
+        "q": "zk6dHqHfHksd.........",
+        "dp": "qfe9fFIu3mu.......",
+        "dq": "cXFO-loeOyU.......",
+        "qi": "AfK1sh0_8sLTb..........."
+    }
+
+
+Using `python-jwt` converting an existing private key in PEM format is quite easy::
+
+    import python_jwt as jwt, jwcrypto.jwk as jwk
+
+    priv_key = jwk.JWK.from_pem(b"""-----BEGIN RSA PRIVATE KEY-----
+    ...
+    -----END RSA PRIVATE KEY-----""")
+
+    print(priv_key.export())
+
+`ACME_REGR` needs to be a valid JSON with a `body` and a `uri` attribute, similar to this::
+
+    {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/<ACCOUNT_NUMBER>"}
+
+The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key.

lemur/common/utils.py

@@ -71,6 +71,23 @@ def parse_private_key(private_key):
     )
 
 
+def get_key_type_from_certificate(body):
+    """
+
+    Helper function to determine key type by pasrding given PEM certificate
+
+    :param body: PEM string
+    :return: Key type string
+    """
+    parsed_cert = parse_certificate(body)
+    if isinstance(parsed_cert.public_key(), rsa.RSAPublicKey):
+        return "RSA{key_size}".format(
+            key_size=parsed_cert.public_key().key_size
+        )
+    elif isinstance(parsed_cert.public_key(), ec.EllipticCurvePublicKey):
+        return get_key_type_from_ec_curve(parsed_cert.public_key().curve.name)
+
+
 def split_pem(data):
     """
     Split a string of several PEM payloads to a list of strings.

lemur/migrations/versions/c301c59688d2_.py

@@ -0,0 +1,114 @@
+"""
+
+This database upgrade updates the key_type information for either
+still valid or expired certificates in the last 30 days. For RSA
+keys, the algorithm is determined based on the key length. For 
+the rest of the keys, the certificate body is parsed to determine 
+the exact key_type information.
+
+Each individual DB change is explicitly committed, and the respective
+log is added to a file named db_upgrade.log in the current working
+directory. Any error encountered while parsing a certificate will
+also be logged along with the certificate ID. If faced with any issue
+while running this upgrade, there is no harm in re-running the upgrade.
+Each run processes only rows for which key_type information is not yet
+determined.
+
+A successful complete run will end up updating the Alembic Version to
+the new Revision ID c301c59688d2. Currently, Lemur supports only RSA
+and ECC certificates. This could be a long-running job depending upon
+the number of DB entries it may process.
+
+Revision ID: c301c59688d2
+Revises: 434c29e40511
+Create Date: 2020-09-21 14:28:50.757998
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'c301c59688d2'
+down_revision = '434c29e40511'
+
+from alembic import op
+from sqlalchemy.sql import text
+from lemur.common import utils
+import time
+import datetime
+
+log_file = open('db_upgrade.log', 'a')
+
+
+def upgrade():
+    log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
+    start_time = time.time()
+
+    # Update RSA keys using the key length information
+    update_key_type_rsa(1024)
+    update_key_type_rsa(2048)
+    update_key_type_rsa(4096)
+
+    # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs.
+    update_key_type()
+
+    log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time))
+    log_file.close()
+
+
+def downgrade():
+    # Change key type column back to null
+    # Going back 32 days instead of 31 to make sure no certificates are skipped
+    stmt = text(
+        "update certificates set key_type=null where not_after > CURRENT_DATE - 32"
+    )
+    op.execute(stmt)
+
+
+"""
+    Helper methods performing updates for RSA and rest of the keys
+"""
+
+
+def update_key_type_rsa(bits):
+    log_file.write("Processing certificate with key type RSA %s\n" % bits)
+
+    stmt = text(
+        f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null"
+    )
+    log_file.write("Query: %s\n" % stmt)
+
+    start_time = time.time()
+    op.execute(stmt)
+    commit()
+
+    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+
+
+def update_key_type():
+    conn = op.get_bind()
+    start_time = time.time()
+
+    # Loop through all certificates that are valid today or expired in the last 30 days.
+    for cert_id, body in conn.execute(
+            text(
+                "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null")
+    ):
+        try:
+            cert_key_type = utils.get_key_type_from_certificate(body)
+        except ValueError as e:
+            log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
+        else:
+            log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
+            stmt = text(
+                "update certificates set key_type=:key_type where id=:id"
+            )
+            stmt = stmt.bindparams(key_type=cert_key_type, id=cert_id)
+            op.execute(stmt)
+
+            commit()
+
+    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+
+
+def commit():
+    stmt = text("commit")
+    op.execute(stmt)

lemur/plugins/lemur_entrust/plugin.py

@@ -1,9 +1,12 @@
-from lemur.plugins.bases import IssuerPlugin, SourcePlugin
+
 import arrow
 import requests
 import json
-from lemur.plugins import lemur_entrust as ENTRUST
+import sys
 from flask import current_app
+
+from lemur.plugins import lemur_entrust as entrust
+from lemur.plugins.bases import IssuerPlugin, SourcePlugin
 from lemur.extensions import metrics
 from lemur.common.utils import validate_conf
 
@@ -17,24 +20,24 @@ def log_status_code(r, *args, **kwargs):
     :param kwargs:
     :return:
     """
-    metrics.send("ENTRUST_status_code_{}".format(r.status_code), "counter", 1)
+    metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1)
 
 
 def determine_end_date(end_date):
     """
     Determine appropriate end date
     :param end_date:
-    :return: validity_end
+    :return: validity_end as string
     """
     # ENTRUST only allows 13 months of max certificate duration
-    max_validity_end = arrow.utcnow().shift(years=1, months=+1).format('YYYY-MM-DD')
+    max_validity_end = arrow.utcnow().shift(years=1, months=+1)
 
     if not end_date:
         end_date = max_validity_end
 
     if end_date > max_validity_end:
         end_date = max_validity_end
-    return end_date
+    return end_date.format('YYYY-MM-DD')
 
 
 def process_options(options):
@@ -49,7 +52,10 @@ def process_options(options):
     # take the value as Cert product-type
     # else default to "STANDARD_SSL"
     authority = options.get("authority").name.upper()
-    product_type = current_app.config.get("ENTRUST_PRODUCT_{0}".format(authority), "STANDARD_SSL")
+    # STANDARD_SSL (cn=domain, san=www.domain),
+    # ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]),
+    # WILDCARD_SSL (unlimited sans, and wildcard)
+    product_type = current_app.config.get(f"ENTRUST_PRODUCT_{authority}", "STANDARD_SSL")
 
     if options.get("validity_end"):
         validity_end = determine_end_date(options.get("validity_end"))
@@ -67,6 +73,7 @@ def process_options(options):
         "eku": "SERVER_AND_CLIENT_AUTH",
         "certType": product_type,
         "certExpiryDate": validity_end,
+        # "keyType": "RSA", Entrust complaining about this parameter
         "tracking": tracking_data
     }
     return data
@@ -86,23 +93,31 @@ def handle_response(my_response):
         404: "Unknown jobId",
         429: "Too many requests"
     }
+
     try:
         d = json.loads(my_response.content)
-    except Exception as e:
+    except ValueError:
         # catch an empty jason object here
-        d = {'errors': 'No detailled message'}
+        d = {'response': 'No detailed message'}
     s = my_response.status_code
     if s > 399:
-        raise Exception("ENTRUST error: {0}\n{1}".format(msg.get(s, s), d['errors']))
+        raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}")
-    current_app.logger.info("Response: {0}, {1} ".format(s, d))
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Response",
+        "status": s,
+        "response": d
+    }
+    current_app.logger.info(log_data)
     return d
 
 
 class EntrustIssuerPlugin(IssuerPlugin):
-    title = "ENTRUST"
+    title = "Entrust"
     slug = "entrust-issuer"
     description = "Enables the creation of certificates by ENTRUST"
-    version = ENTRUST.VERSION
+    version = entrust.VERSION
 
     author = "sirferl"
     author_url = "https://github.com/sirferl/lemur"
@@ -119,7 +134,6 @@ class EntrustIssuerPlugin(IssuerPlugin):
             "ENTRUST_NAME",
             "ENTRUST_EMAIL",
             "ENTRUST_PHONE",
-            "ENTRUST_ISSUING",
         ]
         validate_conf(current_app, required_vars)
 
@@ -142,9 +156,12 @@ class EntrustIssuerPlugin(IssuerPlugin):
         :param issuer_options:
         :return: :raise Exception:
         """
-        current_app.logger.info(
+        log_data = {
-            "Requesting options: {0}".format(issuer_options)
+            "function": f"{__name__}.{sys._getframe().f_code.co_name}",
-        )
+            "message": "Requesting options",
+            "options": issuer_options
+        }
+        current_app.logger.info(log_data)
 
         url = current_app.config.get("ENTRUST_URL") + "/certificates"
 
@@ -156,36 +173,46 @@ class EntrustIssuerPlugin(IssuerPlugin):
         except requests.exceptions.Timeout:
             raise Exception("Timeout for POST")
         except requests.exceptions.RequestException as e:
-            raise Exception("Error for POST {0}".format(e))
+            raise Exception(f"Error for POST {e}")
 
         response_dict = handle_response(response)
         external_id = response_dict['trackingId']
         cert = response_dict['endEntityCert']
-        chain = response_dict['chainCerts'][1]
+        if len(response_dict['chainCerts']) < 2:
-        current_app.logger.info(
+            # certificate signed by CA directly, no ICA included ini the chain
-            "Received Chain: {0}".format(chain)
+            chain = None
-        )
+        else:
+            chain = response_dict['chainCerts'][1]
+
+        log_data["message"] = "Received Chain"
+        log_data["options"] = f"chain: {chain}"
+        current_app.logger.info(log_data)
 
         return cert, chain, external_id
 
     def revoke_certificate(self, certificate, comments):
-        """Revoke a Digicert certificate."""
+        """Revoke an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
 
         # make certificate revoke request
-        revoke_url = "{0}/certificates/{1}/revocations".format(
+        revoke_url = f"{base_url}/certificates/{certificate.external_id}/revocations"
-            base_url, certificate.external_id
+        if not comments or comments == '':
-        )
-        metrics.send("entrust_revoke_certificate", "counter", 1)
-        if comments == '' or not comments:
             comments = "revoked via API"
         data = {
-            "crlReason": "superseded",
+            "crlReason": "superseded",  # enum (keyCompromise, affiliationChanged, superseded, cessationOfOperation)
             "revocationComment": comments
         }
         response = self.session.post(revoke_url, json=data)
+        metrics.send("entrust_revoke_certificate", "counter", 1)
+        return handle_response(response)
 
-        data = handle_response(response)
+    def deactivate_certificate(self, certificate):
+        """Deactivates an Entrust certificate."""
+        base_url = current_app.config.get("ENTRUST_URL")
+        deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations"
+        response = self.session.post(deactivate_url)
+        metrics.send("entrust_deactivate_certificate", "counter", 1)
+        return handle_response(response)
 
     @staticmethod
     def create_authority(options):
@@ -200,7 +227,8 @@ class EntrustIssuerPlugin(IssuerPlugin):
         entrust_root = current_app.config.get("ENTRUST_ROOT")
         entrust_issuing = current_app.config.get("ENTRUST_ISSUING")
         role = {"username": "", "password": "", "name": "entrust"}
-        current_app.logger.info("Creating Auth: {0} {1}".format(options, entrust_issuing))
+        current_app.logger.info(f"Creating Auth: {options} {entrust_issuing}")
+        # body, chain, role
         return entrust_root, "", [role]
 
     def get_ordered_certificate(self, order_id):
@@ -211,10 +239,10 @@ class EntrustIssuerPlugin(IssuerPlugin):
 
 
 class EntrustSourcePlugin(SourcePlugin):
-    title = "ENTRUST"
+    title = "Entrust"
     slug = "entrust-source"
-    description = "Enables the collecion of certificates"
+    description = "Enables the collection of certificates"
-    version = ENTRUST.VERSION
+    version = entrust.VERSION
 
     author = "sirferl"
     author_url = "https://github.com/sirferl/lemur"

lemur/plugins/lemur_entrust/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_entrust/tests/test_entrust.py

@@ -0,0 +1,54 @@
+from unittest.mock import patch, Mock
+
+import arrow
+from cryptography import x509
+from lemur.plugins.lemur_entrust import plugin
+
+
+def config_mock(*args):
+    values = {
+        "ENTRUST_API_CERT": "-----BEGIN CERTIFICATE-----abc-----END CERTIFICATE-----",
+        "ENTRUST_API_KEY": False,
+        "ENTRUST_API_USER": "test",
+        "ENTRUST_API_PASS": "password",
+        "ENTRUST_URL": "http",
+        "ENTRUST_ROOT": None,
+        "ENTRUST_NAME": "test",
+        "ENTRUST_EMAIL": "test@lemur.net",
+        "ENTRUST_PHONE": "0123456",
+        "ENTRUST_PRODUCT_ENTRUST": "ADVANTAGE_SSL"
+    }
+    return values[args[0]]
+
+
+@patch("lemur.plugins.lemur_entrust.plugin.current_app")
+def test_process_options(mock_current_app, authority):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    plugin.determine_end_date = Mock(return_value=arrow.get(2020, 10, 7).format('YYYY-MM-DD'))
+
+    authority.name = "Entrust"
+    names = [u"one.example.com", u"two.example.com", u"three.example.com"]
+    options = {
+        "common_name": "example.com",
+        "owner": "bob@example.com",
+        "description": "test certificate",
+        "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
+        "organization": "Example, Inc.",
+        "organizational_unit": "Example Org",
+        "validity_end": arrow.get(2020, 10, 7),
+        "authority": authority,
+    }
+
+    expected = {
+        "signingAlg": "SHA-2",
+        "eku": "SERVER_AND_CLIENT_AUTH",
+        "certType": "ADVANTAGE_SSL",
+        "certExpiryDate": arrow.get(2020, 10, 7).format('YYYY-MM-DD'),
+        "tracking": {
+            "requesterName": mock_current_app.config.get("ENTRUST_NAME"),
+            "requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"),
+            "requesterPhone": mock_current_app.config.get("ENTRUST_PHONE")
+        }
+    }
+
+    assert expected == plugin.process_options(options)

lemur/tests/conf.py

@@ -15,7 +15,6 @@ def get_random_secret(length):
     secret_key = secret_key + ''.join(random.choice(string.ascii_lowercase) for x in range(round(length / 4)))
     return secret_key + ''.join(random.choice(string.digits) for x in range(round(length / 4)))
 
-
 THREADS_PER_PAGE = 8
 
 # General
@@ -100,7 +99,6 @@ DIGICERT_CIS_API_KEY = "api-key"
 DIGICERT_CIS_ROOTS = {"root": "ROOT"}
 DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"}
 
-
 VERISIGN_URL = "http://example.com"
 VERISIGN_PEM_PATH = "~/"
 VERISIGN_FIRST_NAME = "Jim"
@@ -211,3 +209,41 @@ LDAP_REQUIRED_GROUP = "Lemur Access"
 LDAP_DEFAULT_ROLE = "role1"
 
 ALLOW_CERT_DELETION = True
+
+ENTRUST_API_CERT = "api-cert"
+ENTRUST_API_KEY = get_random_secret(32)
+ENTRUST_API_USER = "user"
+ENTRUST_API_PASS = get_random_secret(32)
+ENTRUST_URL = "https://api.entrust.net/enterprise/v2"
+ENTRUST_ROOT = """
+-----BEGIN CERTIFICATE-----
+MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC
+VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50
+cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs
+IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz
+dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy
+NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu
+dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt
+dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0
+aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj
+YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T
+RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN
+cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW
+wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1
+U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0
+jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN
+BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/
+jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ
+Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v
+1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R
+nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH
+VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g==
+-----END CERTIFICATE-----
+"""
+ENTRUST_NAME = "lemur"
+ENTRUST_EMAIL = "lemur@example.com"
+ENTRUST_PHONE = "123456"
+ENTRUST_ISSUING = ""
+ENTRUST_PRODUCT_ENTRUST = "ADVANTAGE_SSL"

lemur/tests/test_utils.py

@@ -2,11 +2,13 @@ import pytest
 
 from lemur.tests.vectors import (
     SAN_CERT,
+    SAN_CERT_STR,
     INTERMEDIATE_CERT,
     ROOTCA_CERT,
     EC_CERT_EXAMPLE,
     ECDSA_PRIME256V1_CERT,
     ECDSA_SECP384r1_CERT,
+    ECDSA_SECP384r1_CERT_STR,
     DSA_CERT,
 )
 
@@ -106,3 +108,9 @@ def test_is_selfsigned(selfsigned_cert):
     # unsupported algorithm (DSA)
     with pytest.raises(Exception):
         is_selfsigned(DSA_CERT)
+
+
+def test_get_key_type_from_certificate():
+    from lemur.common.utils import get_key_type_from_certificate
+    assert (get_key_type_from_certificate(SAN_CERT_STR) == "RSA2048")
+    assert (get_key_type_from_certificate(ECDSA_SECP384r1_CERT_STR) == "ECCSECP384R1")