diff --git a/docs/administration.rst b/docs/administration.rst index 0cec16a0..f44ad1a3 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1171,6 +1171,23 @@ The following configuration properties are required to use the PowerDNS ACME Plu File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. +ACME Plugin +~~~~~~~~~~~~ + +The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME +account. See :ref:`Using a pre-existing ACME account ` for more details. + + +.. data:: ACME_PRIVATE_KEY + :noindex: + + This is the private key, the account was registered with (in JWK format) + +.. data:: ACME_REGR + :noindex: + + This is the registration for the ACME account, the most important part is the uri attribute (in JSON) + .. _CommandLineInterface: Command Line Interface diff --git a/docs/production/index.rst b/docs/production/index.rst index 9f90c0cc..c6f561ca 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -511,3 +511,47 @@ The following must be added to the config file to activate the pinning (the pinn KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg== -----END CERTIFICATE----- """ + + +.. _AcmeAccountReuse: + +LetsEncrypt: Using a pre-existing ACME account +----------------------------------------------- + +Let's Encrypt allows reusing an existing ACME account, to create and especially revoke certificates. The current +implementation in the acme plugin, only allows for a single account for all ACME authorities, which might be an issue, +when you try to use Let's Encrypt together with another certificate authority that uses the ACME protocol. + +To use an existing account, you need to configure the `ACME_PRIVATE_KEY` and `ACME_REGR` variables in the lemur +configuration. + +`ACME_PRIVATE_KEY` needs to be in the JWK format:: + + { + "kty": "RSA", + "n": "yr1qBwHizA7ME_iV32bY10ILp.....", + "e": "AQAB", + "d": "llBlYhil3I.....", + "p": "-5LW2Lewogo.........", + "q": "zk6dHqHfHksd.........", + "dp": "qfe9fFIu3mu.......", + "dq": "cXFO-loeOyU.......", + "qi": "AfK1sh0_8sLTb..........." + } + + +Using `python-jwt` converting an existing private key in PEM format is quite easy:: + + import python_jwt as jwt, jwcrypto.jwk as jwk + + priv_key = jwk.JWK.from_pem(b"""-----BEGIN RSA PRIVATE KEY----- + ... + -----END RSA PRIVATE KEY-----""") + + print(priv_key.export()) + +`ACME_REGR` needs to be a valid JSON with a `body` and a `uri` attribute, similar to this:: + + {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/"} + +The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key. \ No newline at end of file diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 01cc64ae..283d1eec 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -71,6 +71,23 @@ def parse_private_key(private_key): ) +def get_key_type_from_certificate(body): + """ + + Helper function to determine key type by pasrding given PEM certificate + + :param body: PEM string + :return: Key type string + """ + parsed_cert = parse_certificate(body) + if isinstance(parsed_cert.public_key(), rsa.RSAPublicKey): + return "RSA{key_size}".format( + key_size=parsed_cert.public_key().key_size + ) + elif isinstance(parsed_cert.public_key(), ec.EllipticCurvePublicKey): + return get_key_type_from_ec_curve(parsed_cert.public_key().curve.name) + + def split_pem(data): """ Split a string of several PEM payloads to a list of strings. diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py new file mode 100644 index 00000000..3b0a86f7 --- /dev/null +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -0,0 +1,114 @@ +""" + +This database upgrade updates the key_type information for either +still valid or expired certificates in the last 30 days. For RSA +keys, the algorithm is determined based on the key length. For +the rest of the keys, the certificate body is parsed to determine +the exact key_type information. + +Each individual DB change is explicitly committed, and the respective +log is added to a file named db_upgrade.log in the current working +directory. Any error encountered while parsing a certificate will +also be logged along with the certificate ID. If faced with any issue +while running this upgrade, there is no harm in re-running the upgrade. +Each run processes only rows for which key_type information is not yet +determined. + +A successful complete run will end up updating the Alembic Version to +the new Revision ID c301c59688d2. Currently, Lemur supports only RSA +and ECC certificates. This could be a long-running job depending upon +the number of DB entries it may process. + +Revision ID: c301c59688d2 +Revises: 434c29e40511 +Create Date: 2020-09-21 14:28:50.757998 + +""" + +# revision identifiers, used by Alembic. +revision = 'c301c59688d2' +down_revision = '434c29e40511' + +from alembic import op +from sqlalchemy.sql import text +from lemur.common import utils +import time +import datetime + +log_file = open('db_upgrade.log', 'a') + + +def upgrade(): + log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) + start_time = time.time() + + # Update RSA keys using the key length information + update_key_type_rsa(1024) + update_key_type_rsa(2048) + update_key_type_rsa(4096) + + # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs. + update_key_type() + + log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time)) + log_file.close() + + +def downgrade(): + # Change key type column back to null + # Going back 32 days instead of 31 to make sure no certificates are skipped + stmt = text( + "update certificates set key_type=null where not_after > CURRENT_DATE - 32" + ) + op.execute(stmt) + + +""" + Helper methods performing updates for RSA and rest of the keys +""" + + +def update_key_type_rsa(bits): + log_file.write("Processing certificate with key type RSA %s\n" % bits) + + stmt = text( + f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null" + ) + log_file.write("Query: %s\n" % stmt) + + start_time = time.time() + op.execute(stmt) + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def update_key_type(): + conn = op.get_bind() + start_time = time.time() + + # Loop through all certificates that are valid today or expired in the last 30 days. + for cert_id, body in conn.execute( + text( + "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null") + ): + try: + cert_key_type = utils.get_key_type_from_certificate(body) + except ValueError as e: + log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) + else: + log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + stmt = text( + "update certificates set key_type=:key_type where id=:id" + ) + stmt = stmt.bindparams(key_type=cert_key_type, id=cert_id) + op.execute(stmt) + + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def commit(): + stmt = text("commit") + op.execute(stmt) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 315da8bd..9b7848ed 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -1,9 +1,12 @@ -from lemur.plugins.bases import IssuerPlugin, SourcePlugin + import arrow import requests import json -from lemur.plugins import lemur_entrust as ENTRUST +import sys from flask import current_app + +from lemur.plugins import lemur_entrust as entrust +from lemur.plugins.bases import IssuerPlugin, SourcePlugin from lemur.extensions import metrics from lemur.common.utils import validate_conf @@ -17,24 +20,24 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ - metrics.send("ENTRUST_status_code_{}".format(r.status_code), "counter", 1) + metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1) def determine_end_date(end_date): """ Determine appropriate end date :param end_date: - :return: validity_end + :return: validity_end as string """ # ENTRUST only allows 13 months of max certificate duration - max_validity_end = arrow.utcnow().shift(years=1, months=+1).format('YYYY-MM-DD') + max_validity_end = arrow.utcnow().shift(years=1, months=+1) if not end_date: end_date = max_validity_end if end_date > max_validity_end: end_date = max_validity_end - return end_date + return end_date.format('YYYY-MM-DD') def process_options(options): @@ -49,7 +52,10 @@ def process_options(options): # take the value as Cert product-type # else default to "STANDARD_SSL" authority = options.get("authority").name.upper() - product_type = current_app.config.get("ENTRUST_PRODUCT_{0}".format(authority), "STANDARD_SSL") + # STANDARD_SSL (cn=domain, san=www.domain), + # ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]), + # WILDCARD_SSL (unlimited sans, and wildcard) + product_type = current_app.config.get(f"ENTRUST_PRODUCT_{authority}", "STANDARD_SSL") if options.get("validity_end"): validity_end = determine_end_date(options.get("validity_end")) @@ -67,6 +73,7 @@ def process_options(options): "eku": "SERVER_AND_CLIENT_AUTH", "certType": product_type, "certExpiryDate": validity_end, + # "keyType": "RSA", Entrust complaining about this parameter "tracking": tracking_data } return data @@ -86,23 +93,31 @@ def handle_response(my_response): 404: "Unknown jobId", 429: "Too many requests" } + try: d = json.loads(my_response.content) - except Exception as e: + except ValueError: # catch an empty jason object here - d = {'errors': 'No detailled message'} + d = {'response': 'No detailed message'} s = my_response.status_code if s > 399: - raise Exception("ENTRUST error: {0}\n{1}".format(msg.get(s, s), d['errors'])) - current_app.logger.info("Response: {0}, {1} ".format(s, d)) + raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}") + + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Response", + "status": s, + "response": d + } + current_app.logger.info(log_data) return d class EntrustIssuerPlugin(IssuerPlugin): - title = "ENTRUST" + title = "Entrust" slug = "entrust-issuer" description = "Enables the creation of certificates by ENTRUST" - version = ENTRUST.VERSION + version = entrust.VERSION author = "sirferl" author_url = "https://github.com/sirferl/lemur" @@ -119,7 +134,6 @@ class EntrustIssuerPlugin(IssuerPlugin): "ENTRUST_NAME", "ENTRUST_EMAIL", "ENTRUST_PHONE", - "ENTRUST_ISSUING", ] validate_conf(current_app, required_vars) @@ -142,9 +156,12 @@ class EntrustIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ - current_app.logger.info( - "Requesting options: {0}".format(issuer_options) - ) + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Requesting options", + "options": issuer_options + } + current_app.logger.info(log_data) url = current_app.config.get("ENTRUST_URL") + "/certificates" @@ -156,36 +173,46 @@ class EntrustIssuerPlugin(IssuerPlugin): except requests.exceptions.Timeout: raise Exception("Timeout for POST") except requests.exceptions.RequestException as e: - raise Exception("Error for POST {0}".format(e)) + raise Exception(f"Error for POST {e}") response_dict = handle_response(response) external_id = response_dict['trackingId'] cert = response_dict['endEntityCert'] - chain = response_dict['chainCerts'][1] - current_app.logger.info( - "Received Chain: {0}".format(chain) - ) + if len(response_dict['chainCerts']) < 2: + # certificate signed by CA directly, no ICA included ini the chain + chain = None + else: + chain = response_dict['chainCerts'][1] + + log_data["message"] = "Received Chain" + log_data["options"] = f"chain: {chain}" + current_app.logger.info(log_data) return cert, chain, external_id def revoke_certificate(self, certificate, comments): - """Revoke a Digicert certificate.""" + """Revoke an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") # make certificate revoke request - revoke_url = "{0}/certificates/{1}/revocations".format( - base_url, certificate.external_id - ) - metrics.send("entrust_revoke_certificate", "counter", 1) - if comments == '' or not comments: + revoke_url = f"{base_url}/certificates/{certificate.external_id}/revocations" + if not comments or comments == '': comments = "revoked via API" data = { - "crlReason": "superseded", + "crlReason": "superseded", # enum (keyCompromise, affiliationChanged, superseded, cessationOfOperation) "revocationComment": comments } response = self.session.post(revoke_url, json=data) + metrics.send("entrust_revoke_certificate", "counter", 1) + return handle_response(response) - data = handle_response(response) + def deactivate_certificate(self, certificate): + """Deactivates an Entrust certificate.""" + base_url = current_app.config.get("ENTRUST_URL") + deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" + response = self.session.post(deactivate_url) + metrics.send("entrust_deactivate_certificate", "counter", 1) + return handle_response(response) @staticmethod def create_authority(options): @@ -200,7 +227,8 @@ class EntrustIssuerPlugin(IssuerPlugin): entrust_root = current_app.config.get("ENTRUST_ROOT") entrust_issuing = current_app.config.get("ENTRUST_ISSUING") role = {"username": "", "password": "", "name": "entrust"} - current_app.logger.info("Creating Auth: {0} {1}".format(options, entrust_issuing)) + current_app.logger.info(f"Creating Auth: {options} {entrust_issuing}") + # body, chain, role return entrust_root, "", [role] def get_ordered_certificate(self, order_id): @@ -211,10 +239,10 @@ class EntrustIssuerPlugin(IssuerPlugin): class EntrustSourcePlugin(SourcePlugin): - title = "ENTRUST" + title = "Entrust" slug = "entrust-source" - description = "Enables the collecion of certificates" - version = ENTRUST.VERSION + description = "Enables the collection of certificates" + version = entrust.VERSION author = "sirferl" author_url = "https://github.com/sirferl/lemur" diff --git a/lemur/plugins/lemur_entrust/tests/conftest.py b/lemur/plugins/lemur_entrust/tests/conftest.py new file mode 100644 index 00000000..0e1cd89f --- /dev/null +++ b/lemur/plugins/lemur_entrust/tests/conftest.py @@ -0,0 +1 @@ +from lemur.tests.conftest import * # noqa diff --git a/lemur/plugins/lemur_entrust/tests/test_entrust.py b/lemur/plugins/lemur_entrust/tests/test_entrust.py new file mode 100644 index 00000000..b1cd4c83 --- /dev/null +++ b/lemur/plugins/lemur_entrust/tests/test_entrust.py @@ -0,0 +1,54 @@ +from unittest.mock import patch, Mock + +import arrow +from cryptography import x509 +from lemur.plugins.lemur_entrust import plugin + + +def config_mock(*args): + values = { + "ENTRUST_API_CERT": "-----BEGIN CERTIFICATE-----abc-----END CERTIFICATE-----", + "ENTRUST_API_KEY": False, + "ENTRUST_API_USER": "test", + "ENTRUST_API_PASS": "password", + "ENTRUST_URL": "http", + "ENTRUST_ROOT": None, + "ENTRUST_NAME": "test", + "ENTRUST_EMAIL": "test@lemur.net", + "ENTRUST_PHONE": "0123456", + "ENTRUST_PRODUCT_ENTRUST": "ADVANTAGE_SSL" + } + return values[args[0]] + + +@patch("lemur.plugins.lemur_entrust.plugin.current_app") +def test_process_options(mock_current_app, authority): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2020, 10, 7).format('YYYY-MM-DD')) + + authority.name = "Entrust" + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "organization": "Example, Inc.", + "organizational_unit": "Example Org", + "validity_end": arrow.get(2020, 10, 7), + "authority": authority, + } + + expected = { + "signingAlg": "SHA-2", + "eku": "SERVER_AND_CLIENT_AUTH", + "certType": "ADVANTAGE_SSL", + "certExpiryDate": arrow.get(2020, 10, 7).format('YYYY-MM-DD'), + "tracking": { + "requesterName": mock_current_app.config.get("ENTRUST_NAME"), + "requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"), + "requesterPhone": mock_current_app.config.get("ENTRUST_PHONE") + } + } + + assert expected == plugin.process_options(options) diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index d3badbeb..dc6101b0 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -15,7 +15,6 @@ def get_random_secret(length): secret_key = secret_key + ''.join(random.choice(string.ascii_lowercase) for x in range(round(length / 4))) return secret_key + ''.join(random.choice(string.digits) for x in range(round(length / 4))) - THREADS_PER_PAGE = 8 # General @@ -100,7 +99,6 @@ DIGICERT_CIS_API_KEY = "api-key" DIGICERT_CIS_ROOTS = {"root": "ROOT"} DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"} - VERISIGN_URL = "http://example.com" VERISIGN_PEM_PATH = "~/" VERISIGN_FIRST_NAME = "Jim" @@ -211,3 +209,41 @@ LDAP_REQUIRED_GROUP = "Lemur Access" LDAP_DEFAULT_ROLE = "role1" ALLOW_CERT_DELETION = True + +ENTRUST_API_CERT = "api-cert" +ENTRUST_API_KEY = get_random_secret(32) +ENTRUST_API_USER = "user" +ENTRUST_API_PASS = get_random_secret(32) +ENTRUST_URL = "https://api.entrust.net/enterprise/v2" +ENTRUST_ROOT = """ +-----BEGIN CERTIFICATE----- +MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50 +cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs +IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz +dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy +NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu +dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt +dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0 +aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T +RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN +cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW +wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1 +U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0 +jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP +BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN +BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/ +jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ +Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v +1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R +nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH +VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g== +-----END CERTIFICATE----- +""" +ENTRUST_NAME = "lemur" +ENTRUST_EMAIL = "lemur@example.com" +ENTRUST_PHONE = "123456" +ENTRUST_ISSUING = "" +ENTRUST_PRODUCT_ENTRUST = "ADVANTAGE_SSL" diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 1dac39bb..162e53b0 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -2,11 +2,13 @@ import pytest from lemur.tests.vectors import ( SAN_CERT, + SAN_CERT_STR, INTERMEDIATE_CERT, ROOTCA_CERT, EC_CERT_EXAMPLE, ECDSA_PRIME256V1_CERT, ECDSA_SECP384r1_CERT, + ECDSA_SECP384r1_CERT_STR, DSA_CERT, ) @@ -106,3 +108,9 @@ def test_is_selfsigned(selfsigned_cert): # unsupported algorithm (DSA) with pytest.raises(Exception): is_selfsigned(DSA_CERT) + + +def test_get_key_type_from_certificate(): + from lemur.common.utils import get_key_type_from_certificate + assert (get_key_type_from_certificate(SAN_CERT_STR) == "RSA2048") + assert (get_key_type_from_certificate(ECDSA_SECP384r1_CERT_STR) == "ECCSECP384R1")