From fae37932552b6f60a0046c01ccbdae2075eb655d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 11:09:32 -0700 Subject: [PATCH 01/16] entrrust plugin revised --- lemur/plugins/lemur_entrust/plugin.py | 90 ++++++++++++++++++--------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 315da8bd..3669d9d6 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -1,9 +1,12 @@ -from lemur.plugins.bases import IssuerPlugin, SourcePlugin + import arrow import requests import json -from lemur.plugins import lemur_entrust as ENTRUST +import sys from flask import current_app + +from lemur.plugins import lemur_entrust as entrust +from lemur.plugins.bases import IssuerPlugin, SourcePlugin from lemur.extensions import metrics from lemur.common.utils import validate_conf @@ -24,17 +27,17 @@ def determine_end_date(end_date): """ Determine appropriate end date :param end_date: - :return: validity_end + :return: validity_end as string """ # ENTRUST only allows 13 months of max certificate duration - max_validity_end = arrow.utcnow().shift(years=1, months=+1).format('YYYY-MM-DD') + max_validity_end = arrow.utcnow().shift(years=1, months=+1) if not end_date: end_date = max_validity_end if end_date > max_validity_end: end_date = max_validity_end - return end_date + return end_date.format('YYYY-MM-DD') def process_options(options): @@ -49,6 +52,9 @@ def process_options(options): # take the value as Cert product-type # else default to "STANDARD_SSL" authority = options.get("authority").name.upper() + # STANDARD_SSL (cn=domain, san=www.domain), + # ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]), + # WILDCARD_SSL (unlimited sans, and wildcard) product_type = current_app.config.get("ENTRUST_PRODUCT_{0}".format(authority), "STANDARD_SSL") if options.get("validity_end"): @@ -67,6 +73,7 @@ def process_options(options): "eku": "SERVER_AND_CLIENT_AUTH", "certType": product_type, "certExpiryDate": validity_end, + # "keyType": "RSA", Entrust complaining about this parameter "tracking": tracking_data } return data @@ -86,23 +93,31 @@ def handle_response(my_response): 404: "Unknown jobId", 429: "Too many requests" } + try: d = json.loads(my_response.content) - except Exception as e: + except ValueError: # catch an empty jason object here - d = {'errors': 'No detailled message'} + d = {'response': 'No detailed message'} s = my_response.status_code if s > 399: - raise Exception("ENTRUST error: {0}\n{1}".format(msg.get(s, s), d['errors'])) - current_app.logger.info("Response: {0}, {1} ".format(s, d)) + raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}") + + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Response", + "status": s, + "response": d + } + current_app.logger.info(log_data) return d class EntrustIssuerPlugin(IssuerPlugin): - title = "ENTRUST" + title = "Entrust" slug = "entrust-issuer" description = "Enables the creation of certificates by ENTRUST" - version = ENTRUST.VERSION + version = entrust.VERSION author = "sirferl" author_url = "https://github.com/sirferl/lemur" @@ -119,7 +134,6 @@ class EntrustIssuerPlugin(IssuerPlugin): "ENTRUST_NAME", "ENTRUST_EMAIL", "ENTRUST_PHONE", - "ENTRUST_ISSUING", ] validate_conf(current_app, required_vars) @@ -142,9 +156,12 @@ class EntrustIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ - current_app.logger.info( - "Requesting options: {0}".format(issuer_options) - ) + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Requesting options", + "options": issuer_options + } + current_app.logger.info(log_data) url = current_app.config.get("ENTRUST_URL") + "/certificates" @@ -161,31 +178,41 @@ class EntrustIssuerPlugin(IssuerPlugin): response_dict = handle_response(response) external_id = response_dict['trackingId'] cert = response_dict['endEntityCert'] - chain = response_dict['chainCerts'][1] - current_app.logger.info( - "Received Chain: {0}".format(chain) - ) + if len(response_dict['chainCerts']) < 2: + # certificate signed by CA directly, no ICA included ini the chain + chain = None + else: + chain = response_dict['chainCerts'][1] + + log_data["message"] = "Received Chain" + log_data["options"] = f"chain: {chain}" + current_app.logger.info(log_data) return cert, chain, external_id def revoke_certificate(self, certificate, comments): - """Revoke a Digicert certificate.""" + """Revoke an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") # make certificate revoke request - revoke_url = "{0}/certificates/{1}/revocations".format( - base_url, certificate.external_id - ) - metrics.send("entrust_revoke_certificate", "counter", 1) - if comments == '' or not comments: + revoke_url = f"{base_url}/certificates/{certificate.external_id}/revocations" + if not comments or comments == '': comments = "revoked via API" data = { - "crlReason": "superseded", + "crlReason": "superseded", # enum (keyCompromise, affiliationChanged, superseded, cessationOfOperation) "revocationComment": comments } response = self.session.post(revoke_url, json=data) + metrics.send("entrust_revoke_certificate", "counter", 1) + return handle_response(response) - data = handle_response(response) + def deactivate_certificate(self, certificate, comments): + """Deactivates an Entrust certificate.""" + base_url = current_app.config.get("ENTRUST_URL") + revoke_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" + response = self.session.post(revoke_url) + metrics.send("entrust_revoke_certificate", "counter", 1) + return handle_response(response) @staticmethod def create_authority(options): @@ -200,7 +227,8 @@ class EntrustIssuerPlugin(IssuerPlugin): entrust_root = current_app.config.get("ENTRUST_ROOT") entrust_issuing = current_app.config.get("ENTRUST_ISSUING") role = {"username": "", "password": "", "name": "entrust"} - current_app.logger.info("Creating Auth: {0} {1}".format(options, entrust_issuing)) + current_app.logger.info(f"Creating Auth: {options} {entrust_issuing}") + # body, chain, role return entrust_root, "", [role] def get_ordered_certificate(self, order_id): @@ -211,10 +239,10 @@ class EntrustIssuerPlugin(IssuerPlugin): class EntrustSourcePlugin(SourcePlugin): - title = "ENTRUST" + title = "Entrust" slug = "entrust-source" - description = "Enables the collecion of certificates" - version = ENTRUST.VERSION + description = "Enables the collection of certificates" + version = entrust.VERSION author = "sirferl" author_url = "https://github.com/sirferl/lemur" From 416f39222a7375c1a31462cc4c24b9ab025b1b5d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 17:02:19 -0700 Subject: [PATCH 02/16] testing --- lemur/plugins/lemur_entrust/tests/conftest.py | 1 + .../lemur_entrust/tests/test_entrust.py | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 lemur/plugins/lemur_entrust/tests/conftest.py create mode 100644 lemur/plugins/lemur_entrust/tests/test_entrust.py diff --git a/lemur/plugins/lemur_entrust/tests/conftest.py b/lemur/plugins/lemur_entrust/tests/conftest.py new file mode 100644 index 00000000..0e1cd89f --- /dev/null +++ b/lemur/plugins/lemur_entrust/tests/conftest.py @@ -0,0 +1 @@ +from lemur.tests.conftest import * # noqa diff --git a/lemur/plugins/lemur_entrust/tests/test_entrust.py b/lemur/plugins/lemur_entrust/tests/test_entrust.py new file mode 100644 index 00000000..b1cd4c83 --- /dev/null +++ b/lemur/plugins/lemur_entrust/tests/test_entrust.py @@ -0,0 +1,54 @@ +from unittest.mock import patch, Mock + +import arrow +from cryptography import x509 +from lemur.plugins.lemur_entrust import plugin + + +def config_mock(*args): + values = { + "ENTRUST_API_CERT": "-----BEGIN CERTIFICATE-----abc-----END CERTIFICATE-----", + "ENTRUST_API_KEY": False, + "ENTRUST_API_USER": "test", + "ENTRUST_API_PASS": "password", + "ENTRUST_URL": "http", + "ENTRUST_ROOT": None, + "ENTRUST_NAME": "test", + "ENTRUST_EMAIL": "test@lemur.net", + "ENTRUST_PHONE": "0123456", + "ENTRUST_PRODUCT_ENTRUST": "ADVANTAGE_SSL" + } + return values[args[0]] + + +@patch("lemur.plugins.lemur_entrust.plugin.current_app") +def test_process_options(mock_current_app, authority): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2020, 10, 7).format('YYYY-MM-DD')) + + authority.name = "Entrust" + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "organization": "Example, Inc.", + "organizational_unit": "Example Org", + "validity_end": arrow.get(2020, 10, 7), + "authority": authority, + } + + expected = { + "signingAlg": "SHA-2", + "eku": "SERVER_AND_CLIENT_AUTH", + "certType": "ADVANTAGE_SSL", + "certExpiryDate": arrow.get(2020, 10, 7).format('YYYY-MM-DD'), + "tracking": { + "requesterName": mock_current_app.config.get("ENTRUST_NAME"), + "requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"), + "requesterPhone": mock_current_app.config.get("ENTRUST_PHONE") + } + } + + assert expected == plugin.process_options(options) From edab32d9a13cbde2916424d923d1b2ab9aea4b0c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 17:03:22 -0700 Subject: [PATCH 03/16] setting the required entrust configs --- lemur/tests/conf.py | 48 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index af0c09ce..bf033421 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -1,9 +1,18 @@ # This is just Python which means you can inherit and tweak settings import os +import random +import string _basedir = os.path.abspath(os.path.dirname(__file__)) + +# generate random secrets for unittest +def get_random_secret(length): + input_ascii = string.ascii_letters + string.digits + return ''.join(random.choice(input_ascii) for i in range(length)) + + THREADS_PER_PAGE = 8 # General @@ -86,7 +95,6 @@ DIGICERT_CIS_API_KEY = "api-key" DIGICERT_CIS_ROOTS = {"root": "ROOT"} DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"} - VERISIGN_URL = "http://example.com" VERISIGN_PEM_PATH = "~/" VERISIGN_FIRST_NAME = "Jim" @@ -197,3 +205,41 @@ LDAP_REQUIRED_GROUP = "Lemur Access" LDAP_DEFAULT_ROLE = "role1" ALLOW_CERT_DELETION = True + +ENTRUST_API_CERT = "api-cert" +ENTRUST_API_KEY = get_random_secret(32) +ENTRUST_API_USER = "user" +ENTRUST_API_PASS = get_random_secret(32) +ENTRUST_URL = "https://api.entrust.net/enterprise/v2" +ENTRUST_ROOT = """ +-----BEGIN CERTIFICATE----- +MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50 +cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs +IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz +dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy +NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu +dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt +dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0 +aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T +RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN +cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW +wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1 +U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0 +jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP +BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN +BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/ +jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ +Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v +1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R +nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH +VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g== +-----END CERTIFICATE----- +""" +ENTRUST_NAME = "lemur" +ENTRUST_EMAIL = "lemur@example.com" +ENTRUST_PHONE = "123456" +ENTRUST_ISSUING = "" +ENTRUST_PRODUCT_ENTRUST = "ADVANTAGE_SSL" From cc855e27582a6a04db84006c53e1149dc3546f07 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 17:16:07 -0700 Subject: [PATCH 04/16] modern python style --- lemur/plugins/lemur_entrust/plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 3669d9d6..50b9d929 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -20,7 +20,7 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ - metrics.send("ENTRUST_status_code_{}".format(r.status_code), "counter", 1) + metrics.send("entrust_status_code", "counter", 1, metadata={"status_code": r.status_code}) def determine_end_date(end_date): @@ -55,7 +55,7 @@ def process_options(options): # STANDARD_SSL (cn=domain, san=www.domain), # ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]), # WILDCARD_SSL (unlimited sans, and wildcard) - product_type = current_app.config.get("ENTRUST_PRODUCT_{0}".format(authority), "STANDARD_SSL") + product_type = current_app.config.get(f"ENTRUST_PRODUCT_{authority}", "STANDARD_SSL") if options.get("validity_end"): validity_end = determine_end_date(options.get("validity_end")) @@ -173,7 +173,7 @@ class EntrustIssuerPlugin(IssuerPlugin): except requests.exceptions.Timeout: raise Exception("Timeout for POST") except requests.exceptions.RequestException as e: - raise Exception("Error for POST {0}".format(e)) + raise Exception(f"Error for POST {e}") response_dict = handle_response(response) external_id = response_dict['trackingId'] From 8de9842092d073852907a0242503e941e2d7c0e3 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:22:45 -0700 Subject: [PATCH 05/16] Backfill the key_type column: DB Upgrade --- lemur/common/utils.py | 17 ++++ lemur/migrations/versions/c301c59688d2_.py | 108 +++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 lemur/migrations/versions/c301c59688d2_.py diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 01cc64ae..283d1eec 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -71,6 +71,23 @@ def parse_private_key(private_key): ) +def get_key_type_from_certificate(body): + """ + + Helper function to determine key type by pasrding given PEM certificate + + :param body: PEM string + :return: Key type string + """ + parsed_cert = parse_certificate(body) + if isinstance(parsed_cert.public_key(), rsa.RSAPublicKey): + return "RSA{key_size}".format( + key_size=parsed_cert.public_key().key_size + ) + elif isinstance(parsed_cert.public_key(), ec.EllipticCurvePublicKey): + return get_key_type_from_ec_curve(parsed_cert.public_key().curve.name) + + def split_pem(data): """ Split a string of several PEM payloads to a list of strings. diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py new file mode 100644 index 00000000..6bd94cfb --- /dev/null +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -0,0 +1,108 @@ +""" + +This upgrade of database updates the key_type information for certificates +that are either still valid or have expired in last 30 days. For RSA keys, +the algorithm is determined based on the key length. For rest of the keys, +the certificate body is parsed to determine the exact key type information. + +Each individual change is explicitly committed. The logs are added to file +named upgrade_logs in current working directory. If faced any issue while +running this upgrade, there is no harm in re-running the upgrade. Each run +processes only the keys for which key type information is not yet determined. +A successful end to end run will end up updating the Alembic Version to new +Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported +by Lemur. This could be a long running job depending upon the number of +keys it may process. + +Revision ID: c301c59688d2 +Revises: 434c29e40511 +Create Date: 2020-09-21 14:28:50.757998 + +""" + +# revision identifiers, used by Alembic. +revision = 'c301c59688d2' +down_revision = '434c29e40511' + +from alembic import op +from sqlalchemy.sql import text +from lemur.common import utils +import time + +log_file = open('upgrade_logs', 'a') + + +def upgrade(): + log_file.write("\n*** Starting new run ***\n") + start_time = time.time() + + # Update RSA keys using the key length information + update_key_type_rsa(1024) + update_key_type_rsa(2048) + update_key_type_rsa(4096) + + # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs. + update_key_type() + + log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time)) + log_file.close() + + +def downgrade(): + # Change key type column back to null + # Going back 32 days instead of 31 to make sure no certificates are skipped + stmt = text( + "update certificates set key_type=null where not_after > CURRENT_DATE - 32" + ) + op.execute(stmt) + + +""" + Helper methods performing updates for RSA and rest of the keys +""" + + +def update_key_type_rsa(bits): + log_file.write("Processing certificate with key type RSA %s\n" % bits) + + stmt = text( + "update certificates set key_type='RSA{0}' where bits={0} and not_after > CURRENT_DATE - 31 and key_type is null".format(bits) + ) + log_file.write("Query: %s\n" % stmt) + + start_time = time.time() + op.execute(stmt) + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def update_key_type(): + conn = op.get_bind() + start_time = time.time() + + # Loop through all certificates are valid today or expired in last 30 days + for cert_id, body in conn.execute( + text( + "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null") + ): + try: + cert_key_type = utils.get_key_type_from_certificate(body) + except ValueError: + log_file.write("Error in processing certificate. ID: %s\n" % cert_id) + else: + log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + stmt = text( + "update certificates set key_type=:key_type where id=:id" + ) + stmt = stmt.bindparams(key_type=cert_key_type, id=cert_id) + op.execute(stmt) + + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def commit(): + stmt = text("commit") + op.execute(stmt) From 9211178e77b1c78a17775c27e1caaee8d0e0756f Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:31:38 -0700 Subject: [PATCH 06/16] Added date-time and modified log file name --- lemur/migrations/versions/c301c59688d2_.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 6bd94cfb..2c91783f 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -28,12 +28,13 @@ from alembic import op from sqlalchemy.sql import text from lemur.common import utils import time +import datetime -log_file = open('upgrade_logs', 'a') +log_file = open('db_upgrade.log', 'a') def upgrade(): - log_file.write("\n*** Starting new run ***\n") + log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) start_time = time.time() # Update RSA keys using the key length information From 921e8d8236d3be4b1bf9cfa044fe581f0744a2f2 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:46:15 -0700 Subject: [PATCH 07/16] Add error message to the logs --- lemur/migrations/versions/c301c59688d2_.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 2c91783f..0dd46315 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -6,9 +6,12 @@ the algorithm is determined based on the key length. For rest of the keys, the certificate body is parsed to determine the exact key type information. Each individual change is explicitly committed. The logs are added to file -named upgrade_logs in current working directory. If faced any issue while -running this upgrade, there is no harm in re-running the upgrade. Each run -processes only the keys for which key type information is not yet determined. +named db_upgrade.log in current working directory. Any error encountered +while parsing a certificate will also be logged along with the certificate +ID. If faced any issue while running this upgrade, there is no harm in +re-running the upgrade. Each run processes only the keys for which key type +information is not yet determined. + A successful end to end run will end up updating the Alembic Version to new Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported by Lemur. This could be a long running job depending upon the number of @@ -89,8 +92,8 @@ def update_key_type(): ): try: cert_key_type = utils.get_key_type_from_certificate(body) - except ValueError: - log_file.write("Error in processing certificate. ID: %s\n" % cert_id) + except ValueError as e: + log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) else: log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) stmt = text( From e3fa0726080b6a10434ff1a68871fe14998a133a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 10:17:30 -0700 Subject: [PATCH 08/16] Update c301c59688d2_.py language --- lemur/migrations/versions/c301c59688d2_.py | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 0dd46315..8f1941b2 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -1,21 +1,23 @@ """ -This upgrade of database updates the key_type information for certificates -that are either still valid or have expired in last 30 days. For RSA keys, -the algorithm is determined based on the key length. For rest of the keys, -the certificate body is parsed to determine the exact key type information. +This database upgrade updates the key_type information for either +still valid or expired certificates in the last 30 days. For RSA +keys, the algorithm is determined based on the key length. For +the rest of the keys, the certificate body is parsed to determine +the exact key_type information. -Each individual change is explicitly committed. The logs are added to file -named db_upgrade.log in current working directory. Any error encountered -while parsing a certificate will also be logged along with the certificate -ID. If faced any issue while running this upgrade, there is no harm in -re-running the upgrade. Each run processes only the keys for which key type -information is not yet determined. +Each individual DB change is explicitly committed, and the +respective log is added to a file named db_upgrade.log in the current +working directory. Any error encountered while parsing a certificate +will also be logged along with the certificate ID. If faced with +any issue while running this upgrade, there is no harm in +re-running the upgrade. Each run processes only rows for which +key_type information is not yet determined. -A successful end to end run will end up updating the Alembic Version to new -Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported -by Lemur. This could be a long running job depending upon the number of -keys it may process. +A successful complete run will end up updating the Alembic Version +to the new Revision ID c301c59688d2. Currently, only RSA and ECC +certificates are supported by Lemur. This could be a long-running +job depending upon the number of DB entries it may process. Revision ID: c301c59688d2 Revises: 434c29e40511 From 19b693f636a77b7a460e2a24f9d557aa410f1a7a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 10:21:23 -0700 Subject: [PATCH 09/16] Update c301c59688d2_.py language --- lemur/migrations/versions/c301c59688d2_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 8f1941b2..c96272ff 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -87,7 +87,7 @@ def update_key_type(): conn = op.get_bind() start_time = time.time() - # Loop through all certificates are valid today or expired in last 30 days + # Loop through all certificates that are valid today or expired in the last 30 days. for cert_id, body in conn.execute( text( "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null") From 710290f590fca0b523b73a9f35d213fc6cad6879 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 23 Sep 2020 11:45:36 -0700 Subject: [PATCH 10/16] Formatting changes --- lemur/migrations/versions/c301c59688d2_.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index c96272ff..3b0a86f7 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -6,18 +6,18 @@ keys, the algorithm is determined based on the key length. For the rest of the keys, the certificate body is parsed to determine the exact key_type information. -Each individual DB change is explicitly committed, and the -respective log is added to a file named db_upgrade.log in the current -working directory. Any error encountered while parsing a certificate -will also be logged along with the certificate ID. If faced with -any issue while running this upgrade, there is no harm in -re-running the upgrade. Each run processes only rows for which -key_type information is not yet determined. +Each individual DB change is explicitly committed, and the respective +log is added to a file named db_upgrade.log in the current working +directory. Any error encountered while parsing a certificate will +also be logged along with the certificate ID. If faced with any issue +while running this upgrade, there is no harm in re-running the upgrade. +Each run processes only rows for which key_type information is not yet +determined. -A successful complete run will end up updating the Alembic Version -to the new Revision ID c301c59688d2. Currently, only RSA and ECC -certificates are supported by Lemur. This could be a long-running -job depending upon the number of DB entries it may process. +A successful complete run will end up updating the Alembic Version to +the new Revision ID c301c59688d2. Currently, Lemur supports only RSA +and ECC certificates. This could be a long-running job depending upon +the number of DB entries it may process. Revision ID: c301c59688d2 Revises: 434c29e40511 @@ -72,7 +72,7 @@ def update_key_type_rsa(bits): log_file.write("Processing certificate with key type RSA %s\n" % bits) stmt = text( - "update certificates set key_type='RSA{0}' where bits={0} and not_after > CURRENT_DATE - 31 and key_type is null".format(bits) + f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null" ) log_file.write("Query: %s\n" % stmt) From 12af0ecb457263f9dad344c4af458de473ff6a59 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 23 Sep 2020 11:46:38 -0700 Subject: [PATCH 11/16] UT get_key_type_from_certificate --- lemur/tests/test_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 1dac39bb..162e53b0 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -2,11 +2,13 @@ import pytest from lemur.tests.vectors import ( SAN_CERT, + SAN_CERT_STR, INTERMEDIATE_CERT, ROOTCA_CERT, EC_CERT_EXAMPLE, ECDSA_PRIME256V1_CERT, ECDSA_SECP384r1_CERT, + ECDSA_SECP384r1_CERT_STR, DSA_CERT, ) @@ -106,3 +108,9 @@ def test_is_selfsigned(selfsigned_cert): # unsupported algorithm (DSA) with pytest.raises(Exception): is_selfsigned(DSA_CERT) + + +def test_get_key_type_from_certificate(): + from lemur.common.utils import get_key_type_from_certificate + assert (get_key_type_from_certificate(SAN_CERT_STR) == "RSA2048") + assert (get_key_type_from_certificate(ECDSA_SECP384r1_CERT_STR) == "ECCSECP384R1") From 1983eb79de4b46ec2bbb0e194fe2bce0c333ddfd Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 13:00:14 +0200 Subject: [PATCH 12/16] Add paragraph about reusing ACME accounts --- docs/production/index.rst | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/production/index.rst b/docs/production/index.rst index 9f90c0cc..21fca650 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -511,3 +511,45 @@ The following must be added to the config file to activate the pinning (the pinn KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg== -----END CERTIFICATE----- """ + + +LetsEncrypt: Using a pre-existing ACME account +----------------------------------------------- + +Let's Encrypt allows reusing an existing ACME account, to create and especially revoke certificates. The current +implementation in the acme plugin, only allows for a single account for all ACME authorities, which might be an issue, +when you try to use Let's Encrypt together with another certificate authority that uses the ACME protocol. + +To use an existing account, you need to configure the `ACME_PRIVATE_KEY` and `ACME_REGR` variables in the lemur +configuration. + +`ACME_PRIVATE_KEY` needs to be in the JWK format:: + + { + "kty": "RSA", + "n": "yr1qBwHizA7ME_iV32bY10ILp.....", + "e": "AQAB", + "d": "llBlYhil3I.....", + "p": "-5LW2Lewogo.........", + "q": "zk6dHqHfHksd.........", + "dp": "qfe9fFIu3mu.......", + "dq": "cXFO-loeOyU.......", + "qi": "AfK1sh0_8sLTb..........." + } + + +Using `python-jwt` converting a existing private key in PEM format is quite easy:: + + import python_jwt as jwt, jwcrypto.jwk as jwk + + priv_key = jwk.JWK.from_pem(b"""-----BEGIN RSA PRIVATE KEY----- + ... + -----END RSA PRIVATE KEY-----""") + + print(priv_key.export()) + +`ACME_REGR` needs to be a valid JSON with a `body` and a `uri` attribute, similar to this:: + + {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/"} + +The uri can be retrieved from the ACME create account endpoint, when trying to create a new account, using the existing key. \ No newline at end of file From ae1ead6d7551dab0435e2f553bf2af469158aa4f Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 13:04:51 +0200 Subject: [PATCH 13/16] Document ACME plugin specific configurations --- docs/administration.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 0cec16a0..4c0477aa 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1171,6 +1171,23 @@ The following configuration properties are required to use the PowerDNS ACME Plu File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. +ACME Plugin +~~~~~~~~~~~~ + +The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME +account. + + +.. data:: ACME_PRIVATE_KEY + :noindex: + + This is the private key, the account was registered with (in JWK format) + +.. data:: ACME_REGR + :noindex: + + This is the registration for the ACME account, the most important part is the uri attribute (in JSON) + .. _CommandLineInterface: Command Line Interface From 4f1e09e3afe88c5d04918e0acf1da8effc0c1b16 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 13:20:35 +0200 Subject: [PATCH 14/16] Add reference from configuration options, to more detailed explanation --- docs/administration.rst | 2 +- docs/production/index.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 4c0477aa..f44ad1a3 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1175,7 +1175,7 @@ ACME Plugin ~~~~~~~~~~~~ The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME -account. +account. See :ref:`Using a pre-existing ACME account ` for more details. .. data:: ACME_PRIVATE_KEY diff --git a/docs/production/index.rst b/docs/production/index.rst index 21fca650..bace15d3 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -513,6 +513,8 @@ The following must be added to the config file to activate the pinning (the pinn """ +.. _AcmeAccountReuse: + LetsEncrypt: Using a pre-existing ACME account ----------------------------------------------- From 21c2255c754b48fd2a1887cad64098fa73b99f03 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 19:14:09 +0200 Subject: [PATCH 15/16] Minor spelling improvements --- docs/production/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/production/index.rst b/docs/production/index.rst index bace15d3..c6f561ca 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -540,7 +540,7 @@ configuration. } -Using `python-jwt` converting a existing private key in PEM format is quite easy:: +Using `python-jwt` converting an existing private key in PEM format is quite easy:: import python_jwt as jwt, jwcrypto.jwk as jwk @@ -554,4 +554,4 @@ Using `python-jwt` converting a existing private key in PEM format is quite easy {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/"} -The uri can be retrieved from the ACME create account endpoint, when trying to create a new account, using the existing key. \ No newline at end of file +The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key. \ No newline at end of file From e5961146b9c8bcb2c83fabcface46c578c0da2e5 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 14:22:58 -0600 Subject: [PATCH 16/16] session hook complains about metadata + consistent language. --- lemur/plugins/lemur_entrust/plugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 50b9d929..9b7848ed 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -20,7 +20,7 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ - metrics.send("entrust_status_code", "counter", 1, metadata={"status_code": r.status_code}) + metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1) def determine_end_date(end_date): @@ -206,12 +206,12 @@ class EntrustIssuerPlugin(IssuerPlugin): metrics.send("entrust_revoke_certificate", "counter", 1) return handle_response(response) - def deactivate_certificate(self, certificate, comments): + def deactivate_certificate(self, certificate): """Deactivates an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") - revoke_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" - response = self.session.post(revoke_url) - metrics.send("entrust_revoke_certificate", "counter", 1) + deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" + response = self.session.post(deactivate_url) + metrics.send("entrust_deactivate_certificate", "counter", 1) return handle_response(response) @staticmethod