Merge branch 'master' of github.com:Netflix/lemur into key_type_column

docs/administration.rst

@@ -1171,6 +1171,23 @@ The following configuration properties are required to use the PowerDNS ACME Plu
 
             File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle.
 
+ACME Plugin
+~~~~~~~~~~~~
+
+The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
+account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
+
+
+.. data:: ACME_PRIVATE_KEY
+    :noindex:
+
+            This is the private key, the account was registered with (in JWK format)
+
+.. data:: ACME_REGR
+    :noindex:
+
+            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
+
 .. _CommandLineInterface:
 
 Command Line Interface

docs/production/index.rst

@@ -511,3 +511,47 @@ The following must be added to the config file to activate the pinning (the pinn
     KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
     -----END CERTIFICATE-----
     """
+
+
+.. _AcmeAccountReuse:
+
+LetsEncrypt: Using a pre-existing ACME account
+-----------------------------------------------
+
+Let's Encrypt allows reusing an existing ACME account, to create and especially revoke certificates. The current
+implementation in the acme plugin, only allows for a single account for all ACME authorities, which might be an issue,
+when you try to use Let's Encrypt together with another certificate authority that uses the ACME protocol.
+
+To use an existing account, you need to configure the `ACME_PRIVATE_KEY` and `ACME_REGR` variables in the lemur
+configuration.
+
+`ACME_PRIVATE_KEY` needs to be in the JWK format::
+
+    {
+        "kty": "RSA",
+        "n": "yr1qBwHizA7ME_iV32bY10ILp.....",
+        "e": "AQAB",
+        "d": "llBlYhil3I.....",
+        "p": "-5LW2Lewogo.........",
+        "q": "zk6dHqHfHksd.........",
+        "dp": "qfe9fFIu3mu.......",
+        "dq": "cXFO-loeOyU.......",
+        "qi": "AfK1sh0_8sLTb..........."
+    }
+
+
+Using `python-jwt` converting an existing private key in PEM format is quite easy::
+
+    import python_jwt as jwt, jwcrypto.jwk as jwk
+
+    priv_key = jwk.JWK.from_pem(b"""-----BEGIN RSA PRIVATE KEY-----
+    ...
+    -----END RSA PRIVATE KEY-----""")
+
+    print(priv_key.export())
+
+`ACME_REGR` needs to be a valid JSON with a `body` and a `uri` attribute, similar to this::
+
+    {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/<ACCOUNT_NUMBER>"}
+
+The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key.

lemur/plugins/lemur_entrust/plugin.py

@@ -1,9 +1,12 @@
-from lemur.plugins.bases import IssuerPlugin, SourcePlugin
+
 import arrow
 import requests
 import json
-from lemur.plugins import lemur_entrust as ENTRUST
+import sys
 from flask import current_app
+
+from lemur.plugins import lemur_entrust as entrust
+from lemur.plugins.bases import IssuerPlugin, SourcePlugin
 from lemur.extensions import metrics
 from lemur.common.utils import validate_conf
 
@@ -17,24 +20,24 @@ def log_status_code(r, *args, **kwargs):
     :param kwargs:
     :return:
     """
-    metrics.send("ENTRUST_status_code_{}".format(r.status_code), "counter", 1)
+    metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1)
 
 
 def determine_end_date(end_date):
     """
     Determine appropriate end date
     :param end_date:
-    :return: validity_end
+    :return: validity_end as string
     """
     # ENTRUST only allows 13 months of max certificate duration
-    max_validity_end = arrow.utcnow().shift(years=1, months=+1).format('YYYY-MM-DD')
+    max_validity_end = arrow.utcnow().shift(years=1, months=+1)
 
     if not end_date:
         end_date = max_validity_end
 
     if end_date > max_validity_end:
         end_date = max_validity_end
-    return end_date
+    return end_date.format('YYYY-MM-DD')
 
 
 def process_options(options):
@@ -49,7 +52,10 @@ def process_options(options):
     # take the value as Cert product-type
     # else default to "STANDARD_SSL"
     authority = options.get("authority").name.upper()
-    product_type = current_app.config.get("ENTRUST_PRODUCT_{0}".format(authority), "STANDARD_SSL")
+    # STANDARD_SSL (cn=domain, san=www.domain),
+    # ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]),
+    # WILDCARD_SSL (unlimited sans, and wildcard)
+    product_type = current_app.config.get(f"ENTRUST_PRODUCT_{authority}", "STANDARD_SSL")
 
     if options.get("validity_end"):
         validity_end = determine_end_date(options.get("validity_end"))
@@ -67,6 +73,7 @@ def process_options(options):
         "eku": "SERVER_AND_CLIENT_AUTH",
         "certType": product_type,
         "certExpiryDate": validity_end,
+        # "keyType": "RSA", Entrust complaining about this parameter
         "tracking": tracking_data
     }
     return data
@@ -86,23 +93,31 @@ def handle_response(my_response):
         404: "Unknown jobId",
         429: "Too many requests"
     }
+
     try:
         d = json.loads(my_response.content)
-    except Exception as e:
+    except ValueError:
         # catch an empty jason object here
-        d = {'errors': 'No detailled message'}
+        d = {'response': 'No detailed message'}
     s = my_response.status_code
     if s > 399:
-        raise Exception("ENTRUST error: {0}\n{1}".format(msg.get(s, s), d['errors']))
+        raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}")
-    current_app.logger.info("Response: {0}, {1} ".format(s, d))
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Response",
+        "status": s,
+        "response": d
+    }
+    current_app.logger.info(log_data)
     return d
 
 
 class EntrustIssuerPlugin(IssuerPlugin):
-    title = "ENTRUST"
+    title = "Entrust"
     slug = "entrust-issuer"
     description = "Enables the creation of certificates by ENTRUST"
-    version = ENTRUST.VERSION
+    version = entrust.VERSION
 
     author = "sirferl"
     author_url = "https://github.com/sirferl/lemur"
@@ -119,7 +134,6 @@ class EntrustIssuerPlugin(IssuerPlugin):
             "ENTRUST_NAME",
             "ENTRUST_EMAIL",
             "ENTRUST_PHONE",
-            "ENTRUST_ISSUING",
         ]
         validate_conf(current_app, required_vars)
 
@@ -142,9 +156,12 @@ class EntrustIssuerPlugin(IssuerPlugin):
         :param issuer_options:
         :return: :raise Exception:
         """
-        current_app.logger.info(
+        log_data = {
-            "Requesting options: {0}".format(issuer_options)
+            "function": f"{__name__}.{sys._getframe().f_code.co_name}",
-        )
+            "message": "Requesting options",
+            "options": issuer_options
+        }
+        current_app.logger.info(log_data)
 
         url = current_app.config.get("ENTRUST_URL") + "/certificates"
 
@@ -156,36 +173,46 @@ class EntrustIssuerPlugin(IssuerPlugin):
         except requests.exceptions.Timeout:
             raise Exception("Timeout for POST")
         except requests.exceptions.RequestException as e:
-            raise Exception("Error for POST {0}".format(e))
+            raise Exception(f"Error for POST {e}")
 
         response_dict = handle_response(response)
         external_id = response_dict['trackingId']
         cert = response_dict['endEntityCert']
+        if len(response_dict['chainCerts']) < 2:
+            # certificate signed by CA directly, no ICA included ini the chain
+            chain = None
+        else:
             chain = response_dict['chainCerts'][1]
-        current_app.logger.info(
+
-            "Received Chain: {0}".format(chain)
+        log_data["message"] = "Received Chain"
-        )
+        log_data["options"] = f"chain: {chain}"
+        current_app.logger.info(log_data)
 
         return cert, chain, external_id
 
     def revoke_certificate(self, certificate, comments):
-        """Revoke a Digicert certificate."""
+        """Revoke an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
 
         # make certificate revoke request
-        revoke_url = "{0}/certificates/{1}/revocations".format(
+        revoke_url = f"{base_url}/certificates/{certificate.external_id}/revocations"
-            base_url, certificate.external_id
+        if not comments or comments == '':
-        )
-        metrics.send("entrust_revoke_certificate", "counter", 1)
-        if comments == '' or not comments:
             comments = "revoked via API"
         data = {
-            "crlReason": "superseded",
+            "crlReason": "superseded",  # enum (keyCompromise, affiliationChanged, superseded, cessationOfOperation)
             "revocationComment": comments
         }
         response = self.session.post(revoke_url, json=data)
+        metrics.send("entrust_revoke_certificate", "counter", 1)
+        return handle_response(response)
 
-        data = handle_response(response)
+    def deactivate_certificate(self, certificate):
+        """Deactivates an Entrust certificate."""
+        base_url = current_app.config.get("ENTRUST_URL")
+        deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations"
+        response = self.session.post(deactivate_url)
+        metrics.send("entrust_deactivate_certificate", "counter", 1)
+        return handle_response(response)
 
     @staticmethod
     def create_authority(options):
@@ -200,7 +227,8 @@ class EntrustIssuerPlugin(IssuerPlugin):
         entrust_root = current_app.config.get("ENTRUST_ROOT")
         entrust_issuing = current_app.config.get("ENTRUST_ISSUING")
         role = {"username": "", "password": "", "name": "entrust"}
-        current_app.logger.info("Creating Auth: {0} {1}".format(options, entrust_issuing))
+        current_app.logger.info(f"Creating Auth: {options} {entrust_issuing}")
+        # body, chain, role
         return entrust_root, "", [role]
 
     def get_ordered_certificate(self, order_id):
@@ -211,10 +239,10 @@ class EntrustIssuerPlugin(IssuerPlugin):
 
 
 class EntrustSourcePlugin(SourcePlugin):
-    title = "ENTRUST"
+    title = "Entrust"
     slug = "entrust-source"
-    description = "Enables the collecion of certificates"
+    description = "Enables the collection of certificates"
-    version = ENTRUST.VERSION
+    version = entrust.VERSION
 
     author = "sirferl"
     author_url = "https://github.com/sirferl/lemur"

lemur/plugins/lemur_entrust/tests/conftest.py

@@ -0,0 +1 @@
+from lemur.tests.conftest import *  # noqa

lemur/plugins/lemur_entrust/tests/test_entrust.py

@@ -0,0 +1,54 @@
+from unittest.mock import patch, Mock
+
+import arrow
+from cryptography import x509
+from lemur.plugins.lemur_entrust import plugin
+
+
+def config_mock(*args):
+    values = {
+        "ENTRUST_API_CERT": "-----BEGIN CERTIFICATE-----abc-----END CERTIFICATE-----",
+        "ENTRUST_API_KEY": False,
+        "ENTRUST_API_USER": "test",
+        "ENTRUST_API_PASS": "password",
+        "ENTRUST_URL": "http",
+        "ENTRUST_ROOT": None,
+        "ENTRUST_NAME": "test",
+        "ENTRUST_EMAIL": "test@lemur.net",
+        "ENTRUST_PHONE": "0123456",
+        "ENTRUST_PRODUCT_ENTRUST": "ADVANTAGE_SSL"
+    }
+    return values[args[0]]
+
+
+@patch("lemur.plugins.lemur_entrust.plugin.current_app")
+def test_process_options(mock_current_app, authority):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    plugin.determine_end_date = Mock(return_value=arrow.get(2020, 10, 7).format('YYYY-MM-DD'))
+
+    authority.name = "Entrust"
+    names = [u"one.example.com", u"two.example.com", u"three.example.com"]
+    options = {
+        "common_name": "example.com",
+        "owner": "bob@example.com",
+        "description": "test certificate",
+        "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
+        "organization": "Example, Inc.",
+        "organizational_unit": "Example Org",
+        "validity_end": arrow.get(2020, 10, 7),
+        "authority": authority,
+    }
+
+    expected = {
+        "signingAlg": "SHA-2",
+        "eku": "SERVER_AND_CLIENT_AUTH",
+        "certType": "ADVANTAGE_SSL",
+        "certExpiryDate": arrow.get(2020, 10, 7).format('YYYY-MM-DD'),
+        "tracking": {
+            "requesterName": mock_current_app.config.get("ENTRUST_NAME"),
+            "requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"),
+            "requesterPhone": mock_current_app.config.get("ENTRUST_PHONE")
+        }
+    }
+
+    assert expected == plugin.process_options(options)

lemur/tests/conf.py

@@ -1,9 +1,18 @@
 # This is just Python which means you can inherit and tweak settings
 
 import os
+import random
+import string
 
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
+
+# generate random secrets for unittest
+def get_random_secret(length):
+    input_ascii = string.ascii_letters + string.digits
+    return ''.join(random.choice(input_ascii) for i in range(length))
+
+
 THREADS_PER_PAGE = 8
 
 # General
@@ -87,7 +96,6 @@ DIGICERT_CIS_API_KEY = "api-key"
 DIGICERT_CIS_ROOTS = {"root": "ROOT"}
 DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"}
 
-
 VERISIGN_URL = "http://example.com"
 VERISIGN_PEM_PATH = "~/"
 VERISIGN_FIRST_NAME = "Jim"
@@ -198,3 +206,41 @@ LDAP_REQUIRED_GROUP = "Lemur Access"
 LDAP_DEFAULT_ROLE = "role1"
 
 ALLOW_CERT_DELETION = True
+
+ENTRUST_API_CERT = "api-cert"
+ENTRUST_API_KEY = get_random_secret(32)
+ENTRUST_API_USER = "user"
+ENTRUST_API_PASS = get_random_secret(32)
+ENTRUST_URL = "https://api.entrust.net/enterprise/v2"
+ENTRUST_ROOT = """
+-----BEGIN CERTIFICATE-----
+MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC
+VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50
+cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs
+IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz
+dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy
+NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu
+dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt
+dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0
+aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj
+YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T
+RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN
+cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW
+wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1
+U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0
+jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN
+BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/
+jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ
+Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v
+1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R
+nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH
+VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g==
+-----END CERTIFICATE-----
+"""
+ENTRUST_NAME = "lemur"
+ENTRUST_EMAIL = "lemur@example.com"
+ENTRUST_PHONE = "123456"
+ENTRUST_ISSUING = ""
+ENTRUST_PRODUCT_ENTRUST = "ADVANTAGE_SSL"