Merge branch 'master' into powerdns_doc_update_01

lemur/plugins/lemur_acme/plugin.py

@@ -172,7 +172,7 @@ class AcmeHandler(object):
 
         except (AcmeError, TimeoutError):
             sentry.captureException(extra={"order_url": str(order.uri)})
-            metrics.send("request_certificate_error", "counter", 1)
+            metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri})
             current_app.logger.error(
                 f"Unable to resolve Acme order: {order.uri}", exc_info=True
             )
@@ -183,6 +183,11 @@ class AcmeHandler(object):
             else:
                 raise
 
+        metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri})
+        current_app.logger.info(
+            f"Successfully resolved Acme order: {order.uri}", exc_info=True
+        )
+
         pem_certificate = OpenSSL.crypto.dump_certificate(
             OpenSSL.crypto.FILETYPE_PEM,
             OpenSSL.crypto.load_certificate(
@@ -254,8 +259,9 @@ class AcmeHandler(object):
 
         domains = [options["common_name"]]
         if options.get("extensions"):
-            for name in options["extensions"]["sub_alt_names"]["names"]:
+            for dns_name in options["extensions"]["sub_alt_names"]["names"]:
-                domains.append(name)
+                if dns_name.value not in domains:
+                    domains.append(dns_name.value)
 
         current_app.logger.debug("Got these domains: {0}".format(domains))
         return domains
@@ -640,15 +646,8 @@ class ACMEIssuerPlugin(IssuerPlugin):
         domains = self.acme.get_domains(issuer_options)
         if not create_immediately:
             # Create pending authorizations that we'll need to do the creation
-            authz_domains = []
-            for d in domains:
-                if type(d) == str:
-                    authz_domains.append(d)
-                else:
-                    authz_domains.append(d.value)
-
             dns_authorization = authorization_service.create(
-                account_number, authz_domains, provider_type
+                account_number, domains, provider_type
             )
             # Return id of the DNS Authorization
             return None, None, dns_authorization.id

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -1,4 +1,6 @@
 import unittest
+
+from cryptography.x509 import DNSName
 from requests.models import Response
 
 from mock import MagicMock, Mock, patch
@@ -74,12 +76,14 @@ class TestAcme(unittest.TestCase):
     @patch("acme.client.Client")
     @patch("lemur.plugins.lemur_acme.plugin.current_app")
     @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change")
+    @patch("time.sleep")
     def test_complete_dns_challenge_success(
-        self, mock_wait_for_dns_change, mock_current_app, mock_acme
+        self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme
     ):
         mock_dns_provider = Mock()
         mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
         mock_authz = Mock()
+        mock_sleep.return_value = False
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True)
         mock_authz.authz = []
@@ -179,7 +183,7 @@ class TestAcme(unittest.TestCase):
         options = {
             "common_name": "test.netflix.net",
             "extensions": {
-                "sub_alt_names": {"names": ["test2.netflix.net", "test3.netflix.net"]}
+                "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]}
             },
         }
         result = self.acme.get_domains(options)
@@ -187,6 +191,19 @@ class TestAcme(unittest.TestCase):
             result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"]
         )
 
+    @patch("lemur.plugins.lemur_acme.plugin.current_app")
+    def test_get_domains_san(self, mock_current_app):
+        options = {
+            "common_name": "test.netflix.net",
+            "extensions": {
+                "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]}
+            },
+        }
+        result = self.acme.get_domains(options)
+        self.assertEqual(
+            result, [options["common_name"], "test2.netflix.net"]
+        )
+
     @patch(
         "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge",
         return_value="test",

lemur/plugins/lemur_digicert/plugin.py

@@ -14,21 +14,17 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import json
+
 import arrow
-import requests
-
 import pem
-from retrying import retry
+import requests
-
-from flask import current_app
-
 from cryptography import x509
-
+from flask import current_app
-from lemur.extensions import metrics
 from lemur.common.utils import validate_conf
-from lemur.plugins.bases import IssuerPlugin, SourcePlugin
+from lemur.extensions import metrics
-
 from lemur.plugins import lemur_digicert as digicert
+from lemur.plugins.bases import IssuerPlugin, SourcePlugin
+from retrying import retry
 
 
 def log_status_code(r, *args, **kwargs):
@@ -64,24 +60,37 @@ def signature_hash(signing_algorithm):
     raise Exception("Unsupported signing algorithm.")
 
 
-def determine_validity_years(end_date):
+def determine_validity_years(years):
     """Given an end date determine how many years into the future that date is.
+    :param years:
+    :return: validity in years
+    """
+    default_years = current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1)
+    max_years = current_app.config.get("DIGICERT_MAX_VALIDITY", default_years)
+
+    if years > max_years:
+        return max_years
+    if years not in [1, 2, 3]:
+        return default_years
+    return years
+
+
+def determine_end_date(end_date):
+    """
+    Determine appropriate end date
 
     :param end_date:
-    :return: str validity in years
+    :return: validity_end
     """
-    now = arrow.utcnow()
+    default_years = current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1)
+    max_validity_end = arrow.utcnow().shift(years=current_app.config.get("DIGICERT_MAX_VALIDITY", default_years))
 
-    if end_date < now.shift(years=+1):
+    if not end_date:
-        return 1
+        end_date = arrow.utcnow().shift(years=default_years)
-    elif end_date < now.shift(years=+2):
-        return 2
-    elif end_date < now.shift(years=+3):
-        return 3
 
-    raise Exception(
+    if end_date > max_validity_end:
-        "DigiCert issued certificates cannot exceed three" " years in validity"
+        end_date = max_validity_end
-    )
+    return end_date
 
 
 def get_additional_names(options):
@@ -107,12 +116,6 @@ def map_fields(options, csr):
     :param csr:
     :return: dict or valid DigiCert options
     """
-    if not options.get("validity_years"):
-        if not options.get("validity_end"):
-            options["validity_years"] = current_app.config.get(
-                "DIGICERT_DEFAULT_VALIDITY", 1
-            )
-
     data = dict(
         certificate={
             "common_name": options["common_name"],
@@ -125,9 +128,11 @@ def map_fields(options, csr):
     data["certificate"]["dns_names"] = get_additional_names(options)
 
     if options.get("validity_years"):
-        data["validity_years"] = options["validity_years"]
+        data["validity_years"] = determine_validity_years(options.get("validity_years"))
+    elif options.get("validity_end"):
+        data["custom_expiration_date"] = determine_end_date(options.get("validity_end")).format("YYYY-MM-DD")
     else:
-        data["custom_expiration_date"] = options["validity_end"].format("YYYY-MM-DD")
+        data["validity_years"] = determine_validity_years(0)
 
     if current_app.config.get("DIGICERT_PRIVATE", False):
         if "product" in data:
@@ -144,18 +149,15 @@ def map_cis_fields(options, csr):
 
     :param options:
     :param csr:
-    :return:
+    :return: data
     """
-    if not options.get("validity_years"):
+
-        if not options.get("validity_end"):
+    if options.get("validity_years"):
-            options["validity_end"] = arrow.utcnow().shift(
+        validity_end = determine_end_date(arrow.utcnow().shift(years=options["validity_years"]))
-                years=current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1)
+    elif options.get("validity_end"):
-            )
+        validity_end = determine_end_date(options.get("validity_end"))
-        options["validity_years"] = determine_validity_years(options["validity_end"])
     else:
-        options["validity_end"] = arrow.utcnow().shift(
+        validity_end = determine_end_date(False)
-            years=options["validity_years"]
-        )
 
     data = {
         "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES", {}).get(options['authority'].name),
@@ -164,7 +166,7 @@ def map_cis_fields(options, csr):
         "csr": csr,
         "signature_hash": signature_hash(options.get("signing_algorithm")),
         "validity": {
-            "valid_to": options["validity_end"].format("YYYY-MM-DDTHH:MM") + "Z"
+            "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z"
         },
         "organization": {
             "name": options["organization"],
@@ -173,7 +175,8 @@ def map_cis_fields(options, csr):
     }
     #  possibility to default to a SIGNING_ALGORITHM for a given profile
     if current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name):
-        data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name)
+        data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(
+            options['authority'].name)
 
     return data
 

lemur/plugins/lemur_digicert/tests/test_digicert.py

@@ -1,20 +1,81 @@
-import pytest
-import arrow
 import json
-from unittest.mock import patch
-
-from freezegun import freeze_time
-
-from lemur.tests.vectors import CSR_STR
 
+import arrow
+import pytest
 from cryptography import x509
+from freezegun import freeze_time
+from lemur.plugins.lemur_digicert import plugin
+from lemur.tests.vectors import CSR_STR
+from mock import Mock, patch
 
 
-def test_map_fields_with_validity_end_and_start(app):
+def config_mock(*args):
-    from lemur.plugins.lemur_digicert.plugin import map_fields
+    values = {
+        "DIGICERT_ORG_ID": 111111,
+        "DIGICERT_PRIVATE": False,
+        "DIGICERT_DEFAULT_SIGNING_ALGORITHM": "sha256",
+        "DIGICERT_DEFAULT_VALIDITY": 1,
+        "DIGICERT_MAX_VALIDITY": 2,
+        "DIGICERT_CIS_PROFILE_NAMES": {"digicert": 'digicert'},
+        "DIGICERT_CIS_SIGNING_ALGORITHMS": {"digicert": 'digicert'},
+    }
+    return values[args[0]]
+
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_determine_validity_years(mock_current_app):
+    mock_current_app.config.get = Mock(return_value=2)
+    assert plugin.determine_validity_years(1) == 1
+    assert plugin.determine_validity_years(0) == 2
+    assert plugin.determine_validity_years(3) == 2
+
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_determine_end_date(mock_current_app):
+    mock_current_app.config.get = Mock(return_value=2)
+    with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):
+        assert arrow.get(2018, 11, 3) == plugin.determine_end_date(0)
+        assert arrow.get(2018, 5, 7) == plugin.determine_end_date(arrow.get(2018, 5, 7))
+        assert arrow.get(2018, 11, 3) == plugin.determine_end_date(arrow.get(2020, 5, 7))
+
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_map_fields_with_validity_years(mock_current_app):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
 
         names = [u"one.example.com", u"two.example.com", u"three.example.com"]
+        options = {
+            "common_name": "example.com",
+            "owner": "bob@example.com",
+            "description": "test certificate",
+            "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
+            "validity_years": 2
+        }
+        expected = {
+            "certificate": {
+                "csr": CSR_STR,
+                "common_name": "example.com",
+                "dns_names": names,
+                "signature_hash": "sha256",
+            },
+            "organization": {"id": 111111},
+            "validity_years": 2,
+        }
+        assert expected == plugin.map_fields(options, CSR_STR)
 
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_map_fields_with_validity_end_and_start(mock_current_app):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7))
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
+
+        names = [u"one.example.com", u"two.example.com", u"three.example.com"]
         options = {
             "common_name": "example.com",
             "owner": "bob@example.com",
@@ -24,9 +85,7 @@ def test_map_fields_with_validity_end_and_start(app):
             "validity_start": arrow.get(2016, 10, 30),
         }
 
-    data = map_fields(options, CSR_STR)
+        expected = {
-
-    assert data == {
             "certificate": {
                 "csr": CSR_STR,
                 "common_name": "example.com",
@@ -37,66 +96,18 @@ def test_map_fields_with_validity_end_and_start(app):
             "custom_expiration_date": arrow.get(2017, 5, 7).format("YYYY-MM-DD"),
         }
 
+        assert expected == plugin.map_fields(options, CSR_STR)
 
-def test_map_fields_with_validity_years(app):
+
-    from lemur.plugins.lemur_digicert.plugin import map_fields
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_map_cis_fields_with_validity_years(mock_current_app, authority):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    plugin.determine_end_date = Mock(return_value=arrow.get(2018, 11, 3))
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
 
         names = [u"one.example.com", u"two.example.com", u"three.example.com"]
-
-    options = {
-        "common_name": "example.com",
-        "owner": "bob@example.com",
-        "description": "test certificate",
-        "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
-        "validity_years": 2,
-        "validity_end": arrow.get(2017, 10, 30),
-    }
-
-    data = map_fields(options, CSR_STR)
-
-    assert data == {
-        "certificate": {
-            "csr": CSR_STR,
-            "common_name": "example.com",
-            "dns_names": names,
-            "signature_hash": "sha256",
-        },
-        "organization": {"id": 111111},
-        "validity_years": 2,
-    }
-
-
-def test_map_cis_fields(app, authority):
-    from lemur.plugins.lemur_digicert.plugin import map_cis_fields
-
-    names = [u"one.example.com", u"two.example.com", u"three.example.com"]
-
-    options = {
-        "common_name": "example.com",
-        "owner": "bob@example.com",
-        "description": "test certificate",
-        "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
-        "organization": "Example, Inc.",
-        "organizational_unit": "Example Org",
-        "validity_end": arrow.get(2017, 5, 7),
-        "validity_start": arrow.get(2016, 10, 30),
-        "authority": authority,
-    }
-
-    data = map_cis_fields(options, CSR_STR)
-
-    assert data == {
-        "common_name": "example.com",
-        "csr": CSR_STR,
-        "additional_dns_names": names,
-        "signature_hash": "sha256",
-        "organization": {"name": "Example, Inc.", "units": ["Example Org"]},
-        "validity": {
-            "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z"
-        },
-        "profile_name": None,
-    }
-
         options = {
             "common_name": "example.com",
             "owner": "bob@example.com",
@@ -108,10 +119,7 @@ def test_map_cis_fields(app, authority):
             "authority": authority,
         }
 
-    with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):
+        expected = {
-        data = map_cis_fields(options, CSR_STR)
-
-        assert data == {
             "common_name": "example.com",
             "csr": CSR_STR,
             "additional_dns_names": names,
@@ -123,17 +131,55 @@ def test_map_cis_fields(app, authority):
             "profile_name": None,
         }
 
+        assert expected == plugin.map_cis_fields(options, CSR_STR)
 
-def test_signature_hash(app):
-    from lemur.plugins.lemur_digicert.plugin import signature_hash
 
-    assert signature_hash(None) == "sha256"
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
-    assert signature_hash("sha256WithRSA") == "sha256"
+def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, authority):
-    assert signature_hash("sha384WithRSA") == "sha384"
+    mock_current_app.config.get = Mock(side_effect=config_mock)
-    assert signature_hash("sha512WithRSA") == "sha512"
+    plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7))
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
+
+        names = [u"one.example.com", u"two.example.com", u"three.example.com"]
+        options = {
+            "common_name": "example.com",
+            "owner": "bob@example.com",
+            "description": "test certificate",
+            "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
+            "organization": "Example, Inc.",
+            "organizational_unit": "Example Org",
+            "validity_end": arrow.get(2017, 5, 7),
+            "validity_start": arrow.get(2016, 10, 30),
+            "authority": authority
+        }
+
+        expected = {
+            "common_name": "example.com",
+            "csr": CSR_STR,
+            "additional_dns_names": names,
+            "signature_hash": "sha256",
+            "organization": {"name": "Example, Inc.", "units": ["Example Org"]},
+            "validity": {
+                "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z"
+            },
+            "profile_name": None,
+        }
+
+        assert expected == plugin.map_cis_fields(options, CSR_STR)
+
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_signature_hash(mock_current_app, app):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    assert plugin.signature_hash(None) == "sha256"
+    assert plugin.signature_hash("sha256WithRSA") == "sha256"
+    assert plugin.signature_hash("sha384WithRSA") == "sha384"
+    assert plugin.signature_hash("sha512WithRSA") == "sha512"
 
     with pytest.raises(Exception):
-        signature_hash("sdfdsf")
+        plugin.signature_hash("sdfdsf")
 
 
 def test_issuer_plugin_create_certificate(

lemur/plugins/lemur_kubernetes/plugin.py

@@ -96,7 +96,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain):
     if secret_format == "TLS":
         secret["type"] = "kubernetes.io/tls"
         secret["data"] = {
-            "tls.crt": base64encode(cert_chain),
+            "tls.crt": base64encode(body),
             "tls.key": base64encode(private_key),
         }
     if secret_format == "Certificate":

lemur/plugins/lemur_verisign/plugin.py

@@ -98,10 +98,14 @@ def process_options(options):
     :param options:
     :return: dict or valid verisign options
     """
+    # if there is a config variable with VERISIGN_PRODUCT_<upper(authority.name)> take the value as Cert product-type
+    # else default to "Server", to be compatoible with former versions
+    authority = options.get("authority").name.upper()
+    product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority), "Server")
     data = {
         "challenge": get_psuedo_random_string(),
         "serverType": "Apache",
-        "certProductType": "Server",
+        "certProductType": product_type,
         "firstName": current_app.config.get("VERISIGN_FIRST_NAME"),
         "lastName": current_app.config.get("VERISIGN_LAST_NAME"),
         "signatureAlgorithm": "sha256WithRSAEncryption",
@@ -111,11 +115,6 @@ def process_options(options):
 
     data["subject_alt_names"] = ",".join(get_additional_names(options))
 
-    if options.get("validity_end") > arrow.utcnow().shift(years=2):
-        raise Exception(
-            "Verisign issued certificates cannot exceed two years in validity"
-        )
-
     if options.get("validity_end"):
         # VeriSign (Symantec) only accepts strictly smaller than 2 year end date
         if options.get("validity_end") < arrow.utcnow().shift(years=2, days=-1):
@@ -210,7 +209,7 @@ class VerisignIssuerPlugin(IssuerPlugin):
 
         response = self.session.post(url, data=data)
         try:
-            cert = handle_response(response.content)["Response"]["Certificate"]
+            response_dict = handle_response(response.content)
         except KeyError:
             metrics.send(
                 "verisign_create_certificate_error",
@@ -222,8 +221,13 @@ class VerisignIssuerPlugin(IssuerPlugin):
                 extra={"common_name": issuer_options.get("common_name", "")}
             )
             raise Exception(f"Error with Verisign: {response.content}")
-        # TODO add external id
+        authority = issuer_options.get("authority").name.upper()
-        return cert, current_app.config.get("VERISIGN_INTERMEDIATE"), None
+        cert = response_dict['Response']['Certificate']
+        external_id = None
+        if 'Transaction_ID' in response_dict['Response'].keys():
+            external_id = response_dict['Response']['Transaction_ID']
+        chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE"))
+        return cert, chain, external_id
 
     @staticmethod
     def create_authority(options):

requirements-dev.txt

@@ -5,37 +5,39 @@
 #    pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in
 #
 aspy.yaml==1.3.0          # via pre-commit
-bleach==3.1.0             # via readme-renderer
+bleach==3.1.1             # via readme-renderer
 certifi==2019.11.28       # via requests
+cffi==1.14.0              # via cryptography
 cfgv==2.0.1               # via pre-commit
 chardet==3.0.4            # via requests
+cryptography==2.8         # via secretstorage
 docutils==0.15.2          # via readme-renderer
 flake8==3.5.0
 identify==1.4.9           # via pre-commit
 idna==2.8                 # via requests
-importlib-metadata==1.3.0  # via keyring, pre-commit, twine
 invoke==1.3.0
+jeepney==0.4.2            # via secretstorage
 keyring==21.0.0           # via twine
 mccabe==0.6.1             # via flake8
-more-itertools==8.0.2     # via zipp
 nodeenv==1.3.3
 pkginfo==1.5.0.1          # via twine
 pre-commit==1.21.0
 pycodestyle==2.3.1        # via flake8
+pycparser==2.19           # via cffi
 pyflakes==1.6.0           # via flake8
 pygments==2.5.2           # via readme-renderer
 pyyaml==5.2
 readme-renderer==24.0     # via twine
 requests-toolbelt==0.9.1  # via twine
 requests==2.22.0          # via requests-toolbelt, twine
-six==1.13.0               # via bleach, cfgv, pre-commit, readme-renderer
+secretstorage==3.1.2      # via keyring
+six==1.13.0               # via bleach, cfgv, cryptography, pre-commit, readme-renderer
 toml==0.10.0              # via pre-commit
 tqdm==4.41.1              # via twine
 twine==3.1.1
 urllib3==1.25.7           # via requests
 virtualenv==16.7.9        # via pre-commit
 webencodings==0.5.1       # via bleach
-zipp==0.6.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools