acme v2 support

lemur/plugins/lemur_acme/plugin.py

@@ -11,14 +11,16 @@
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 .. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
 """
+import datetime
 import json
+import time
 
 import OpenSSL.crypto
 import josepy as jose
 from acme import challenges, messages
-from acme.client import Client
+from acme.client import BackwardsCompatibleClientV2, ClientNetwork
 from acme.messages import Error as AcmeError
-from acme.errors import PollError
+from acme.errors import PollError, WildcardUnsupportedError
 from botocore.exceptions import ClientError
 from flask import current_app
 
@@ -31,13 +33,13 @@ from lemur.plugins.bases import IssuerPlugin
 from lemur.plugins.lemur_acme import cloudflare, dyn, route53
 
 
-def find_dns_challenge(authz):
+def find_dns_challenge(authorizations):
-    for combo in authz.body.resolved_combinations:
+    dns_challenges = []
-        if (
+    for authz in authorizations:
-            len(combo) == 1 and
+        for combo in authz.body.challenges:
-            isinstance(combo[0].chall, challenges.DNS01)
+            if isinstance(combo.chall, challenges.DNS01):
-        ):
+                dns_challenges.append(combo)
-            yield combo[0]
+    return dns_challenges
 
 
 class AuthorizationRecord(object):
@@ -48,66 +50,65 @@ class AuthorizationRecord(object):
         self.change_id = change_id
 
 
-def start_dns_challenge(acme_client, account_number, host, dns_provider):
+def maybe_remove_wildcard(host):
+    return host.replace("*.", "")
+
+
+def start_dns_challenge(acme_client, account_number, host, dns_provider, order):
     current_app.logger.debug("Starting DNS challenge for {0}".format(host))
-    authz = acme_client.request_domain_challenges(host)
 
-    [dns_challenge] = find_dns_challenge(authz)
+    dns_challenges = find_dns_challenge(order.authorizations)
+    change_ids = []
 
-    change_id = dns_provider.create_txt_record(
+    for dns_challenge in find_dns_challenge(order.authorizations):
-        dns_challenge.validation_domain_name(host),
+        change_id = dns_provider.create_txt_record(
-        dns_challenge.validation(acme_client.key),
+            dns_challenge.validation_domain_name(maybe_remove_wildcard(host)),
-        account_number
+            dns_challenge.validation(acme_client.client.net.key),
-    )
+            account_number
+        )
+        change_ids.append(change_id)
 
     return AuthorizationRecord(
         host,
-        authz,
+        order.authorizations,
-        dns_challenge,
+        dns_challenges,
-        change_id,
+        change_ids
     )
 
 
 def complete_dns_challenge(acme_client, account_number, authz_record, dns_provider):
-    current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.host))
+    current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.authz[0].body.identifier.value))
-    dns_provider.wait_for_dns_change(authz_record.change_id, account_number=account_number)
+    for change_id in authz_record.change_id:
+        dns_provider.wait_for_dns_change(change_id, account_number=account_number)
 
-    response = authz_record.dns_challenge.response(acme_client.key)
+    for dns_challenge in authz_record.dns_challenge:
 
-    verified = response.simple_verify(
+        response = dns_challenge.response(acme_client.client.net.key)
-        authz_record.dns_challenge.chall,
-        authz_record.host,
-        acme_client.key.public_key()
-    )
 
-    if not verified:
+        verified = response.simple_verify(
-        raise ValueError("Failed verification")
+            dns_challenge.chall,
+            authz_record.host,
+            acme_client.client.net.key.public_key()
+        )
 
-    acme_client.answer_challenge(authz_record.dns_challenge, response)
+        if not verified:
+            raise ValueError("Failed verification")
+
+        time.sleep(5)
+        acme_client.answer_challenge(dns_challenge, response)
 
 
-def request_certificate(acme_client, authorizations, csr):
+def request_certificate(acme_client, authorizations, csr, order):
-    cert_response, _ = acme_client.poll_and_request_issuance(
+    for authorization in authorizations:
-        jose.util.ComparableX509(
+        for authz in authorization.authz:
-            OpenSSL.crypto.load_certificate_request(
+            authorization_resource, _ = acme_client.poll(authz)
-                OpenSSL.crypto.FILETYPE_PEM,
-                csr
-            )
-        ),
-        authzrs=[authz_record.authz for authz_record in authorizations],
-        mintime=60,
-        max_attempts=10,
-    )
 
-    pem_certificate = OpenSSL.crypto.dump_certificate(
+    deadline = datetime.datetime.now() + datetime.timedelta(seconds=90)
-        OpenSSL.crypto.FILETYPE_PEM, cert_response.body
+    orderr = acme_client.finalize_order(order, deadline)
-    ).decode('utf-8')
+    pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
-
+                                           OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
-    full_chain = []
+                                                                           orderr.fullchain_pem)).decode()
-    for cert in acme_client.fetch_chain(cert_response):
+    pem_certificate_chain = orderr.fullchain_pem[len(pem_certificate):].lstrip()
-        chain = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
-        full_chain.append(chain.decode("utf-8"))
-    pem_certificate_chain = "\n".join(full_chain)
 
     current_app.logger.debug("{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)))
     return pem_certificate, pem_certificate_chain
@@ -127,15 +128,12 @@ def setup_acme_client(authority):
     key = jose.JWKRSA(key=generate_private_key('RSA2048'))
 
     current_app.logger.debug("Connecting with directory at {0}".format(directory_url))
-    client = Client(directory_url, key)
-
-    registration = client.register(
-        messages.NewRegistration.from_data(email=email)
-    )
 
+    net = ClientNetwork(key, account=None)
+    client = BackwardsCompatibleClientV2(net, key, directory_url)
+    registration = client.new_account_and_tos(messages.NewRegistration.from_data(email=email))
     current_app.logger.debug("Connected: {0}".format(registration.uri))
 
-    client.agree_to_tos(registration)
     return client, registration
 
 
@@ -156,26 +154,25 @@ def get_domains(options):
     return domains
 
 
-def get_authorizations(acme_client, account_number, domains, dns_provider):
+def get_authorizations(acme_client, order, order_info, dns_provider):
     authorizations = []
-    for domain in domains:
+    for domain in order.body.identifiers:
-        authz_record = start_dns_challenge(acme_client, account_number, domain, dns_provider)
+        authz_record = start_dns_challenge(acme_client, order_info.account_number, domain.value, dns_provider, order)
         authorizations.append(authz_record)
     return authorizations
 
 
 def finalize_authorizations(acme_client, account_number, dns_provider, authorizations):
-    try:
+    for authz_record in authorizations:
-        for authz_record in authorizations:
+        complete_dns_challenge(acme_client, account_number, authz_record, dns_provider)
-            complete_dns_challenge(acme_client, account_number, authz_record, dns_provider)
+    for authz_record in authorizations:
-    finally:
+        dns_challenges = authz_record.dns_challenge
-        for authz_record in authorizations:
+        for dns_challenge in dns_challenges:
-            dns_challenge = authz_record.dns_challenge
             dns_provider.delete_txt_record(
                 authz_record.change_id,
                 account_number,
-                dns_challenge.validation_domain_name(authz_record.host),
+                dns_challenge.validation_domain_name(maybe_remove_wildcard(authz_record.host)),
-                dns_challenge.validation(acme_client.key)
+                dns_challenge.validation(acme_client.client.net.key)
             )
 
     return authorizations
@@ -265,15 +262,21 @@ class ACMEIssuerPlugin(IssuerPlugin):
                 order_info = authorization_service.get(pending_cert.external_id)
                 dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
                 dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
+                try:
+                    order = acme_client.new_order(pending_cert.csr)
+                except WildcardUnsupportedError:
+                    raise Exception("The currently selected ACME CA endpoint does"
+                                    " not support issuing wildcard certificates.")
+
+                authorizations = get_authorizations(acme_client, order, order_info, dns_provider_type)
 
-                authorizations = get_authorizations(
-                    acme_client, order_info.account_number, order_info.domains, dns_provider_type)
                 pending.append({
                     "acme_client": acme_client,
                     "account_number": order_info.account_number,
                     "dns_provider_type": dns_provider_type,
                     "authorizations": authorizations,
                     "pending_cert": pending_cert,
+                    "order": order,
                 })
             except (ClientError, ValueError, Exception):
                 current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
@@ -288,12 +291,13 @@ class ACMEIssuerPlugin(IssuerPlugin):
                     entry["acme_client"],
                     entry["account_number"],
                     entry["dns_provider_type"],
-                    entry["authorizations"]
+                    entry["authorizations"],
                 )
                 pem_certificate, pem_certificate_chain = request_certificate(
                     entry["acme_client"],
                     entry["authorizations"],
-                    entry["pending_cert"].csr
+                    entry["pending_cert"].csr,
+                    entry["order"]
                 )
 
                 cert = {

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -35,6 +35,7 @@ class TestAcme(unittest.TestCase):
     @patch('lemur.plugins.lemur_acme.plugin.find_dns_challenge')
     def test_start_dns_challenge(self, mock_find_dns_challenge, mock_len, mock_app, mock_acme):
         assert mock_len
+        mock_order = Mock()
         mock_app.logger.debug = Mock()
         mock_authz = Mock()
         mock_authz.body.resolved_combinations = []
@@ -51,7 +52,7 @@ class TestAcme(unittest.TestCase):
         iterable = mock_find_dns_challenge.return_value
         iterator = iter(values)
         iterable.__iter__.return_value = iterator
-        result = plugin.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider)
+        result = plugin.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order)
         self.assertEqual(type(result), plugin.AuthorizationRecord)
 
     @patch('acme.client.Client')
@@ -63,7 +64,15 @@ class TestAcme(unittest.TestCase):
         mock_authz = Mock()
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True)
-
+        mock_authz.authz = []
+        mock_authz_record = Mock()
+        mock_authz_record.body.identifier.value = "test"
+        mock_authz.authz.append(mock_authz_record)
+        mock_authz.change_id = []
+        mock_authz.change_id.append("123")
+        mock_authz.dns_challenge = []
+        dns_challenge = Mock()
+        mock_authz.dns_challenge.append(dns_challenge)
         plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
 
     @patch('acme.client.Client')
@@ -75,6 +84,15 @@ class TestAcme(unittest.TestCase):
         mock_authz = Mock()
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False)
+        mock_authz.authz = []
+        mock_authz_record = Mock()
+        mock_authz_record.body.identifier.value = "test"
+        mock_authz.authz.append(mock_authz_record)
+        mock_authz.change_id = []
+        mock_authz.change_id.append("123")
+        mock_authz.dns_challenge = []
+        dns_challenge = Mock()
+        mock_authz.dns_challenge.append(dns_challenge)
         self.assertRaises(
             ValueError,
             plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
@@ -96,8 +114,8 @@ class TestAcme(unittest.TestCase):
         mock_authz.append(mock_authz_record)
         mock_acme.fetch_chain = Mock(return_value="mock_chain")
         mock_crypto.dump_certificate = Mock(return_value=b'chain')
-
+        mock_order = Mock()
-        plugin.request_certificate(mock_acme, [], "mock_csr")
+        plugin.request_certificate(mock_acme, [], "mock_csr", mock_order)
 
     def test_setup_acme_client_fail(self):
         mock_authority = Mock()
@@ -105,7 +123,7 @@ class TestAcme(unittest.TestCase):
         with self.assertRaises(Exception):
             plugin.setup_acme_client(mock_authority)
 
-    @patch('lemur.plugins.lemur_acme.plugin.Client')
+    @patch('lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
     def test_setup_acme_client_success(self, mock_current_app, mock_acme):
         mock_authority = Mock()
@@ -146,7 +164,13 @@ class TestAcme(unittest.TestCase):
 
     @patch('lemur.plugins.lemur_acme.plugin.start_dns_challenge', return_value="test")
     def test_get_authorizations(self, mock_start_dns_challenge):
-        result = plugin.get_authorizations("acme_client", "account_number", ["domains"], "dns_provider")
+        mock_order = Mock()
+        mock_order.body.identifiers = []
+        mock_domain = Mock()
+        mock_order.body.identifiers.append(mock_domain)
+        mock_order_info = Mock()
+        mock_order_info.account_number = 1
+        result = plugin.get_authorizations("acme_client", mock_order, mock_order_info, "dns_provider")
         self.assertEqual(result, ["test"])
 
     @patch('lemur.plugins.lemur_acme.plugin.complete_dns_challenge', return_value="test")

requirements-dev.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
 #
-aspy.yaml==1.1.0          # via pre-commit
+aspy.yaml==1.1.1          # via pre-commit
 cached-property==1.4.2    # via pre-commit
 certifi==2018.4.16        # via requests
 cfgv==1.0.0               # via pre-commit
@@ -12,7 +12,7 @@ chardet==3.0.4            # via requests
 flake8==3.5.0
 identify==1.0.16          # via pre-commit
 idna==2.6                 # via requests
-invoke==0.23.0
+invoke==1.0.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.0
 pkginfo==1.4.2            # via twine

requirements-docs.txt

@@ -15,8 +15,8 @@ asyncpool==1.0
 babel==2.5.3              # via sphinx
 bcrypt==3.1.4
 blinker==1.4
-boto3==1.7.14
+boto3==1.7.19
-botocore==1.10.14
+botocore==1.10.19
 certifi==2018.4.16
 cffi==1.11.5
 click==6.7
@@ -49,7 +49,7 @@ lockfile==0.12.2
 mako==1.0.7
 markupsafe==1.0
 marshmallow-sqlalchemy==0.13.2
-marshmallow==2.15.1
+marshmallow==2.15.2
 mock==2.0.0
 ndg-httpsclient==0.5.0
 packaging==17.1           # via sphinx
@@ -66,11 +66,11 @@ pynacl==1.2.1
 pyopenssl==17.2.0
 pyparsing==2.2.0          # via packaging
 pyrfc3339==1.0
-python-dateutil==2.7.2
+python-dateutil==2.7.3
 python-editor==1.0.3
 pytz==2018.4
 pyyaml==3.12
-raven[flask]==6.7.0
+raven[flask]==6.8.0
 requests[security]==2.11.1
 retrying==1.3.3
 s3transfer==0.1.13

requirements-tests.txt

@@ -7,9 +7,9 @@
 asn1crypto==0.24.0        # via cryptography
 attrs==18.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.7.16             # via moto
+boto3==1.7.21             # via moto
 boto==2.48.0              # via moto
-botocore==1.10.16         # via boto3, moto, s3transfer
+botocore==1.10.21         # via boto3, moto, s3transfer
 certifi==2018.4.16        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -21,7 +21,7 @@ docker-pycreds==0.2.3     # via docker
 docker==3.3.0             # via moto
 docutils==0.14            # via botocore
 factory-boy==2.11.1
-faker==0.8.13
+faker==0.8.15
 flask==1.0.2              # via pytest-flask
 freezegun==0.3.10
 idna==2.6                 # via cryptography, requests
@@ -35,7 +35,7 @@ mock==2.0.0               # via moto
 more-itertools==4.1.0     # via pytest
 moto==1.3.3
 nose==1.3.7
-pbr==4.0.2                # via mock
+pbr==4.0.3                # via mock
 pluggy==0.6.0             # via pytest
 py==1.5.3                 # via pytest
 pyaml==17.12.1            # via moto
@@ -47,7 +47,7 @@ pytest==3.5.1
 python-dateutil==2.6.1    # via botocore, faker, freezegun, moto
 pytz==2018.4              # via moto
 pyyaml==3.12              # via pyaml
-requests-mock==1.4.0
+requests-mock==1.5.0
 requests==2.18.4          # via aws-xray-sdk, docker, moto, requests-mock, responses
 responses==0.9.0          # via moto
 s3transfer==0.1.13        # via boto3

requirements.txt

@@ -13,8 +13,8 @@ asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.7.16
+boto3==1.7.21
-botocore==1.10.16         # via boto3, s3transfer
+botocore==1.10.21         # via boto3, s3transfer
 certifi==2018.4.16
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 click==6.7                # via flask
@@ -46,11 +46,11 @@ lockfile==0.12.2
 mako==1.0.7               # via alembic
 markupsafe==1.0           # via jinja2, mako
 marshmallow-sqlalchemy==0.13.2
-marshmallow==2.15.1
+marshmallow==2.15.2
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.0
 paramiko==2.4.1
-pbr==4.0.2                # via mock
+pbr==4.0.3                # via mock
 pem==17.1.0
 psycopg2==2.7.4
 pyasn1-modules==0.2.1     # via python-ldap
@@ -60,12 +60,12 @@ pyjwt==1.6.1
 pynacl==1.2.1             # via paramiko
 pyopenssl==17.2.0
 pyrfc3339==1.0            # via acme
-python-dateutil==2.7.2    # via alembic, arrow, botocore
+python-dateutil==2.7.3    # via alembic, arrow, botocore
 python-editor==1.0.3      # via alembic
 python-ldap==3.0.0
 pytz==2018.4              # via acme, flask-restful, pyrfc3339
 pyyaml==3.12              # via cloudflare
-raven[flask]==6.7.0
+raven[flask]==6.8.0
 requests[security]==2.11.1
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3