From d00dd9d2956b7a4f28339712344810092600ec65 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 14:29:23 +0200 Subject: [PATCH 01/46] Initial structure for ACME http challenge --- lemur/plugins/lemur_acme/plugin.py | 94 +++++++++++++++++++++++++++++- setup.py | 1 + 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 8bc1485f..d1c86017 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -26,6 +26,7 @@ from flask import current_app from lemur.authorizations import service as authorization_service from lemur.common.utils import generate_private_key +from lemur.destinations import service as destination_service from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider from lemur.extensions import metrics, sentry @@ -436,7 +437,7 @@ class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" slug = "acme-issuer" description = ( - "Enables the creation of certificates via ACME CAs (including Let's Encrypt)" + "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the DNS-01 challenge" ) version = acme.VERSION @@ -746,3 +747,94 @@ class ACMEIssuerPlugin(IssuerPlugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): # Needed to override issuer function. pass + + +class ACMEHttpIssuerPlugin(IssuerPlugin): + title = "Acme HTTP-01" + slug = "acme-http-issuer" + description = ( + "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the HTTP-01 challenge" + ) + version = acme.VERSION + + author = "Netflix" + author_url = "https://github.com/netflix/lemur.git" + + destination_list = [] + + options = [ + { + "name": "acme_url", + "type": "str", + "required": True, + "validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", + "helpMessage": "Must be a valid web url starting with http[s]://", + }, + { + "name": "telephone", + "type": "str", + "default": "", + "helpMessage": "Telephone to use", + }, + { + "name": "email", + "type": "str", + "default": "", + "validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", + "helpMessage": "Email to use", + }, + { + "name": "certificate", + "type": "textarea", + "default": "", + "validation": "/^-----BEGIN CERTIFICATE-----/", + "helpMessage": "Certificate to use", + }, + { + "name": "tokenDestination", + "type": "select", + "required": True, + "available": destination_list, + "helpMessage": "The destination to use to deploy the token.", + }, + ] + + def __init__(self, *args, **kwargs): + super(ACMEHttpIssuerPlugin, self).__init__(*args, **kwargs) + + if len(self.destination_list) == 0: + destinations = destination_service.get_all() + for destination in destinations: + # we only want to use sftp destinations here + if destination.plugin_name == "sftp-destination": + self.destination_list.append(destination.label) + + def create_certificate(self, csr, issuer_options): + pass + + @staticmethod + def create_authority(options): + """ + Creates an authority, this authority is then used by Lemur to allow a user + to specify which Certificate Authority they want to sign their certificate. + + :param options: + :return: + """ + role = {"username": "", "password": "", "name": "acme"} + plugin_options = options.get("plugin", {}).get("plugin_options") + if not plugin_options: + error = "Invalid options for lemur_acme plugin: {}".format(options) + current_app.logger.error(error) + raise InvalidConfiguration(error) + # Define static acme_root based off configuration variable by default. However, if user has passed a + # certificate, use this certificate as the root. + acme_root = current_app.config.get("ACME_ROOT") + for option in plugin_options: + if option.get("name") == "certificate": + acme_root = option.get("value") + return acme_root, "", [role] + + def cancel_ordered_certificate(self, pending_cert, **kwargs): + # Needed to override issuer function. + pass \ No newline at end of file diff --git a/setup.py b/setup.py index 4da14c3d..7fcbc101 100644 --- a/setup.py +++ b/setup.py @@ -132,6 +132,7 @@ setup( 'lemur.plugins': [ 'verisign_issuer = lemur.plugins.lemur_verisign.plugin:VerisignIssuerPlugin', 'acme_issuer = lemur.plugins.lemur_acme.plugin:ACMEIssuerPlugin', + 'acme_http_issuer = lemur.plugins.lemur_acme.plugin:ACMEHttpIssuerPlugin', 'aws_destination = lemur.plugins.lemur_aws.plugin:AWSDestinationPlugin', 'aws_source = lemur.plugins.lemur_aws.plugin:AWSSourcePlugin', 'aws_s3 = lemur.plugins.lemur_aws.plugin:S3DestinationPlugin', From 348d8477dd3303fe400d0d2eca46cc845a5f1818 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 14:57:32 +0200 Subject: [PATCH 02/46] Refactor destination plugin, to allow upload of ACME http-challenge tokens --- lemur/plugins/lemur_sftp/plugin.py | 57 +++++++++++++++++++----------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 66784048..7c4e93c4 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -16,6 +16,7 @@ .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov """ +from os import path import paramiko @@ -95,18 +96,14 @@ class SFTPDestinationPlugin(DestinationPlugin): }, ] + # this is called when using this as a default destination plugin def upload(self, name, body, private_key, cert_chain, options, **kwargs): current_app.logger.debug("SFTP destination plugin is started") cn = common_name(parse_certificate(body)) - host = self.get_option("host", options) - port = self.get_option("port", options) - user = self.get_option("user", options) - password = self.get_option("password", options) - ssh_priv_key = self.get_option("privateKeyPath", options) - ssh_priv_key_pass = self.get_option("privateKeyPass", options) dst_path = self.get_option("destinationPath", options) + dst_path_cn = dst_path + "/" + cn export_format = self.get_option("exportFormat", options) # prepare files for upload @@ -121,6 +118,31 @@ class SFTPDestinationPlugin(DestinationPlugin): # store chain in the separate file files[cn + ".ca.bundle.pem"] = cert_chain + self.upload_file(dst_path_cn, files, options) + + # this is called from the acme http challenge + def upload_acme_token(self, token, thumbprint, options, **kwargs): + + current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge") + + dst_path = self.get_option("destinationPath", options) + dst_path = path.join(dst_path, "/.well-known/acme-challenge/") + + # prepare files for upload + files = {token: thumbprint} + + self.upload_file(dst_path, files, options) + + # here the file is uploaded for real, this helps to keep this class DRY + def upload_file(self, dst_path, files, options): + + host = self.get_option("host", options) + port = self.get_option("port", options) + user = self.get_option("user", options) + password = self.get_option("password", options) + ssh_priv_key = self.get_option("privateKeyPath", options) + ssh_priv_key_pass = self.get_option("privateKeyPass", options) + # upload files try: current_app.logger.debug( @@ -156,33 +178,26 @@ class SFTPDestinationPlugin(DestinationPlugin): sftp.mkdir(dst_path) except IOError: current_app.logger.debug("{0} already exist, resuming".format(dst_path)) - try: - dst_path_cn = dst_path + "/" + cn - current_app.logger.debug("Creating {0}".format(dst_path_cn)) - sftp.mkdir(dst_path_cn) - except IOError: - current_app.logger.debug( - "{0} already exist, resuming".format(dst_path_cn) - ) # upload certificate files to the sftp destination for filename, data in files.items(): current_app.logger.debug( - "Uploading {0} to {1}".format(filename, dst_path_cn) + "Uploading {0} to {1}".format(filename, dst_path) ) try: - with sftp.open(dst_path_cn + "/" + filename, "w") as f: + with sftp.open(dst_path + "/" + filename, "w") as f: f.write(data) - except (PermissionError) as permerror: + except PermissionError as permerror: if permerror.errno == 13: current_app.logger.debug( - "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(filename, dst_path_cn) + "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format( + filename, dst_path) ) - sftp.chmod(dst_path_cn + "/" + filename, 0o600) - with sftp.open(dst_path_cn + "/" + filename, "w") as f: + sftp.chmod(dst_path + "/" + filename, 0o600) + with sftp.open(dst_path + "/" + filename, "w") as f: f.write(data) # read only for owner, -r-------- - sftp.chmod(dst_path_cn + "/" + filename, 0o400) + sftp.chmod(dst_path + "/" + filename, 0o400) ssh.close() From 3012995c76e87b29ab30d5c3077c3e228852c4d6 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 18:32:30 +0200 Subject: [PATCH 03/46] Improve naming, make it possible to create directories recursively with SFTP --- lemur/plugins/lemur_sftp/plugin.py | 52 ++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 7c4e93c4..f2a8c9bf 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -16,7 +16,7 @@ .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov """ -from os import path +from os import path, walk import paramiko @@ -121,15 +121,17 @@ class SFTPDestinationPlugin(DestinationPlugin): self.upload_file(dst_path_cn, files, options) # this is called from the acme http challenge - def upload_acme_token(self, token, thumbprint, options, **kwargs): + def upload_acme_token(self, token_path, token, options, **kwargs): current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge") dst_path = self.get_option("destinationPath", options) - dst_path = path.join(dst_path, "/.well-known/acme-challenge/") + dst_path = path.join(dst_path, ".well-known/acme-challenge/") + + _, filename = path.split(token_path) # prepare files for upload - files = {token: thumbprint} + files = {filename: token} self.upload_file(dst_path, files, options) @@ -169,15 +171,37 @@ class SFTPDestinationPlugin(DestinationPlugin): ) raise paramiko.ssh_exception.AuthenticationException + # split the path into it's segments, so we can create it recursively + allparts = [] + path_copy = dst_path + while True: + parts = path.split(path_copy) + if parts[0] == path_copy: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path_copy: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path_copy = parts[0] + allparts.insert(0, parts[1]) + # open the sftp session inside the ssh connection sftp = ssh.open_sftp() - # make sure that the destination path exist - try: - current_app.logger.debug("Creating {0}".format(dst_path)) - sftp.mkdir(dst_path) - except IOError: - current_app.logger.debug("{0} already exist, resuming".format(dst_path)) + # make sure that the destination path exists, recursively + remote_path = allparts[0] + for part in allparts: + try: + if part != "/" and part != "": + remote_path = path.join(remote_path, part); + sftp.stat(remote_path) + except IOError: + current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path)) + try: + sftp.mkdir(remote_path) + except IOError as ioerror: + current_app.logger.debug("Couldn't create {0}, error message: {1}".format(remote_path, ioerror)) # upload certificate files to the sftp destination for filename, data in files.items(): @@ -185,7 +209,7 @@ class SFTPDestinationPlugin(DestinationPlugin): "Uploading {0} to {1}".format(filename, dst_path) ) try: - with sftp.open(dst_path + "/" + filename, "w") as f: + with sftp.open(path.join(dst_path, filename), "w") as f: f.write(data) except PermissionError as permerror: if permerror.errno == 13: @@ -193,11 +217,11 @@ class SFTPDestinationPlugin(DestinationPlugin): "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format( filename, dst_path) ) - sftp.chmod(dst_path + "/" + filename, 0o600) - with sftp.open(dst_path + "/" + filename, "w") as f: + sftp.chmod(path.join(dst_path, filename), 0o600) + with sftp.open(path.join(dst_path, filename), "w") as f: f.write(data) # read only for owner, -r-------- - sftp.chmod(dst_path + "/" + filename, 0o400) + sftp.chmod(path.join(dst_path, filename), 0o400) ssh.close() From e06bdcf2a3d75a598a09b73f613d0e59b8604774 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 18:33:40 +0200 Subject: [PATCH 04/46] Implement create_certificate for HTTP-01 challenge --- lemur/plugins/lemur_acme/plugin.py | 73 ++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index d1c86017..f8165c4d 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -30,6 +30,8 @@ from lemur.destinations import service as destination_service from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider from lemur.extensions import metrics, sentry + +from lemur.plugins.base import plugins from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns @@ -83,19 +85,19 @@ class AcmeHandler(object): def maybe_add_extension(self, host, dns_provider_options): if dns_provider_options and dns_provider_options.get( - "acme_challenge_extension" + "acme_challenge_extension" ): host = host + dns_provider_options.get("acme_challenge_extension") return host def start_dns_challenge( - self, - acme_client, - account_number, - host, - dns_provider, - order, - dns_provider_options, + self, + acme_client, + account_number, + host, + dns_provider, + order, + dns_provider_options, ): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) @@ -210,12 +212,12 @@ class AcmeHandler(object): if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: pem_certificate_chain = orderr.fullchain_pem[ - len(pem_certificate) : # noqa - ].lstrip() + len(pem_certificate): # noqa + ].lstrip() current_app.logger.debug( "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) @@ -692,6 +694,7 @@ class ACMEIssuerPlugin(IssuerPlugin): account_number = None provider_type = None + acme_client.new_order() domains = self.acme.get_domains(issuer_options) if not create_immediately: # Create pending authorizations that we'll need to do the creation @@ -810,7 +813,51 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): self.destination_list.append(destination.label) def create_certificate(self, csr, issuer_options): - pass + """ + Creates an ACME certificate using the HTTP-01 challenge. + + :param csr: + :param issuer_options: + :return: :raise Exception: + """ + self.acme = AcmeHandler() + authority = issuer_options.get("authority") + create_immediately = issuer_options.get("create_immediately", False) + acme_client, registration = self.acme.setup_acme_client(authority) + + orderr = acme_client.new_order(csr) + challenge = None + + for authz in orderr.authorizations: + # Choosing challenge. + # authz.body.challenges is a set of ChallengeBody objects. + for i in authz.body.challenges: + # Find the supported challenge. + if isinstance(i.chall, challenges.HTTP01): + challenge = i + + if challenge is None: + raise Exception('HTTP-01 challenge was not offered by the CA server.') + else: + # Here we probably should create a pending certificate and make use of celery, but for now + # I'll ignore all of that + for option in json.loads(issuer_options["authority"].options): + if option["name"] == "tokenDestination": + token_destination = destination_service.get_by_label(option["value"]) + + if token_destination is None: + raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') + + destination_plugin = plugins.get(token_destination.plugin_name) + destination_plugin.upload_acme_token(challenge.chall.path, challenge.chall.token, token_destination.options) + + current_app.logger.info("Uploaded HTTP-01 challenge token, trying to poll and finalize the order") + + pem_certificate, pem_certificate_chain = self.acme.request_certificate( + acme_client, orderr.authorizations, csr + ) + # TODO add external ID (if possible) + return pem_certificate, pem_certificate_chain, None @staticmethod def create_authority(options): @@ -837,4 +884,4 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): # Needed to override issuer function. - pass \ No newline at end of file + pass From b93d271f318d56935f741e1846e8525c890c2f81 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 08:39:41 +0200 Subject: [PATCH 05/46] Fix flake8 --- lemur/plugins/lemur_acme/plugin.py | 2 +- lemur/plugins/lemur_sftp/plugin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f8165c4d..d42573df 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -212,7 +212,7 @@ class AcmeHandler(object): if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: pem_certificate_chain = orderr.fullchain_pem[ diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index f2a8c9bf..e44052d2 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -16,7 +16,7 @@ .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov """ -from os import path, walk +from os import path import paramiko @@ -194,7 +194,7 @@ class SFTPDestinationPlugin(DestinationPlugin): for part in allparts: try: if part != "/" and part != "": - remote_path = path.join(remote_path, part); + remote_path = path.join(remote_path, part) sftp.stat(remote_path) except IOError: current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path)) From b2de9866528ade2fe815ae2656ab90b602e17b62 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 09:24:26 +0200 Subject: [PATCH 06/46] Split tests into handler, and dns specifics --- .../tests/{test_acme.py => test_acme_dns.py} | 88 +------- .../lemur_acme/tests/test_acme_handler.py | 194 ++++++++++++++++++ 2 files changed, 196 insertions(+), 86 deletions(-) rename lemur/plugins/lemur_acme/tests/{test_acme.py => test_acme_dns.py} (81%) create mode 100644 lemur/plugins/lemur_acme/tests/test_acme_handler.py diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py similarity index 81% rename from lemur/plugins/lemur_acme/tests/test_acme.py rename to lemur/plugins/lemur_acme/tests/test_acme_dns.py index ab246563..8074ca93 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -8,7 +8,7 @@ from lemur.common.utils import generate_private_key from mock import MagicMock -class TestAcme(unittest.TestCase): +class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") def setUp(self, mock_dns_provider_service): self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() @@ -39,19 +39,6 @@ class TestAcme(unittest.TestCase): result = yield self.acme.get_dns_challenges(host, mock_authz) self.assertEqual(result, mock_entry) - def test_strip_wildcard(self): - expected = ("example.com", False) - result = self.acme.strip_wildcard("example.com") - self.assertEqual(expected, result) - - expected = ("example.com", True) - result = self.acme.strip_wildcard("*.example.com") - self.assertEqual(expected, result) - - def test_authz_record(self): - a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") - self.assertEqual(type(a), plugin.AuthorizationRecord) - @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) @@ -68,7 +55,7 @@ class TestAcme(unittest.TestCase): from acme import challenges c = challenges.DNS01() - mock_entry.chall = TestAcme.test_complete_dns_challenge_fail + mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail mock_authz.body.resolved_combinations.append(mock_entry) mock_acme.request_domain_challenges = Mock(return_value=mock_authz) mock_dns_provider = Mock() @@ -336,77 +323,6 @@ class TestAcme(unittest.TestCase): dyn = provider.get_dns_provider("dyn") assert dyn - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificate(mock_cert) - self.assertEqual( - result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} - ) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - mock_cert2 = Mock() - mock_cert2.external_id = 2 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificates([mock_cert, mock_cert2]) - self.assertEqual(len(result), 2) - self.assertEqual( - result[0]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, - ) - self.assertEqual( - result[1]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, - ) - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py new file mode 100644 index 00000000..60ebf409 --- /dev/null +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -0,0 +1,194 @@ +import unittest +from unittest.mock import patch, Mock + +from cryptography.x509 import DNSName +from lemur.plugins.lemur_acme import plugin +from mock import MagicMock + + +class TestAcmeHandler(unittest.TestCase): + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + def setUp(self, mock_dns_provider_service): + self.acme = plugin.AcmeHandler() + mock_dns_provider = Mock() + mock_dns_provider.name = "cloudflare" + mock_dns_provider.credentials = "{}" + mock_dns_provider.provider_type = "cloudflare" + self.acme.dns_providers_for_domain = { + "www.test.com": [mock_dns_provider], + "test.fakedomain.net": [mock_dns_provider], + } + + def test_strip_wildcard(self): + expected = ("example.com", False) + result = self.acme.strip_wildcard("example.com") + self.assertEqual(expected, result) + + expected = ("example.com", True) + result = self.acme.strip_wildcard("*.example.com") + self.assertEqual(expected, result) + + def test_authz_record(self): + a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") + self.assertEqual(type(a), plugin.AuthorizationRecord) + + def test_setup_acme_client_fail(self): + mock_authority = Mock() + mock_authority.options = [] + with self.assertRaises(Exception): + self.acme.setup_acme_client(mock_authority) + + @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_setup_acme_client_success(self, mock_current_app, mock_acme): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_client = Mock() + mock_registration = Mock() + mock_registration.uri = "http://test.com" + mock_client.register = mock_registration + mock_client.agree_to_tos = Mock(return_value=True) + mock_acme.return_value = mock_client + mock_current_app.config = {} + result_client, result_registration = self.acme.setup_acme_client(mock_authority) + assert result_client + assert result_registration + + @patch('lemur.plugins.lemur_acme.plugin.current_app') + def test_get_domains_single(self, mock_current_app): + options = {"common_name": "test.netflix.net"} + result = self.acme.get_domains(options) + self.assertEqual(result, [options["common_name"]]) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_get_domains_multiple(self, mock_current_app): + options = { + "common_name": "test.netflix.net", + "extensions": { + "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]} + }, + } + result = self.acme.get_domains(options) + self.assertEqual( + result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] + ) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_get_domains_san(self, mock_current_app): + options = { + "common_name": "test.netflix.net", + "extensions": { + "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]} + }, + } + result = self.acme.get_domains(options) + self.assertEqual( + result, [options["common_name"], "test2.netflix.net"] + ) + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", + return_value="test", + ) + def test_get_authorizations(self, mock_start_dns_challenge): + mock_order = Mock() + mock_order.body.identifiers = [] + mock_domain = Mock() + mock_order.body.identifiers.append(mock_domain) + mock_order_info = Mock() + mock_order_info.account_number = 1 + mock_order_info.domains = ["test.fakedomain.net"] + result = self.acme.get_authorizations( + "acme_client", mock_order, mock_order_info + ) + self.assertEqual(result, ["test"]) + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge", + return_value="test", + ) + def test_finalize_authorizations(self, mock_complete_dns_challenge): + mock_authz = [] + mock_authz_record = MagicMock() + mock_authz_record.authz = Mock() + mock_authz_record.change_id = 1 + mock_authz_record.dns_challenge.validation_domain_name = Mock() + mock_authz_record.dns_challenge.validation = Mock() + mock_authz.append(mock_authz_record) + mock_dns_provider = Mock() + mock_dns_provider.delete_txt_record = Mock() + + mock_acme_client = Mock() + result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) + self.assertEqual(result, mock_authz) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + def test_get_ordered_certificate( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificate(mock_cert) + self.assertEqual( + result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} + ) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + def test_get_ordered_certificates( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + mock_cert2 = Mock() + mock_cert2.external_id = 2 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificates([mock_cert, mock_cert2]) + self.assertEqual(len(result), 2) + self.assertEqual( + result[0]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, + ) + self.assertEqual( + result[1]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, + ) From d6719b729c93aaa1806f901a8efb8197b04f6e0b Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 10:23:49 +0200 Subject: [PATCH 07/46] Implement some test for AcmeHttpIssuerPlugin --- .../lemur_acme/tests/test_acme_http.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 lemur/plugins/lemur_acme/tests/test_acme_http.py diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py new file mode 100644 index 00000000..f6183062 --- /dev/null +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -0,0 +1,122 @@ +import unittest +from unittest.mock import patch, Mock + +from acme import challenges +from lemur.plugins.lemur_acme import plugin +from mock import MagicMock + + +class TestAcmeHttp(unittest.TestCase): + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + def setUp(self, mock_dns_provider_service, mock_destination_provider): + self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin() + self.acme = plugin.AcmeHandler() + mock_dns_provider = Mock() + mock_dns_provider.name = "cloudflare" + mock_dns_provider.credentials = "{}" + mock_dns_provider.provider_type = "cloudflare" + self.acme.dns_providers_for_domain = { + "www.test.com": [mock_dns_provider], + "test.fakedomain.net": [mock_dns_provider], + } + mock_destination_provider = Mock() + mock_destination_provider.label = "mock-sftp-destination" + mock_destination_provider.plugin_name = "sftp-destination" + self.ACMEHttpIssuerPlugin.destination_list = ["mock-sftp-destination", "mock-s3-destination"] + + @patch("acme.client.Client") + @patch("OpenSSL.crypto", return_value="mock_cert") + @patch("josepy.util.ComparableX509") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_request_certificate( + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, + ): + mock_cert_response = Mock() + mock_cert_response.body = "123" + mock_cert_response_full = [mock_cert_response, True] + mock_acme.poll_and_request_issuance = Mock(return_value=mock_cert_response_full) + mock_authz = [] + mock_authz_record = MagicMock() + mock_authz_record.authz = Mock() + mock_authz.append(mock_authz_record) + mock_acme.fetch_chain = Mock(return_value="mock_chain") + mock_crypto.dump_certificate = Mock(return_value=b"chain") + mock_order = Mock() + mock_current_app.config = {} + self.acme.request_certificate(mock_acme, [], mock_order) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_create_authority(self, mock_current_app): + mock_current_app.config = Mock() + options = { + "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} + } + acme_root, b, role = self.ACMEHttpIssuerPlugin.create_authority(options) + self.assertEqual(acme_root, "123") + self.assertEqual(b, "") + self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}]) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate( + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + mock_dns_provider = Mock() + mock_dns_provider.credentials = '{"account_id": 1}' + mock_dns_provider.provider_type = "route53" + mock_dns_provider_service.get.return_value = mock_dns_provider + + mock_destination = Mock() + mock_destination.label = "mock-sftp-destination" + mock_destination.plugin_name = "SFTPDestinationPlugin" + mock_destination_service.get_by_label.return_value = mock_destination + + mock_destination_plugin = Mock() + mock_destination_plugin.upload_acme_token.return_value = True + mock_plugin_manager_get.return_value = mock_destination_plugin + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + result = provider.create_certificate(csr, issuer_options) + assert result From 76dcfbd528aa3a835e0062be0685ee5cba7edbca Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 11:35:27 +0200 Subject: [PATCH 08/46] Add more tests --- lemur/plugins/lemur_acme/plugin.py | 1 + .../lemur_acme/tests/test_acme_http.py | 132 +++++++++++++----- 2 files changed, 100 insertions(+), 33 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index d42573df..a3d15b6d 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -841,6 +841,7 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): else: # Here we probably should create a pending certificate and make use of celery, but for now # I'll ignore all of that + token_destination = None for option in json.loads(issuer_options["authority"].options): if option["name"] == "tokenDestination": token_destination = destination_service.get_by_label(option["value"]) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index f6183062..14d46344 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -3,7 +3,6 @@ from unittest.mock import patch, Mock from acme import challenges from lemur.plugins.lemur_acme import plugin -from mock import MagicMock class TestAcmeHttp(unittest.TestCase): @@ -25,33 +24,6 @@ class TestAcmeHttp(unittest.TestCase): mock_destination_provider.plugin_name = "sftp-destination" self.ACMEHttpIssuerPlugin.destination_list = ["mock-sftp-destination", "mock-s3-destination"] - @patch("acme.client.Client") - @patch("OpenSSL.crypto", return_value="mock_cert") - @patch("josepy.util.ComparableX509") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_request_certificate( - self, - mock_current_app, - mock_get_dns_challenges, - mock_jose, - mock_crypto, - mock_acme, - ): - mock_cert_response = Mock() - mock_cert_response.body = "123" - mock_cert_response_full = [mock_cert_response, True] - mock_acme.poll_and_request_issuance = Mock(return_value=mock_cert_response_full) - mock_authz = [] - mock_authz_record = MagicMock() - mock_authz_record.authz = Mock() - mock_authz.append(mock_authz_record) - mock_acme.fetch_chain = Mock(return_value="mock_chain") - mock_crypto.dump_certificate = Mock(return_value=b"chain") - mock_order = Mock() - mock_current_app.config = {} - self.acme.request_certificate(mock_acme, [], mock_order) - @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_create_authority(self, mock_current_app): mock_current_app.config = Mock() @@ -97,11 +69,6 @@ class TestAcmeHttp(unittest.TestCase): mock_client.new_order.return_value = mock_order_resource mock_acme.return_value = (mock_client, "") - mock_dns_provider = Mock() - mock_dns_provider.credentials = '{"account_id": 1}' - mock_dns_provider.provider_type = "route53" - mock_dns_provider_service.get.return_value = mock_dns_provider - mock_destination = Mock() mock_destination.label = "mock-sftp-destination" mock_destination.plugin_name = "SFTPDestinationPlugin" @@ -120,3 +87,102 @@ class TestAcmeHttp(unittest.TestCase): mock_request_certificate.return_value = ("pem_certificate", "chain") result = provider.create_certificate(csr, issuer_options) assert result + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate_missing_destination_token( + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + mock_destination = Mock() + mock_destination.label = "mock-sftp-destination" + mock_destination.plugin_name = "SFTPDestinationPlugin" + mock_destination_service.get_by_label.return_value = mock_destination + + mock_destination_plugin = Mock() + mock_destination_plugin.upload_acme_token.return_value = True + mock_plugin_manager_get.return_value = mock_destination_plugin + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + with self.assertRaisesRegexp(Exception, "No token_destination configured"): + provider.create_certificate(csr, issuer_options) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate_missing_http_challenge( + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.DNS01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + with self.assertRaisesRegexp(Exception, "HTTP-01 challenge was not offered"): + provider.create_certificate(csr, issuer_options) From e3e5ef7d66a35cfcb35ddefd3dbeb0e143e43206 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 11:56:12 +0200 Subject: [PATCH 09/46] Refactor AcmeHandler, Move DNS stuff into AcmeDnsHandler --- lemur/plugins/lemur_acme/plugin.py | 264 +++++++++--------- .../plugins/lemur_acme/tests/test_acme_dns.py | 119 +++++++- .../lemur_acme/tests/test_acme_handler.py | 119 +------- .../lemur_acme/tests/test_acme_http.py | 30 +- 4 files changed, 250 insertions(+), 282 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index a3d15b6d..c82ac529 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -48,33 +48,6 @@ class AuthorizationRecord(object): class AcmeHandler(object): - def __init__(self): - self.dns_providers_for_domain = {} - try: - self.all_dns_providers = dns_provider_service.get_all_dns_providers() - except Exception as e: - metrics.send("AcmeHandler_init_error", "counter", 1) - sentry.captureException() - current_app.logger.error(f"Unable to fetch DNS Providers: {e}") - self.all_dns_providers = [] - - def get_dns_challenges(self, host, authorizations): - """Get dns challenges for provided domain""" - - domain_to_validate, is_wildcard = self.strip_wildcard(host) - dns_challenges = [] - for authz in authorizations: - if not authz.body.identifier.value.lower() == domain_to_validate.lower(): - continue - if is_wildcard and not authz.body.wildcard: - continue - if not is_wildcard and authz.body.wildcard: - continue - for combo in authz.body.challenges: - if isinstance(combo.chall, challenges.DNS01): - dns_challenges.append(combo) - - return dns_challenges def strip_wildcard(self, host): """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" @@ -90,91 +63,6 @@ class AcmeHandler(object): host = host + dns_provider_options.get("acme_challenge_extension") return host - def start_dns_challenge( - self, - acme_client, - account_number, - host, - dns_provider, - order, - dns_provider_options, - ): - current_app.logger.debug("Starting DNS challenge for {0}".format(host)) - - change_ids = [] - dns_challenges = self.get_dns_challenges(host, order.authorizations) - host_to_validate, _ = self.strip_wildcard(host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) - - if not dns_challenges: - sentry.captureException() - metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1) - raise Exception("Unable to determine DNS challenges from authorizations") - - for dns_challenge in dns_challenges: - change_id = dns_provider.create_txt_record( - dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key), - account_number, - ) - change_ids.append(change_id) - - return AuthorizationRecord( - host, order.authorizations, dns_challenges, change_ids - ) - - def complete_dns_challenge(self, acme_client, authz_record): - current_app.logger.debug( - "Finalizing DNS challenge for {0}".format( - authz_record.authz[0].body.identifier.value - ) - ) - dns_providers = self.dns_providers_for_domain.get(authz_record.host) - if not dns_providers: - metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) - raise Exception( - "No DNS providers found for domain: {}".format(authz_record.host) - ) - - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - for change_id in authz_record.change_id: - try: - dns_provider_plugin.wait_for_dns_change( - change_id, account_number=account_number - ) - except Exception: - metrics.send("complete_dns_challenge_error", "counter", 1) - sentry.captureException() - current_app.logger.debug( - f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " - f"{account_number}", - exc_info=True, - ) - raise - - for dns_challenge in authz_record.dns_challenge: - response = dns_challenge.response(acme_client.client.net.key) - - verified = response.simple_verify( - dns_challenge.chall, - authz_record.host, - acme_client.client.net.key.public_key(), - ) - - if not verified: - metrics.send("complete_dns_challenge_verification_error", "counter", 1) - raise ValueError("Failed verification") - - time.sleep(5) - res = acme_client.answer_challenge(dns_challenge, response) - current_app.logger.debug(f"answer_challenge response: {res}") - def request_certificate(self, acme_client, authorizations, order): for authorization in authorizations: for authz in authorization.authz: @@ -310,6 +198,135 @@ class AcmeHandler(object): current_app.logger.debug("Got these domains: {0}".format(domains)) return domains + +class AcmeDnsHandler(AcmeHandler): + + def __init__(self): + self.dns_providers_for_domain = {} + try: + self.all_dns_providers = dns_provider_service.get_all_dns_providers() + except Exception as e: + metrics.send("AcmeHandler_init_error", "counter", 1) + sentry.captureException() + current_app.logger.error(f"Unable to fetch DNS Providers: {e}") + self.all_dns_providers = [] + + def get_dns_challenges(self, host, authorizations): + """Get dns challenges for provided domain""" + + domain_to_validate, is_wildcard = self.strip_wildcard(host) + dns_challenges = [] + for authz in authorizations: + if not authz.body.identifier.value.lower() == domain_to_validate.lower(): + continue + if is_wildcard and not authz.body.wildcard: + continue + if not is_wildcard and authz.body.wildcard: + continue + for combo in authz.body.challenges: + if isinstance(combo.chall, challenges.DNS01): + dns_challenges.append(combo) + + return dns_challenges + + def get_dns_provider(self, type): + provider_types = { + "cloudflare": cloudflare, + "dyn": dyn, + "route53": route53, + "ultradns": ultradns, + "powerdns": powerdns + } + provider = provider_types.get(type) + if not provider: + raise UnknownProvider("No such DNS provider: {}".format(type)) + return provider + + def start_dns_challenge( + self, + acme_client, + account_number, + host, + dns_provider, + order, + dns_provider_options, + ): + current_app.logger.debug("Starting DNS challenge for {0}".format(host)) + + change_ids = [] + dns_challenges = self.get_dns_challenges(host, order.authorizations) + host_to_validate, _ = self.strip_wildcard(host) + host_to_validate = self.maybe_add_extension( + host_to_validate, dns_provider_options + ) + + if not dns_challenges: + sentry.captureException() + metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1) + raise Exception("Unable to determine DNS challenges from authorizations") + + for dns_challenge in dns_challenges: + change_id = dns_provider.create_txt_record( + dns_challenge.validation_domain_name(host_to_validate), + dns_challenge.validation(acme_client.client.net.key), + account_number, + ) + change_ids.append(change_id) + + return AuthorizationRecord( + host, order.authorizations, dns_challenges, change_ids + ) + + def complete_dns_challenge(self, acme_client, authz_record): + current_app.logger.debug( + "Finalizing DNS challenge for {0}".format( + authz_record.authz[0].body.identifier.value + ) + ) + dns_providers = self.dns_providers_for_domain.get(authz_record.host) + if not dns_providers: + metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) + raise Exception( + "No DNS providers found for domain: {}".format(authz_record.host) + ) + + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + for change_id in authz_record.change_id: + try: + dns_provider_plugin.wait_for_dns_change( + change_id, account_number=account_number + ) + except Exception: + metrics.send("complete_dns_challenge_error", "counter", 1) + sentry.captureException() + current_app.logger.debug( + f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " + f"{account_number}", + exc_info=True, + ) + raise + + for dns_challenge in authz_record.dns_challenge: + response = dns_challenge.response(acme_client.client.net.key) + + verified = response.simple_verify( + dns_challenge.chall, + authz_record.host, + acme_client.client.net.key.public_key(), + ) + + if not verified: + metrics.send("complete_dns_challenge_verification_error", "counter", 1) + raise ValueError("Failed verification") + + time.sleep(5) + res = acme_client.answer_challenge(dns_challenge, response) + current_app.logger.debug(f"answer_challenge response: {res}") + def get_authorizations(self, acme_client, order, order_info): authorizations = [] @@ -421,19 +438,6 @@ class AcmeHandler(object): sentry.captureException() pass - def get_dns_provider(self, type): - provider_types = { - "cloudflare": cloudflare, - "dyn": dyn, - "route53": route53, - "ultradns": ultradns, - "powerdns": powerdns - } - provider = provider_types.get(type) - if not provider: - raise UnknownProvider("No such DNS provider: {}".format(type)) - return provider - class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" @@ -487,7 +491,7 @@ class ACMEIssuerPlugin(IssuerPlugin): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) def get_dns_provider(self, type): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() provider_types = { "cloudflare": cloudflare, @@ -502,14 +506,14 @@ class ACMEIssuerPlugin(IssuerPlugin): return provider def get_all_zones(self, dns_provider): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) return dns_provider_plugin.get_zones(account_number=account_number) def get_ordered_certificate(self, pending_cert): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) if pending_cert.dns_provider_id: @@ -555,7 +559,7 @@ class ACMEIssuerPlugin(IssuerPlugin): return cert def get_ordered_certificates(self, pending_certs): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() pending = [] certs = [] for pending_cert in pending_certs: @@ -665,7 +669,7 @@ class ACMEIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() authority = issuer_options.get("authority") create_immediately = issuer_options.get("create_immediately", False) acme_client, registration = self.acme.setup_acme_client(authority) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py index 8074ca93..6b4371d6 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -12,7 +12,7 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") def setUp(self, mock_dns_provider_service): self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() - self.acme = plugin.AcmeHandler() + self.acme = plugin.AcmeDnsHandler() mock_dns_provider = Mock() mock_dns_provider.name = "cloudflare" mock_dns_provider.credentials = "{}" @@ -42,7 +42,7 @@ class TestAcmeDns(unittest.TestCase): @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") def test_start_dns_challenge( self, mock_get_dns_challenges, mock_len, mock_app, mock_acme ): @@ -124,7 +124,7 @@ class TestAcmeDns(unittest.TestCase): @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") @patch("josepy.util.ComparableX509") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( self, @@ -326,9 +326,9 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate( self, @@ -360,3 +360,110 @@ class TestAcmeDns(unittest.TestCase): mock_request_certificate.return_value = ("pem_certificate", "chain") result = provider.create_certificate(csr, issuer_options) assert result + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.start_dns_challenge", + return_value="test", + ) + def test_get_authorizations(self, mock_start_dns_challenge): + mock_order = Mock() + mock_order.body.identifiers = [] + mock_domain = Mock() + mock_order.body.identifiers.append(mock_domain) + mock_order_info = Mock() + mock_order_info.account_number = 1 + mock_order_info.domains = ["test.fakedomain.net"] + result = self.acme.get_authorizations( + "acme_client", mock_order, mock_order_info + ) + self.assertEqual(result, ["test"]) + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.complete_dns_challenge", + return_value="test", + ) + def test_finalize_authorizations(self, mock_complete_dns_challenge): + mock_authz = [] + mock_authz_record = MagicMock() + mock_authz_record.authz = Mock() + mock_authz_record.change_id = 1 + mock_authz_record.dns_challenge.validation_domain_name = Mock() + mock_authz_record.dns_challenge.validation = Mock() + mock_authz.append(mock_authz_record) + mock_dns_provider = Mock() + mock_dns_provider.delete_txt_record = Mock() + + mock_acme_client = Mock() + result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) + self.assertEqual(result, mock_authz) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") + def test_get_ordered_certificate( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificate(mock_cert) + self.assertEqual( + result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} + ) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") + def test_get_ordered_certificates( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + mock_cert2 = Mock() + mock_cert2.external_id = 2 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificates([mock_cert, mock_cert2]) + self.assertEqual(len(result), 2) + self.assertEqual( + result[0]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, + ) + self.assertEqual( + result[1]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, + ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 60ebf409..b586aa9f 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -3,21 +3,11 @@ from unittest.mock import patch, Mock from cryptography.x509 import DNSName from lemur.plugins.lemur_acme import plugin -from mock import MagicMock class TestAcmeHandler(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - def setUp(self, mock_dns_provider_service): + def setUp(self): self.acme = plugin.AcmeHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "cloudflare" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "cloudflare" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } def test_strip_wildcard(self): expected = ("example.com", False) @@ -85,110 +75,3 @@ class TestAcmeHandler(unittest.TestCase): self.assertEqual( result, [options["common_name"], "test2.netflix.net"] ) - - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", - return_value="test", - ) - def test_get_authorizations(self, mock_start_dns_challenge): - mock_order = Mock() - mock_order.body.identifiers = [] - mock_domain = Mock() - mock_order.body.identifiers.append(mock_domain) - mock_order_info = Mock() - mock_order_info.account_number = 1 - mock_order_info.domains = ["test.fakedomain.net"] - result = self.acme.get_authorizations( - "acme_client", mock_order, mock_order_info - ) - self.assertEqual(result, ["test"]) - - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge", - return_value="test", - ) - def test_finalize_authorizations(self, mock_complete_dns_challenge): - mock_authz = [] - mock_authz_record = MagicMock() - mock_authz_record.authz = Mock() - mock_authz_record.change_id = 1 - mock_authz_record.dns_challenge.validation_domain_name = Mock() - mock_authz_record.dns_challenge.validation = Mock() - mock_authz.append(mock_authz_record) - mock_dns_provider = Mock() - mock_dns_provider.delete_txt_record = Mock() - - mock_acme_client = Mock() - result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) - self.assertEqual(result, mock_authz) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificate(mock_cert) - self.assertEqual( - result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} - ) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - mock_cert2 = Mock() - mock_cert2.external_id = 2 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificates([mock_cert, mock_cert2]) - self.assertEqual(len(result), 2) - self.assertEqual( - result[0]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, - ) - self.assertEqual( - result[1]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, - ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index 14d46344..ea81b5c4 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -6,19 +6,11 @@ from lemur.plugins.lemur_acme import plugin class TestAcmeHttp(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - def setUp(self, mock_dns_provider_service, mock_destination_provider): + def setUp(self, mock_destination_provider): self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin() self.acme = plugin.AcmeHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "cloudflare" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "cloudflare" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } + mock_destination_provider = Mock() mock_destination_provider.label = "mock-sftp-destination" mock_destination_provider.plugin_name = "sftp-destination" @@ -38,20 +30,14 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate( self, mock_authorization_service, mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, mock_current_app, - mock_dns_provider_service, mock_destination_service, mock_plugin_manager_get, mock_acme, @@ -91,20 +77,14 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate_missing_destination_token( self, mock_authorization_service, mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, mock_current_app, - mock_dns_provider_service, mock_destination_service, mock_plugin_manager_get, mock_acme, @@ -145,20 +125,14 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate_missing_http_challenge( self, mock_authorization_service, mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, mock_current_app, - mock_dns_provider_service, mock_destination_service, mock_plugin_manager_get, mock_acme, From 66cab6abd3e76e818e5987cacea83e6853204c78 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 17:40:15 +0200 Subject: [PATCH 10/46] Make http-01 challenge work for SAN certificates --- lemur/plugins/lemur_acme/plugin.py | 45 ++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index c82ac529..841531a5 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -830,17 +830,17 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): acme_client, registration = self.acme.setup_acme_client(authority) orderr = acme_client.new_order(csr) - challenge = None + chall = [] for authz in orderr.authorizations: # Choosing challenge. # authz.body.challenges is a set of ChallengeBody objects. for i in authz.body.challenges: # Find the supported challenge. if isinstance(i.chall, challenges.HTTP01): - challenge = i + chall.append(i) - if challenge is None: + if len(chall) == 0: raise Exception('HTTP-01 challenge was not offered by the CA server.') else: # Here we probably should create a pending certificate and make use of celery, but for now @@ -854,13 +854,42 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') destination_plugin = plugins.get(token_destination.plugin_name) - destination_plugin.upload_acme_token(challenge.chall.path, challenge.chall.token, token_destination.options) - current_app.logger.info("Uploaded HTTP-01 challenge token, trying to poll and finalize the order") + for challenge in chall: + response, validation = challenge.response_and_validation(acme_client.net.key) + + destination_plugin.upload_acme_token(challenge.chall.path, validation, token_destination.options) + + # Let the CA server know that we are ready for the challenge. + acme_client.answer_challenge(challenge, response) + + current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") + + # Wait for challenge status and then issue a certificate. + + for authz in orderr.authorizations: + authzr, resp = acme_client.poll(authz) + current_app.logger.info(authzr.body.status) + + # It is possible to set a deadline time. + finalized_orderr = acme_client.finalize_order(orderr, datetime.datetime.now() + datetime.timedelta(minutes=1)) + + pem_certificate = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, finalized_orderr.fullchain_pem + ), + ).decode() + + if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ + and datetime.datetime.now() < datetime.datetime.strptime( + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") + else: + pem_certificate_chain = finalized_orderr.fullchain_pem[ + len(pem_certificate): # noqa + ].lstrip() - pem_certificate, pem_certificate_chain = self.acme.request_certificate( - acme_client, orderr.authorizations, csr - ) # TODO add external ID (if possible) return pem_certificate, pem_certificate_chain, None From d24fae0bacf0aad20ed088097cbceab308739147 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 17:40:51 +0200 Subject: [PATCH 11/46] Fix permissions on acme token upload, dont append well-known automatically --- lemur/plugins/lemur_acme/plugin.py | 3 +-- lemur/plugins/lemur_sftp/plugin.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 841531a5..a3d3fffe 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -866,7 +866,6 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") # Wait for challenge status and then issue a certificate. - for authz in orderr.authorizations: authzr, resp = acme_client.poll(authz) current_app.logger.info(authzr.body.status) @@ -883,7 +882,7 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: pem_certificate_chain = finalized_orderr.fullchain_pem[ diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index e44052d2..1c974a28 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -126,7 +126,6 @@ class SFTPDestinationPlugin(DestinationPlugin): current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge") dst_path = self.get_option("destinationPath", options) - dst_path = path.join(dst_path, ".well-known/acme-challenge/") _, filename = path.split(token_path) @@ -220,8 +219,8 @@ class SFTPDestinationPlugin(DestinationPlugin): sftp.chmod(path.join(dst_path, filename), 0o600) with sftp.open(path.join(dst_path, filename), "w") as f: f.write(data) - # read only for owner, -r-------- - sftp.chmod(path.join(dst_path, filename), 0o400) + # most likely the upload user isn't the webuser, -rw-r--r-- + sftp.chmod(path.join(dst_path, filename), 0o644) ssh.close() From 41ea59d7e39119f993ca33238f7f5b0fdf0da508 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Thu, 1 Oct 2020 08:08:55 +0200 Subject: [PATCH 12/46] Remove unneeded polling --- lemur/plugins/lemur_acme/plugin.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index a3d3fffe..f02133a4 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -865,11 +865,6 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") - # Wait for challenge status and then issue a certificate. - for authz in orderr.authorizations: - authzr, resp = acme_client.poll(authz) - current_app.logger.info(authzr.body.status) - # It is possible to set a deadline time. finalized_orderr = acme_client.finalize_order(orderr, datetime.datetime.now() + datetime.timedelta(minutes=1)) From 215070b327c98def2d1e86a33d874cd010ede67f Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Thu, 1 Oct 2020 08:09:17 +0200 Subject: [PATCH 13/46] Fix create_certificate tests --- .../lemur_acme/tests/test_acme_http.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index ea81b5c4..2e2ae0fb 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -46,13 +46,23 @@ class TestAcmeHttp(unittest.TestCase): mock_authority = Mock() mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + mock_current_app.config = {} + mock_order_resource = Mock() mock_order_resource.authorizations = [Mock()] mock_order_resource.authorizations[0].body.challenges = [Mock()] - mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = (Mock(), "Anything-goes") + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') mock_client = Mock() mock_client.new_order.return_value = mock_order_resource + mock_client.answer_challenge.return_value = True + + mock_finalized_order = Mock() + mock_finalized_order.fullchain_pem = "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n" + mock_client.finalize_order.return_value = mock_finalized_order + mock_acme.return_value = (mock_client, "") mock_destination = Mock() @@ -71,8 +81,11 @@ class TestAcmeHttp(unittest.TestCase): } csr = "123" mock_request_certificate.return_value = ("pem_certificate", "chain") - result = provider.create_certificate(csr, issuer_options) - assert result + pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options) + + self.assertEqual(pem_certificate, "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n") + self.assertEqual(pem_certificate_chain, + "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") From 81b078604c7f3d183420b2f6ce8c62aebbff7143 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 6 Oct 2020 11:04:04 +0200 Subject: [PATCH 14/46] Implement revoke certificate for ACME --- lemur/plugins/lemur_acme/plugin.py | 71 ++++++++++++++++++- .../lemur_acme/tests/test_acme_handler.py | 28 ++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f02133a4..b8cbdc55 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -49,6 +49,29 @@ class AuthorizationRecord(object): class AcmeHandler(object): + def reuse_account(self, authority): + if not authority.options: + raise InvalidAuthority("Invalid authority. Options not set") + existing_key = False + existing_regr = False + + for option in json.loads(authority.options): + if option["name"] == "acme_private_key" and option["value"]: + existing_key = True + if option["name"] == "acme_regr" and option["value"]: + existing_regr = True + + if not existing_key and current_app.config.get("ACME_PRIVATE_KEY"): + existing_key = True + + if not existing_regr and current_app.config.get("ACME_REGR"): + existing_regr = True + + if existing_key and existing_regr: + return True + else: + return False + def strip_wildcard(self, host): """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" prefix = "*." @@ -755,6 +778,28 @@ class ACMEIssuerPlugin(IssuerPlugin): # Needed to override issuer function. pass + def revoke_certificate(self, certificate, comments): + self.acme = AcmeDnsHandler() + if not self.acme.reuse_account(certificate.authority): + raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") + acme_client, _ = self.acme.setup_acme_client(certificate.authority) + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, certificate.body)) + + try: + acme_client.revoke(fullchain_com, 0) # revocation reason = 0 + except (errors.ConflictError, errors.ClientError, errors.Error) as e: + # Certificate already revoked. + current_app.logger.error("Certificate revocation failed with message: " + e.detail) + metrics.send("acme_revoke_certificate_failure", "counter", 1) + return False + + current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) + metrics.send("acme_revoke_certificate_success", "counter", 1) + return True + class ACMEHttpIssuerPlugin(IssuerPlugin): title = "Acme HTTP-01" @@ -884,8 +929,8 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): len(pem_certificate): # noqa ].lstrip() - # TODO add external ID (if possible) - return pem_certificate, pem_certificate_chain, None + # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate + return pem_certificate, pem_certificate_chain, validation[0:128] @staticmethod def create_authority(options): @@ -913,3 +958,25 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): # Needed to override issuer function. pass + + def revoke_certificate(self, certificate, comments): + self.acme = AcmeHandler() + if not self.acme.reuse_account(certificate.authority): + raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") + acme_client, _ = self.acme.setup_acme_client(certificate.authority) + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, certificate.body)) + + try: + acme_client.revoke(fullchain_com, 0) # revocation reason = 0 + except (errors.ConflictError, errors.ClientError, errors.Error) as e: + # Certificate already revoked. + current_app.logger.error("Certificate revocation failed with message: " + e.detail) + metrics.send("acme_revoke_certificate_failure", "counter", 1) + return False + + current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) + metrics.send("acme_revoke_certificate_success", "counter", 1) + return True diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index b586aa9f..57ddcee6 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -28,6 +28,34 @@ class TestAcmeHandler(unittest.TestCase): with self.assertRaises(Exception): self.acme.setup_acme_client(mock_authority) + def test_reuse_account_not_defined(self): + mock_authority = Mock() + mock_authority.options = [] + with self.assertRaises(Exception): + self.acme.reuse_account(mock_authority) + + def test_reuse_account_from_authority(self): + mock_authority = Mock() + mock_authority.options = '[{"name": "acme_private_key", "value": "PRIVATE_KEY"}, {"name": "acme_regr", "value": "ACME_REGR"}]' + + self.assertTrue(self.acme.reuse_account(mock_authority)) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_reuse_account_from_config(self, mock_current_app): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_current_app.config = {"ACME_PRIVATE_KEY": "PRIVATE_KEY", "ACME_REGR": "ACME_REGR"} + + self.assertTrue(self.acme.reuse_account(mock_authority)) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_reuse_account_no_configuration(self, mock_current_app): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_current_app.config = {} + + self.assertFalse(self.acme.reuse_account(mock_authority)) + @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_setup_acme_client_success(self, mock_current_app, mock_acme): From 235653b55842022588faddff10d1bd9c5d8734c4 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 7 Oct 2020 11:43:17 +0200 Subject: [PATCH 15/46] Refactor destination selection for acme-http authorities, to load destinations dynamically --- lemur/plugins/lemur_acme/plugin.py | 9 ++------- .../angular/authorities/authority/authority.js | 8 +++++++- .../authorities/authority/options.tpl.html | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index b8cbdc55..106103d2 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -721,7 +721,6 @@ class ACMEIssuerPlugin(IssuerPlugin): account_number = None provider_type = None - acme_client.new_order() domains = self.acme.get_domains(issuer_options) if not create_immediately: # Create pending authorizations that we'll need to do the creation @@ -844,9 +843,8 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): }, { "name": "tokenDestination", - "type": "select", + "type": "destinationSelect", "required": True, - "available": destination_list, "helpMessage": "The destination to use to deploy the token.", }, ] @@ -871,7 +869,6 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): """ self.acme = AcmeHandler() authority = issuer_options.get("authority") - create_immediately = issuer_options.get("create_immediately", False) acme_client, registration = self.acme.setup_acme_client(authority) orderr = acme_client.new_order(csr) @@ -888,12 +885,10 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): if len(chall) == 0: raise Exception('HTTP-01 challenge was not offered by the CA server.') else: - # Here we probably should create a pending certificate and make use of celery, but for now - # I'll ignore all of that token_destination = None for option in json.loads(issuer_options["authority"].options): if option["name"] == "tokenDestination": - token_destination = destination_service.get_by_label(option["value"]) + token_destination = destination_service.get(option["value"]) if token_destination is None: raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') diff --git a/lemur/static/app/angular/authorities/authority/authority.js b/lemur/static/app/angular/authorities/authority/authority.js index a449cff5..82f38a92 100644 --- a/lemur/static/app/angular/authorities/authority/authority.js +++ b/lemur/static/app/angular/authorities/authority/authority.js @@ -34,7 +34,7 @@ angular.module('lemur') }; }) - .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster) { + .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster, DestinationService) { $scope.authority = LemurRestangular.restangularizeElement(null, {}, 'authorities'); // set the defaults AuthorityService.getDefaults($scope.authority).then(function () { @@ -52,6 +52,12 @@ angular.module('lemur') }); }); + $scope.getDestinations = function() { + return DestinationService.findDestinationsByName('').then(function(destinations) { + $scope.destinations = destinations; + }); + }; + $scope.getAuthoritiesByName = function (value) { return AuthorityService.findAuthorityByName(value).then(function (authorities) { $scope.authorities = authorities; diff --git a/lemur/static/app/angular/authorities/authority/options.tpl.html b/lemur/static/app/angular/authorities/authority/options.tpl.html index adf8eacc..e683c688 100644 --- a/lemur/static/app/angular/authorities/authority/options.tpl.html +++ b/lemur/static/app/angular/authorities/authority/options.tpl.html @@ -72,11 +72,28 @@
+ + + + + {{$select.selected.label}} + +
+ + + + + + +