From 749aa772ba459a40dd59cc3b3ebd1e73cee99ef2 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 26 Oct 2020 11:57:33 -0700 Subject: [PATCH 1/5] First change to get CNAME redirection working --- docs/administration.rst | 14 ++++++++++++++ lemur/plugins/lemur_acme/plugin.py | 22 +++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 846a4c34..80d88feb 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -620,6 +620,20 @@ If you are not using a metric provider you do not need to configure any of these Plugin Specific Options ----------------------- +ACME Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. data:: ACME_DNS_PROVIDER_TYPES + :noindex: + + Dictionary of ACME DNS Providers and their requirements. + +.. data:: ACME_ENABLE_DELEGATED_CNAME + :noindex: + + Enables delegated DNS domain validation using CNAMES. When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges. + + Active Directory Certificate Services Plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 16d61a0f..9177d6e8 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -16,6 +16,7 @@ import json import time import OpenSSL.crypto +import dns.resolver import josepy as jose from acme import challenges, errors, messages from acme.client import BackwardsCompatibleClientV2, ClientNetwork @@ -23,7 +24,6 @@ from acme.errors import PollError, TimeoutError, WildcardUnsupportedError from acme.messages import Error as AcmeError from botocore.exceptions import ClientError from flask import current_app - from lemur.authorizations import service as authorization_service from lemur.common.utils import generate_private_key from lemur.dns_providers import service as dns_provider_service @@ -287,6 +287,13 @@ class AcmeHandler(object): authorizations = [] for domain in order_info.domains: + + # Replace domain if doing CNAME delegation + if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): + cname = self.get_cname(domain) + if cname: + domain = cname + if not self.dns_providers_for_domain.get(domain): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 @@ -407,6 +414,19 @@ class AcmeHandler(object): raise UnknownProvider("No such DNS provider: {}".format(type)) return provider + def get_cname(self, domain): + """ + :param domain: Domain name to look up a CNAME for. + :param record_type: Type of DNS record to lookup. + :return: First CNAME target or False if no CNAME record exists. + """ + try: + result = dns.resolver.query(domain, 'CNAME') + if len(result) > 0: + return str(result[0].target).rstrip('.') + except dns.exception.DNSException: + return False + class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" From b47667b73e08f47a649c79427a252e0a628ec831 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 28 Oct 2020 20:51:35 -0700 Subject: [PATCH 2/5] cname redirection working --- lemur/plugins/lemur_acme/plugin.py | 48 ++++++++++++++------- lemur/plugins/lemur_acme/tests/test_acme.py | 12 +++--- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 2a09dbdf..4105465a 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -37,7 +37,8 @@ from retrying import retry class AuthorizationRecord(object): - def __init__(self, host, authz, dns_challenge, change_id): + def __init__(self, domain, host, authz, dns_challenge, change_id): + self.domain = domain self.host = host self.authz = authz self.dns_challenge = dns_challenge @@ -91,6 +92,7 @@ class AcmeHandler(object): self, acme_client, account_number, + domain, host, dns_provider, order, @@ -99,11 +101,9 @@ class AcmeHandler(object): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) change_ids = [] - dns_challenges = self.get_dns_challenges(host, order.authorizations) + dns_challenges = self.get_dns_challenges(domain, order.authorizations) host_to_validate, _ = self.strip_wildcard(host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) if not dns_challenges: sentry.captureException() @@ -111,15 +111,20 @@ class AcmeHandler(object): raise Exception("Unable to determine DNS challenges from authorizations") for dns_challenge in dns_challenges: + + # Only prepend '_acme-challenge' if not using CNAME redirection + if domain == host: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate) + change_id = dns_provider.create_txt_record( - dns_challenge.validation_domain_name(host_to_validate), + host_to_validate, dns_challenge.validation(acme_client.client.net.key), account_number, ) change_ids.append(change_id) return AuthorizationRecord( - host, order.authorizations, dns_challenges, change_ids + domain, host, order.authorizations, dns_challenges, change_ids ) def complete_dns_challenge(self, acme_client, authz_record): @@ -312,18 +317,23 @@ class AcmeHandler(object): for domain in order_info.domains: - # Replace domain if doing CNAME delegation + # If CNAME exists, set host to the target address + host = domain if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): - cname = self.get_cname(domain) - if cname: - domain = cname + val_domain, _ = self.strip_wildcard(domain) + val_domain = challenges.DNS01().validation_domain_name(val_domain) + cname_res = self.get_cname(val_domain) + if cname_res: + host = cname_res + self.autodetect_dns_providers(host) - if not self.dns_providers_for_domain.get(domain): + if not self.dns_providers_for_domain.get(host): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 ) - raise Exception("No DNS providers found for domain: {}".format(domain)) - for dns_provider in self.dns_providers_for_domain[domain]: + raise Exception("No DNS providers found for domain: {}".format(host)) + + for dns_provider in self.dns_providers_for_domain[host]: dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") @@ -331,6 +341,7 @@ class AcmeHandler(object): acme_client, account_number, domain, + host, dns_provider_plugin, order, dns_provider.options, @@ -377,10 +388,12 @@ class AcmeHandler(object): host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) + if authz_record.domain == authz_record.host: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate), dns_provider_plugin.delete_txt_record( authz_record.change_id, account_number, - dns_challenge.validation_domain_name(host_to_validate), + host_to_validate, dns_challenge.validation(acme_client.client.net.key), ) @@ -409,13 +422,16 @@ class AcmeHandler(object): host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) for dns_challenge in dns_challenges: + if authz_record.domain == authz_record.host: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate), try: dns_provider_plugin.delete_txt_record( authz_record.change_id, account_number, - dns_challenge.validation_domain_name(host_to_validate), + host_to_validate, dns_challenge.validation(acme_client.client.net.key), ) except Exception as e: diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index ab246563..cce97d32 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -49,7 +49,7 @@ class TestAcme(unittest.TestCase): self.assertEqual(expected, result) def test_authz_record(self): - a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") + a = plugin.AuthorizationRecord("domain", "host", "authz", "challenge", "id") self.assertEqual(type(a), plugin.AuthorizationRecord) @patch("acme.client.Client") @@ -79,7 +79,7 @@ class TestAcme(unittest.TestCase): iterator = iter(values) iterable.__iter__.return_value = iterator result = self.acme.start_dns_challenge( - mock_acme, "accountid", "host", mock_dns_provider, mock_order, {} + mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {} ) self.assertEqual(type(result), plugin.AuthorizationRecord) @@ -270,11 +270,9 @@ class TestAcme(unittest.TestCase): result, [options["common_name"], "test2.netflix.net"] ) - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", - return_value="test", - ) - def test_get_authorizations(self, mock_start_dns_challenge): + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", return_value="test") + @patch("lemur.plugins.lemur_acme.plugin.current_app", return_value=False) + def test_get_authorizations(self, mock_current_app, mock_start_dns_challenge): mock_order = Mock() mock_order.body.identifiers = [] mock_domain = Mock() From 33a006bbeba013e4e3f99c5f257458d9199c942a Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 28 Oct 2020 22:24:37 -0700 Subject: [PATCH 3/5] fixing delete with optional validation --- lemur/plugins/lemur_acme/plugin.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 4105465a..63fa5626 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -385,11 +385,9 @@ class AcmeHandler(object): dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") host_to_validate, _ = self.strip_wildcard(authz_record.host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) if authz_record.domain == authz_record.host: - host_to_validate = dns_challenge.validation_domain_name(host_to_validate), + host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate) dns_provider_plugin.delete_txt_record( authz_record.change_id, account_number, From 2b91077d9217a55a8ca022027091e7e0b88e025f Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 29 Oct 2020 13:51:22 -0700 Subject: [PATCH 4/5] updating variables based on feedback --- lemur/plugins/lemur_acme/plugin.py | 55 ++++++++++----------- lemur/plugins/lemur_acme/tests/test_acme.py | 4 +- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 63fa5626..0baa0478 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -37,9 +37,9 @@ from retrying import retry class AuthorizationRecord(object): - def __init__(self, domain, host, authz, dns_challenge, change_id): + def __init__(self, domain, target_domain, authz, dns_challenge, change_id): self.domain = domain - self.host = host + self.target_domain = target_domain self.authz = authz self.dns_challenge = dns_challenge self.change_id = change_id @@ -93,16 +93,16 @@ class AcmeHandler(object): acme_client, account_number, domain, - host, + target_domain, dns_provider, order, dns_provider_options, ): - current_app.logger.debug("Starting DNS challenge for {0}".format(host)) + current_app.logger.debug("Starting DNS challenge for {0}".format(target_domain)) change_ids = [] dns_challenges = self.get_dns_challenges(domain, order.authorizations) - host_to_validate, _ = self.strip_wildcard(host) + host_to_validate, _ = self.strip_wildcard(target_domain) host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) if not dns_challenges: @@ -113,7 +113,7 @@ class AcmeHandler(object): for dns_challenge in dns_challenges: # Only prepend '_acme-challenge' if not using CNAME redirection - if domain == host: + if domain == target_domain: host_to_validate = dns_challenge.validation_domain_name(host_to_validate) change_id = dns_provider.create_txt_record( @@ -124,7 +124,7 @@ class AcmeHandler(object): change_ids.append(change_id) return AuthorizationRecord( - domain, host, order.authorizations, dns_challenges, change_ids + domain, target_domain, order.authorizations, dns_challenges, change_ids ) def complete_dns_challenge(self, acme_client, authz_record): @@ -133,11 +133,11 @@ class AcmeHandler(object): authz_record.authz[0].body.identifier.value ) ) - dns_providers = self.dns_providers_for_domain.get(authz_record.host) + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) if not dns_providers: metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) raise Exception( - "No DNS providers found for domain: {}".format(authz_record.host) + "No DNS providers found for domain: {}".format(authz_record.target_domain) ) for dns_provider in dns_providers: @@ -165,7 +165,7 @@ class AcmeHandler(object): verified = response.simple_verify( dns_challenge.chall, - authz_record.host, + authz_record.target_domain, acme_client.client.net.key.public_key(), ) @@ -318,22 +318,22 @@ class AcmeHandler(object): for domain in order_info.domains: # If CNAME exists, set host to the target address - host = domain + target_domain = domain if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): - val_domain, _ = self.strip_wildcard(domain) - val_domain = challenges.DNS01().validation_domain_name(val_domain) - cname_res = self.get_cname(val_domain) - if cname_res: - host = cname_res - self.autodetect_dns_providers(host) + cname_result, _ = self.strip_wildcard(domain) + cname_result = challenges.DNS01().validation_domain_name(cname_result) + cname_result = self.get_cname(cname_result) + if cname_result: + target_domain = cname_result + self.autodetect_dns_providers(target_domain) - if not self.dns_providers_for_domain.get(host): + if not self.dns_providers_for_domain.get(target_domain): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 ) - raise Exception("No DNS providers found for domain: {}".format(host)) + raise Exception("No DNS providers found for domain: {}".format(target_domain)) - for dns_provider in self.dns_providers_for_domain[host]: + for dns_provider in self.dns_providers_for_domain[target_domain]: dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") @@ -341,7 +341,7 @@ class AcmeHandler(object): acme_client, account_number, domain, - host, + target_domain, dns_provider_plugin, order, dns_provider.options, @@ -376,7 +376,7 @@ class AcmeHandler(object): for authz_record in authorizations: dns_challenges = authz_record.dns_challenge for dns_challenge in dns_challenges: - dns_providers = self.dns_providers_for_domain.get(authz_record.host) + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) for dns_provider in dns_providers: # Grab account number (For Route53) dns_provider_plugin = self.get_dns_provider( @@ -384,9 +384,9 @@ class AcmeHandler(object): ) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") - host_to_validate, _ = self.strip_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) - if authz_record.domain == authz_record.host: + if authz_record.domain == authz_record.target_domain: host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate) dns_provider_plugin.delete_txt_record( authz_record.change_id, @@ -410,20 +410,20 @@ class AcmeHandler(object): :return: """ for authz_record in authorizations: - dns_providers = self.dns_providers_for_domain.get(authz_record.host) + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) for dns_provider in dns_providers: # Grab account number (For Route53) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_challenges = authz_record.dns_challenge - host_to_validate, _ = self.strip_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) for dns_challenge in dns_challenges: - if authz_record.domain == authz_record.host: + if authz_record.domain == authz_record.target_domain: host_to_validate = dns_challenge.validation_domain_name(host_to_validate), try: dns_provider_plugin.delete_txt_record( @@ -455,7 +455,6 @@ class AcmeHandler(object): def get_cname(self, domain): """ :param domain: Domain name to look up a CNAME for. - :param record_type: Type of DNS record to lookup. :return: First CNAME target or False if no CNAME record exists. """ try: diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index cce97d32..89ca6ee1 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -97,7 +97,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True) mock_authz.authz = [] - mock_authz.host = "www.test.com" + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" mock_authz.authz.append(mock_authz_record) @@ -121,7 +121,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False) mock_authz.authz = [] - mock_authz.host = "www.test.com" + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" mock_authz.authz.append(mock_authz_record) From ca465e3c9eb06834e19c477ccd39b45f0ca4fd7e Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 29 Oct 2020 14:42:51 -0700 Subject: [PATCH 5/5] updating debug string with target_domain --- lemur/plugins/lemur_acme/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 0baa0478..07324f35 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -98,7 +98,7 @@ class AcmeHandler(object): order, dns_provider_options, ): - current_app.logger.debug("Starting DNS challenge for {0}".format(target_domain)) + current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.") change_ids = [] dns_challenges = self.get_dns_challenges(domain, order.authorizations)