From bb026b8b59879cdc159a696ccf173359e428250f Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 13 Aug 2018 14:22:59 -0700 Subject: [PATCH 1/4] Allow LetsEncrypt renewals and requesting certificates without specifying DNS provider --- .gitignore | 1 + lemur/certificates/models.py | 2 +- lemur/certificates/schemas.py | 2 + lemur/dns_providers/cli.py | 27 + lemur/dns_providers/models.py | 2 + lemur/dns_providers/service.py | 18 + lemur/manage.py | 2 + lemur/plugins/lemur_acme/dyn.py | 35 +- lemur/plugins/lemur_acme/plugin.py | 523 ++++++++++-------- lemur/plugins/lemur_acme/route53.py | 39 +- lemur/plugins/lemur_acme/tests/test_acme.py | 77 +-- lemur/schemas.py | 2 +- .../certificate/tracking.tpl.html | 13 +- requirements-dev.txt | 2 +- requirements-docs.txt | 8 +- requirements-tests.txt | 20 +- requirements.txt | 6 +- 17 files changed, 479 insertions(+), 300 deletions(-) create mode 100644 lemur/dns_providers/cli.py diff --git a/.gitignore b/.gitignore index 100161ae..97af00ca 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ *.db *.pid *.enc +*.env MANIFEST test.conf pip-log.txt diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index f5b60ad5..b337c755 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -101,7 +101,7 @@ class Certificate(db.Model): serial = Column(String(128)) cn = Column(String(128)) deleted = Column(Boolean, index=True) - dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='cascade'), nullable=True) + dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='CASCADE'), nullable=True) not_before = Column(ArrowType) not_after = Column(ArrowType) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index e88b6e73..e0d50cc6 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -10,6 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError from lemur.authorities.schemas import AuthorityNestedOutputSchema +from lemur.dns_providers.schemas import DnsProvidersNestedOutputSchema from lemur.common import validators, missing from lemur.common.fields import ArrowDateTime, Hex from lemur.common.schema import LemurInputSchema, LemurOutputSchema @@ -223,6 +224,7 @@ class CertificateOutputSchema(LemurOutputSchema): notifications = fields.Nested(NotificationNestedOutputSchema, many=True) replaces = fields.Nested(CertificateNestedOutputSchema, many=True) authority = fields.Nested(AuthorityNestedOutputSchema) + dns_provider = fields.Nested(DnsProvidersNestedOutputSchema) roles = fields.Nested(RoleNestedOutputSchema, many=True) endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[]) replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced') diff --git a/lemur/dns_providers/cli.py b/lemur/dns_providers/cli.py new file mode 100644 index 00000000..b111ca0e --- /dev/null +++ b/lemur/dns_providers/cli.py @@ -0,0 +1,27 @@ +from flask_script import Manager + +from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.dns_providers.service import get_all_dns_providers, set_domains +from lemur.extensions import metrics +from lemur.plugins.base import plugins + +manager = Manager(usage="Iterates through all DNS providers and sets DNS zones in the database.") + + +@manager.command +def get_all_zones(): + """ + Retrieves all DNS providers from the database. Refreshes the zones associated with each DNS provider + """ + print("[+] Starting dns provider zone lookup and configuration.") + dns_providers = get_all_dns_providers() + acme_plugin = plugins.get("acme-issuer") + + for dns_provider in dns_providers: + zones = acme_plugin.get_all_zones(dns_provider) + set_domains(dns_provider, zones) + + status = SUCCESS_METRIC_STATUS + + metrics.send('get_all_zones', 'counter', 1, metric_tags={'status': status}) + print("[+] Done with dns provider zone lookup and configuration.") diff --git a/lemur/dns_providers/models.py b/lemur/dns_providers/models.py index 0cf41730..d48cd0d1 100644 --- a/lemur/dns_providers/models.py +++ b/lemur/dns_providers/models.py @@ -1,5 +1,6 @@ from sqlalchemy import Column, Integer, String, text, Text from sqlalchemy.dialects.postgresql import JSON +from sqlalchemy.orm import relationship from sqlalchemy_utils import ArrowType from lemur.database import db @@ -22,6 +23,7 @@ class DnsProvider(db.Model): status = Column(String(length=128), nullable=True) options = Column(JSON, nullable=True) domains = Column(JSON, nullable=True) + certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id') def __init__(self, name, description, provider_type, credentials): self.name = name diff --git a/lemur/dns_providers/service.py b/lemur/dns_providers/service.py index c96b864a..bf50bba1 100644 --- a/lemur/dns_providers/service.py +++ b/lemur/dns_providers/service.py @@ -22,6 +22,15 @@ def get(dns_provider_id): return provider +def get_all_dns_providers(): + """ + Retrieves all dns providers within Lemur. + + :return: + """ + return DnsProvider.query.all() + + def get_friendly(dns_provider_id): """ Retrieves a dns provider by its lemur assigned ID. @@ -96,6 +105,15 @@ def get_types(): return provider_config +def set_domains(dns_provider, domains): + """ + Increments pending certificate attempt counter and updates it in the database. + """ + dns_provider.domains = domains + database.update(dns_provider) + return dns_provider + + def create(data): provider_name = data.get("name") diff --git a/lemur/manage.py b/lemur/manage.py index 919f70e4..6b1e1013 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -15,6 +15,7 @@ from flask_script import Manager, Command, Option, prompt_pass from flask_migrate import Migrate, MigrateCommand, stamp from flask_script.commands import ShowUrls, Clean, Server +from lemur.dns_providers.cli import manager as dns_provider_manager from lemur.sources.cli import manager as source_manager from lemur.policies.cli import manager as policy_manager from lemur.reporting.cli import manager as report_manager @@ -539,6 +540,7 @@ def main(): manager.add_command("report", report_manager) manager.add_command("policy", policy_manager) manager.add_command("pending_certs", pending_certificate_manager) + manager.add_command("dns_providers", dns_provider_manager) manager.run() diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 6ce117bf..9bab3a65 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -5,7 +5,7 @@ import dns.exception import dns.name import dns.query import dns.resolver -from dyn.tm.errors import DynectGetError +from dyn.tm.errors import DynectCreateError from dyn.tm.session import DynectSession from dyn.tm.zones import Node, Zone, get_all_zones from flask import current_app @@ -49,6 +49,7 @@ def wait_for_dns_change(change_id, account_number=None): break time.sleep(20) if not status: + # TODO: Delete associated DNS text record here raise Exception("Unable to query DNS token for fqdn {}.".format(fqdn)) return @@ -70,6 +71,15 @@ def get_zone_name(domain): return zone_name +def get_zones(account_number): + get_dynect_session() + zones = get_all_zones() + zone_list = [] + for zone in zones: + zone_list.append(zone.name) + return zone_list + + def create_txt_record(domain, token, account_number): get_dynect_session() zone_name = get_zone_name(domain) @@ -77,21 +87,20 @@ def create_txt_record(domain, token, account_number): node_name = '.'.join(domain.split('.')[:-zone_parts]) fqdn = "{0}.{1}".format(node_name, zone_name) zone = Zone(zone_name) + try: - # Delete all stale ACME TXT records - delete_acme_txt_records(domain) - except DynectGetError as e: - if ( - "No such zone." in e.message or - "Host is not in this zone" in e.message or - "Host not found in this zone" in e.message - ): - current_app.logger.debug("Unable to delete ACME TXT records. They probably don't exist yet: {}".format(e)) + zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5) + zone.publish() + current_app.logger.debug("TXT record created: {0}, token: {1}".format(fqdn, token)) + except DynectCreateError as e: + if "Cannot duplicate existing record data" in e.message: + current_app.logger.debug( + "Unable to add record. Domain: {}. Token: {}. " + "Record already exists: {}".format(domain, token, e), exc_info=True + ) else: raise - zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5) - zone.publish() - current_app.logger.debug("TXT record created: {0}".format(fqdn)) + change_id = (fqdn, token) return change_id diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 6d07c063..1dab1362 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -33,15 +33,6 @@ from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme import cloudflare, dyn, route53 -def find_dns_challenge(authorizations): - dns_challenges = [] - for authz in authorizations: - for combo in authz.body.challenges: - if isinstance(combo.chall, challenges.DNS01): - dns_challenges.append(combo) - return dns_challenges - - class AuthorizationRecord(object): def __init__(self, host, authz, dns_challenge, change_id): self.host = host @@ -50,192 +41,249 @@ class AuthorizationRecord(object): self.change_id = change_id -def maybe_remove_wildcard(host): - return host.replace("*.", "") +class AcmeHandler(object): + def __init__(self): + self.dns_providers_for_domain = {} + self.all_dns_providers = dns_provider_service.get_all_dns_providers() + def find_dns_challenge(self, authorizations): + dns_challenges = [] + for authz in authorizations: + for combo in authz.body.challenges: + if isinstance(combo.chall, challenges.DNS01): + dns_challenges.append(combo) + return dns_challenges -def maybe_add_extension(host, dns_provider_options): - if dns_provider_options and dns_provider_options.get("acme_challenge_extension"): - host = host + dns_provider_options.get("acme_challenge_extension") - return host + def maybe_remove_wildcard(self, host): + return host.replace("*.", "") + def maybe_add_extension(self, host, dns_provider_options): + if dns_provider_options and dns_provider_options.get("acme_challenge_extension"): + host = host + dns_provider_options.get("acme_challenge_extension") + return host -def start_dns_challenge(acme_client, account_number, host, dns_provider, order, dns_provider_options): - current_app.logger.debug("Starting DNS challenge for {0}".format(host)) + def start_dns_challenge(self, acme_client, account_number, host, dns_provider, order, dns_provider_options): + current_app.logger.debug("Starting DNS challenge for {0}".format(host)) - dns_challenges = find_dns_challenge(order.authorizations) - change_ids = [] + dns_challenges = self.find_dns_challenge(order.authorizations) + change_ids = [] - host_to_validate = maybe_remove_wildcard(host) - host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options) + host_to_validate = self.maybe_remove_wildcard(host) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) - for dns_challenge in find_dns_challenge(order.authorizations): - change_id = dns_provider.create_txt_record( - dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key), - account_number - ) - change_ids.append(change_id) - - return AuthorizationRecord( - host, - order.authorizations, - dns_challenges, - change_ids - ) - - -def complete_dns_challenge(acme_client, account_number, authz_record, dns_provider): - current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.authz[0].body.identifier.value)) - for change_id in authz_record.change_id: - dns_provider.wait_for_dns_change(change_id, account_number=account_number) - - for dns_challenge in authz_record.dns_challenge: - - response = dns_challenge.response(acme_client.client.net.key) - - verified = response.simple_verify( - dns_challenge.chall, - authz_record.host, - acme_client.client.net.key.public_key() - ) - - if not verified: - raise ValueError("Failed verification") - - time.sleep(5) - acme_client.answer_challenge(dns_challenge, response) - - -def request_certificate(acme_client, authorizations, csr, order): - for authorization in authorizations: - for authz in authorization.authz: - authorization_resource, _ = acme_client.poll(authz) - - deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) - - try: - orderr = acme_client.finalize_order(order, deadline) - except AcmeError: - current_app.logger.error("Unable to resolve Acme order: {}".format(order), exc_info=True) - raise - - pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, - orderr.fullchain_pem)).decode() - pem_certificate_chain = orderr.fullchain_pem[len(pem_certificate):].lstrip() - - current_app.logger.debug("{0} {1}".format(type(pem_certificate), type(pem_certificate_chain))) - return pem_certificate, pem_certificate_chain - - -def setup_acme_client(authority): - if not authority.options: - raise InvalidAuthority("Invalid authority. Options not set") - options = {} - - for option in json.loads(authority.options): - options[option["name"]] = option.get("value") - email = options.get('email', current_app.config.get('ACME_EMAIL')) - tel = options.get('telephone', current_app.config.get('ACME_TEL')) - directory_url = options.get('acme_url', current_app.config.get('ACME_DIRECTORY_URL')) - - existing_key = options.get('acme_private_key', current_app.config.get('ACME_PRIVATE_KEY')) - existing_regr = options.get('acme_regr', current_app.config.get('ACME_REGR')) - - if existing_key and existing_regr: - # Reuse the same account for each certificate issuance - key = jose.JWK.json_loads(existing_key) - regr = messages.RegistrationResource.json_loads(existing_regr) - current_app.logger.debug("Connecting with directory at {0}".format(directory_url)) - net = ClientNetwork(key, account=regr) - client = BackwardsCompatibleClientV2(net, key, directory_url) - return client, {} - else: - # Create an account for each certificate issuance - key = jose.JWKRSA(key=generate_private_key('RSA2048')) - - current_app.logger.debug("Connecting with directory at {0}".format(directory_url)) - - net = ClientNetwork(key, account=None) - client = BackwardsCompatibleClientV2(net, key, directory_url) - registration = client.new_account_and_tos(messages.NewRegistration.from_data(email=email)) - current_app.logger.debug("Connected: {0}".format(registration.uri)) - - return client, registration - - -def get_domains(options): - """ - Fetches all domains currently requested - :param options: - :return: - """ - current_app.logger.debug("Fetching domains") - - domains = [options['common_name']] - if options.get('extensions'): - for name in options['extensions']['sub_alt_names']['names']: - domains.append(name) - - current_app.logger.debug("Got these domains: {0}".format(domains)) - return domains - - -def get_authorizations(acme_client, order, order_info, dns_provider, dns_provider_options): - authorizations = [] - for domain in order_info.domains: - authz_record = start_dns_challenge(acme_client, order_info.account_number, domain, dns_provider, order, - dns_provider_options) - authorizations.append(authz_record) - return authorizations - - -def finalize_authorizations(acme_client, account_number, dns_provider, authorizations, dns_provider_options): - for authz_record in authorizations: - complete_dns_challenge(acme_client, account_number, authz_record, dns_provider) - for authz_record in authorizations: - dns_challenges = authz_record.dns_challenge - host_to_validate = maybe_remove_wildcard(authz_record.host) - host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options) - for dns_challenge in dns_challenges: - dns_provider.delete_txt_record( - authz_record.change_id, - account_number, + for dns_challenge in self.find_dns_challenge(order.authorizations): + change_id = dns_provider.create_txt_record( dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key) + dns_challenge.validation(acme_client.client.net.key), + account_number ) + change_ids.append(change_id) - return authorizations + return AuthorizationRecord( + host, + order.authorizations, + dns_challenges, + change_ids + ) + def complete_dns_challenge(self, acme_client, authz_record): + current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.authz[0].body.identifier.value)) + dns_providers = self.dns_providers_for_domain.get(authz_record.host) + if not dns_providers: + raise Exception("No DNS providers found for domain: {}".format(authz_record.host)) -def cleanup_dns_challenges(acme_client, account_number, dns_provider, authorizations, dns_provider_options): - """ - Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called - on an exception + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + for change_id in authz_record.change_id: + dns_provider_plugin.wait_for_dns_change(change_id, account_number=account_number) - :param acme_client: - :param account_number: - :param dns_provider: - :param authorizations: - :param dns_provider_options: - :return: - """ - for authz_record in authorizations: - dns_challenges = authz_record.dns_challenge - host_to_validate = maybe_remove_wildcard(authz_record.host) - host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options) - for dns_challenge in dns_challenges: - try: - dns_provider.delete_txt_record( - authz_record.change_id, - account_number, - dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key) + for dns_challenge in authz_record.dns_challenge: + response = dns_challenge.response(acme_client.client.net.key) + + verified = response.simple_verify( + dns_challenge.chall, + authz_record.host, + acme_client.client.net.key.public_key() ) - except Exception: - # If this fails, it's most likely because the record doesn't exist or we're not authorized to modify it. - pass + + if not verified: + raise ValueError("Failed verification") + + time.sleep(5) + acme_client.answer_challenge(dns_challenge, response) + + def request_certificate(self, acme_client, authorizations, order): + for authorization in authorizations: + for authz in authorization.authz: + authorization_resource, _ = acme_client.poll(authz) + + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + + try: + orderr = acme_client.finalize_order(order, deadline) + except AcmeError: + current_app.logger.error("Unable to resolve Acme order: {}".format(order), exc_info=True) + raise + + pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, + orderr.fullchain_pem)).decode() + pem_certificate_chain = orderr.fullchain_pem[len(pem_certificate):].lstrip() + + current_app.logger.debug("{0} {1}".format(type(pem_certificate), type(pem_certificate_chain))) + return pem_certificate, pem_certificate_chain + + def setup_acme_client(self, authority): + if not authority.options: + raise InvalidAuthority("Invalid authority. Options not set") + options = {} + + for option in json.loads(authority.options): + options[option["name"]] = option.get("value") + email = options.get('email', current_app.config.get('ACME_EMAIL')) + tel = options.get('telephone', current_app.config.get('ACME_TEL')) + directory_url = options.get('acme_url', current_app.config.get('ACME_DIRECTORY_URL')) + + existing_key = options.get('acme_private_key', current_app.config.get('ACME_PRIVATE_KEY')) + existing_regr = options.get('acme_regr', current_app.config.get('ACME_REGR')) + + if existing_key and existing_regr: + # Reuse the same account for each certificate issuance + key = jose.JWK.json_loads(existing_key) + regr = messages.RegistrationResource.json_loads(existing_regr) + current_app.logger.debug("Connecting with directory at {0}".format(directory_url)) + net = ClientNetwork(key, account=regr) + client = BackwardsCompatibleClientV2(net, key, directory_url) + return client, {} + else: + # Create an account for each certificate issuance + key = jose.JWKRSA(key=generate_private_key('RSA2048')) + + current_app.logger.debug("Connecting with directory at {0}".format(directory_url)) + + net = ClientNetwork(key, account=None) + client = BackwardsCompatibleClientV2(net, key, directory_url) + registration = client.new_account_and_tos(messages.NewRegistration.from_data(email=email)) + current_app.logger.debug("Connected: {0}".format(registration.uri)) + + return client, registration + + def get_domains(self, options): + """ + Fetches all domains currently requested + :param options: + :return: + """ + current_app.logger.debug("Fetching domains") + + domains = [options['common_name']] + if options.get('extensions'): + for name in options['extensions']['sub_alt_names']['names']: + domains.append(name) + + current_app.logger.debug("Got these domains: {0}".format(domains)) + return domains + + def get_authorizations(self, acme_client, order, order_info): + authorizations = [] + + for domain in order_info.domains: + if not self.dns_providers_for_domain.get(domain): + raise Exception("No DNS providers found for domain: {}".format(domain)) + for dns_provider in self.dns_providers_for_domain[domain]: + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + authz_record = self.start_dns_challenge(acme_client, account_number, domain, + dns_provider_plugin, + order, + dns_provider.options) + authorizations.append(authz_record) + return authorizations + + def autodetect_dns_providers(self, domain): + """ + Get DNS providers associated with a domain when it has not been provided for certificate creation. + :param domain: + :return: dns_providers: List of DNS providers that have the correct zone. + """ + self.dns_providers_for_domain[domain] = [] + for dns_provider in self.all_dns_providers: + for name in dns_provider.domains: + if domain.endswith(name): + self.dns_providers_for_domain[domain].append(dns_provider) + return self.dns_providers_for_domain + + def finalize_authorizations(self, acme_client, authorizations): + for authz_record in authorizations: + self.complete_dns_challenge(acme_client, authz_record) + for authz_record in authorizations: + dns_challenges = authz_record.dns_challenge + for dns_challenge in dns_challenges: + dns_providers = self.dns_providers_for_domain.get(authz_record.host) + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + host_to_validate = self.maybe_remove_wildcard(authz_record.host) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) + dns_provider_plugin.delete_txt_record( + authz_record.change_id, + account_number, + dns_challenge.validation_domain_name(host_to_validate), + dns_challenge.validation(acme_client.client.net.key) + ) + + return authorizations + + def cleanup_dns_challenges(self, acme_client, authorizations): + """ + Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called + on an exception + + :param acme_client: + :param account_number: + :param dns_provider: + :param authorizations: + :param dns_provider_options: + :return: + """ + for authz_record in authorizations: + dns_providers = self.dns_providers_for_domain.get(authz_record.host) + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_challenges = authz_record.dns_challenge + host_to_validate = self.maybe_remove_wildcard(authz_record.host) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) + for dns_challenge in dns_challenges: + try: + dns_provider.delete_txt_record( + authz_record.change_id, + account_number, + dns_challenge.validation_domain_name(host_to_validate), + dns_challenge.validation(acme_client.client.net.key) + ) + except Exception as e: + # If this fails, it's most likely because the record doesn't exist (It was already cleaned up) + # or we're not authorized to modify it. + pass + + def get_dns_provider(self, type): + provider_types = { + 'cloudflare': cloudflare, + 'dyn': dyn, + 'route53': route53, + } + provider = provider_types.get(type) + if not provider: + raise UnknownProvider("No such DNS provider: {}".format(type)) + return provider class ACMEIssuerPlugin(IssuerPlugin): @@ -279,6 +327,7 @@ class ACMEIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) + self.acme = AcmeHandler() def get_dns_provider(self, type): provider_types = { @@ -291,22 +340,40 @@ class ACMEIssuerPlugin(IssuerPlugin): raise UnknownProvider("No such DNS provider: {}".format(type)) return provider + def get_all_zones(self, dns_provider): + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + return dns_provider_plugin.get_zones(account_number=account_number) + def get_ordered_certificate(self, pending_cert): - acme_client, registration = setup_acme_client(pending_cert.authority) + acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) - dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) - dns_provider_options = dns_provider.options - dns_provider_type = self.get_dns_provider(dns_provider.provider_type) + if pending_cert.dns_provider_id: + dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) + + for domain in order_info.domains: + # Currently, we only support specifying one DNS provider per certificate, even if that + # certificate has multiple SANs that may belong to different providers. + self.acme.dns_providers_for_domain[domain] = [dns_provider] + else: + for domain in order_info.domains: + self.acme.autodetect_dns_providers(domain) + try: - authorizations = get_authorizations( - acme_client, order_info.account_number, order_info.domains, dns_provider_type, dns_provider_options) + order = acme_client.new_order(pending_cert.csr) + except WildcardUnsupportedError: + raise Exception("The currently selected ACME CA endpoint does" + " not support issuing wildcard certificates.") + try: + authorizations = self.acme.get_authorizations(acme_client, order, order_info) except ClientError: current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert.name), exc_info=True) return False - authorizations = finalize_authorizations( - acme_client, order_info.account_number, dns_provider_type, authorizations, dns_provider_options) - pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr) + authorizations = self.acme.finalize_authorizations(acme_client, authorizations) + pem_certificate, pem_certificate_chain = self.acme.request_certificate( + acme_client, authorizations, order) cert = { 'body': "\n".join(str(pem_certificate).splitlines()), 'chain': "\n".join(str(pem_certificate_chain).splitlines()), @@ -319,28 +386,32 @@ class ACMEIssuerPlugin(IssuerPlugin): certs = [] for pending_cert in pending_certs: try: - acme_client, registration = setup_acme_client(pending_cert.authority) + acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) - dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) - dns_provider_options = dns_provider.options - dns_provider_type = self.get_dns_provider(dns_provider.provider_type) + if pending_cert.dns_provider_id: + dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) + + for domain in order_info.domains: + # Currently, we only support specifying one DNS provider per certificate, even if that + # certificate has multiple SANs that may belong to different providers. + self.acme.dns_providers_for_domain[domain] = [dns_provider] + else: + for domain in order_info.domains: + self.acme.autodetect_dns_providers(domain) + try: order = acme_client.new_order(pending_cert.csr) except WildcardUnsupportedError: raise Exception("The currently selected ACME CA endpoint does" " not support issuing wildcard certificates.") - authorizations = get_authorizations(acme_client, order, order_info, dns_provider_type, - dns_provider_options) + authorizations = self.acme.get_authorizations(acme_client, order, order_info) pending.append({ "acme_client": acme_client, - "account_number": order_info.account_number, - "dns_provider_type": dns_provider_type, "authorizations": authorizations, "pending_cert": pending_cert, "order": order, - "dns_provider_options": dns_provider_options, }) except (ClientError, ValueError, Exception) as e: current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) @@ -352,17 +423,13 @@ class ACMEIssuerPlugin(IssuerPlugin): for entry in pending: try: - entry["authorizations"] = finalize_authorizations( + entry["authorizations"] = self.acme.finalize_authorizations( entry["acme_client"], - entry["account_number"], - entry["dns_provider_type"], entry["authorizations"], - entry["dns_provider_options"], ) - pem_certificate, pem_certificate_chain = request_certificate( + pem_certificate, pem_certificate_chain = self.acme.request_certificate( entry["acme_client"], entry["authorizations"], - entry["pending_cert"].csr, entry["order"] ) @@ -383,12 +450,9 @@ class ACMEIssuerPlugin(IssuerPlugin): "last_error": e, }) # Ensure DNS records get deleted - cleanup_dns_challenges( + self.acme.cleanup_dns_challenges( entry["acme_client"], - entry["account_number"], - entry["dns_provider_type"], entry["authorizations"], - entry["dns_provider_options"], ) return certs @@ -402,21 +466,25 @@ class ACMEIssuerPlugin(IssuerPlugin): """ authority = issuer_options.get('authority') create_immediately = issuer_options.get('create_immediately', False) - acme_client, registration = setup_acme_client(authority) - dns_provider = issuer_options.get('dns_provider') - dns_provider_options = dns_provider.options - if not dns_provider: - raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.") - credentials = json.loads(dns_provider.credentials) + acme_client, registration = self.acme.setup_acme_client(authority) + dns_provider = issuer_options.get('dns_provider', {}) + # TODO: IF NOT DNS PROVIDER, AUTODISCOVER + if dns_provider: + dns_provider_options = dns_provider.options + credentials = json.loads(dns_provider.credentials) + current_app.logger.debug("Using DNS provider: {0}".format(dns_provider.provider_type)) + dns_provider_plugin = __import__(dns_provider.provider_type, globals(), locals(), [], 1) + account_number = credentials.get("account_id") + if dns_provider.provider_type == 'route53' and not account_number: + error = "Route53 DNS Provider {} does not have an account number configured.".format(dns_provider.name) + current_app.logger.error(error) + raise InvalidConfiguration(error) + else: + dns_provider = {} + dns_provider_options = None + account_number = None - current_app.logger.debug("Using DNS provider: {0}".format(dns_provider.provider_type)) - dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1) - account_number = credentials.get("account_id") - if dns_provider.provider_type == 'route53' and not account_number: - error = "Route53 DNS Provider {} does not have an account number configured.".format(dns_provider.name) - current_app.logger.error(error) - raise InvalidConfiguration(error) - domains = get_domains(issuer_options) + domains = self.acme.get_domains(issuer_options) if not create_immediately: # Create pending authorizations that we'll need to do the creation authz_domains = [] @@ -426,14 +494,15 @@ class ACMEIssuerPlugin(IssuerPlugin): else: authz_domains.append(d.value) - dns_authorization = authorization_service.create(account_number, authz_domains, dns_provider.provider_type) + dns_authorization = authorization_service.create(account_number, authz_domains, + dns_provider.get("provider_type")) # Return id of the DNS Authorization return None, None, dns_authorization.id - authorizations = get_authorizations(acme_client, account_number, domains, dns_provider_type, + authorizations = self.acme.get_authorizations(acme_client, account_number, domains, dns_provider_plugin, dns_provider_options) - finalize_authorizations(acme_client, account_number, dns_provider_type, authorizations, dns_provider_options) - pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, csr) + self.acme.finalize_authorizations(acme_client, account_number, dns_provider_plugin, authorizations, dns_provider_options) + pem_certificate, pem_certificate_chain = self.acme.request_certificate(acme_client, authorizations, csr) # TODO add external ID (if possible) return pem_certificate, pem_certificate_chain, None diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index 94823d80..3b6c5b32 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -31,6 +31,16 @@ def find_zone_id(domain, client=None): return zones[0][1] +@sts_client('route53') +def get_zones(client=None): + paginator = client.get_paginator("list_hosted_zones") + zones = [] + for page in paginator.paginate(): + for zone in page["HostedZones"]: + zones.append(zone["Name"][:-1]) # We need [:-1] to strip out the trailing dot. + return zones + + @sts_client('route53') def change_txt_record(action, zone_id, domain, value, client=None): current_txt_records = [] @@ -50,7 +60,13 @@ def change_txt_record(action, zone_id, domain, value, client=None): raise # For some reason TXT records need to be # manually quoted. - current_txt_records.append({"Value": '"{}"'.format(value)}) + seen = False + for record in current_txt_records: + for k, v in record.items(): + if '"{}"'.format(value) == v: + seen = True + if not seen: + current_txt_records.append({"Value": '"{}"'.format(value)}) if action == "DELETE" and len(current_txt_records) > 1: # If we want to delete one record out of many, we'll update the record to not include the deleted value instead. @@ -95,10 +111,17 @@ def create_txt_record(host, value, account_number): def delete_txt_record(change_ids, account_number, host, value): for change_id in change_ids: zone_id, _ = change_id - change_txt_record( - "DELETE", - zone_id, - host, - value, - account_number=account_number - ) + try: + change_txt_record( + "DELETE", + zone_id, + host, + value, + account_number=account_number + ) + except Exception as e: + if "but it was not found" in e.response.get("Error", {}).get("Message"): + # We tried to delete a record that doesn't exist. We'll ignore this error. + pass + else: + raise diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 69f4e438..a9b82698 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -7,8 +7,15 @@ from lemur.plugins.lemur_acme import plugin class TestAcme(unittest.TestCase): - def setUp(self): + @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service') + def setUp(self, mock_dns_provider_service): self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() + self.acme = plugin.AcmeHandler() + mock_dns_provider = Mock() + mock_dns_provider.name = "cloudflare" + mock_dns_provider.credentials = "{}" + mock_dns_provider.provider_type = "cloudflare" + self.acme.dns_providers_for_domain = {"www.test.com": [mock_dns_provider], "test.fakedomain.net": [mock_dns_provider]} @patch('lemur.plugins.lemur_acme.plugin.len', return_value=1) def test_find_dns_challenge(self, mock_len): @@ -22,7 +29,7 @@ class TestAcme(unittest.TestCase): mock_entry = Mock() mock_entry.chall = c mock_authz.body.resolved_combinations.append(mock_entry) - result = yield plugin.find_dns_challenge(mock_authz) + result = yield self.acme.find_dns_challenge(mock_authz) self.assertEqual(result, mock_entry) def test_authz_record(self): @@ -32,7 +39,7 @@ class TestAcme(unittest.TestCase): @patch('acme.client.Client') @patch('lemur.plugins.lemur_acme.plugin.current_app') @patch('lemur.plugins.lemur_acme.plugin.len', return_value=1) - @patch('lemur.plugins.lemur_acme.plugin.find_dns_challenge') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge') def test_start_dns_challenge(self, mock_find_dns_challenge, mock_len, mock_app, mock_acme): assert mock_len mock_order = Mock() @@ -42,7 +49,7 @@ class TestAcme(unittest.TestCase): mock_entry = MagicMock() from acme import challenges c = challenges.DNS01() - mock_entry.chall = c + mock_entry.chall = TestAcme.test_complete_dns_challenge_fail mock_authz.body.resolved_combinations.append(mock_entry) mock_acme.request_domain_challenges = Mock(return_value=mock_authz) mock_dns_provider = Mock() @@ -52,19 +59,20 @@ class TestAcme(unittest.TestCase): iterable = mock_find_dns_challenge.return_value iterator = iter(values) iterable.__iter__.return_value = iterator - result = plugin.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order, {}) + result = self.acme.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order, {}) self.assertEqual(type(result), plugin.AuthorizationRecord) @patch('acme.client.Client') @patch('lemur.plugins.lemur_acme.plugin.current_app') - def test_complete_dns_challenge_success(self, mock_current_app, mock_acme): + @patch('lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change') + def test_complete_dns_challenge_success(self, mock_wait_for_dns_change, mock_current_app, mock_acme): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) - mock_authz = Mock() mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True) mock_authz.authz = [] + mock_authz.host = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" mock_authz.authz.append(mock_authz_record) @@ -73,11 +81,12 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge = [] dns_challenge = Mock() mock_authz.dns_challenge.append(dns_challenge) - plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider) + self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch('acme.client.Client') @patch('lemur.plugins.lemur_acme.plugin.current_app') - def test_complete_dns_challenge_fail(self, mock_current_app, mock_acme): + @patch('lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change') + def test_complete_dns_challenge_fail(self, mock_wait_for_dns_change, mock_current_app, mock_acme): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -85,6 +94,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False) mock_authz.authz = [] + mock_authz.host = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" mock_authz.authz.append(mock_authz_record) @@ -95,13 +105,13 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.append(dns_challenge) self.assertRaises( ValueError, - plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider) + self.acme.complete_dns_challenge(mock_acme, mock_authz) ) @patch('acme.client.Client') @patch('OpenSSL.crypto', return_value="mock_cert") @patch('josepy.util.ComparableX509') - @patch('lemur.plugins.lemur_acme.plugin.find_dns_challenge') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge') @patch('lemur.plugins.lemur_acme.plugin.current_app') def test_request_certificate(self, mock_current_app, mock_find_dns_challenge, mock_jose, mock_crypto, mock_acme): mock_cert_response = Mock() @@ -115,13 +125,13 @@ class TestAcme(unittest.TestCase): mock_acme.fetch_chain = Mock(return_value="mock_chain") mock_crypto.dump_certificate = Mock(return_value=b'chain') mock_order = Mock() - plugin.request_certificate(mock_acme, [], "mock_csr", mock_order) + self.acme.request_certificate(mock_acme, [], mock_order) def test_setup_acme_client_fail(self): mock_authority = Mock() mock_authority.options = [] with self.assertRaises(Exception): - plugin.setup_acme_client(mock_authority) + self.acme.setup_acme_client(mock_authority) @patch('lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2') @patch('lemur.plugins.lemur_acme.plugin.current_app') @@ -135,7 +145,7 @@ class TestAcme(unittest.TestCase): mock_client.agree_to_tos = Mock(return_value=True) mock_acme.return_value = mock_client mock_current_app.config = {} - result_client, result_registration = plugin.setup_acme_client(mock_authority) + result_client, result_registration = self.acme.setup_acme_client(mock_authority) assert result_client assert result_registration @@ -144,7 +154,7 @@ class TestAcme(unittest.TestCase): options = { "common_name": "test.netflix.net" } - result = plugin.get_domains(options) + result = self.acme.get_domains(options) self.assertEqual(result, [options["common_name"]]) @patch('lemur.plugins.lemur_acme.plugin.current_app') @@ -160,10 +170,10 @@ class TestAcme(unittest.TestCase): } } } - result = plugin.get_domains(options) + result = self.acme.get_domains(options) self.assertEqual(result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"]) - @patch('lemur.plugins.lemur_acme.plugin.start_dns_challenge', return_value="test") + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge', return_value="test") def test_get_authorizations(self, mock_start_dns_challenge): mock_order = Mock() mock_order.body.identifiers = [] @@ -172,10 +182,10 @@ class TestAcme(unittest.TestCase): mock_order_info = Mock() mock_order_info.account_number = 1 mock_order_info.domains = ["test.fakedomain.net"] - result = plugin.get_authorizations("acme_client", mock_order, mock_order_info, "dns_provider", {}) + result = self.acme.get_authorizations("acme_client", mock_order, mock_order_info) self.assertEqual(result, ["test"]) - @patch('lemur.plugins.lemur_acme.plugin.complete_dns_challenge', return_value="test") + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge', return_value="test") def test_finalize_authorizations(self, mock_complete_dns_challenge): mock_authz = [] mock_authz_record = MagicMock() @@ -188,7 +198,7 @@ class TestAcme(unittest.TestCase): mock_dns_provider.delete_txt_record = Mock() mock_acme_client = Mock() - result = plugin.finalize_authorizations(mock_acme_client, "account_number", mock_dns_provider, mock_authz, {}) + result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) self.assertEqual(result, mock_authz) @patch('lemur.plugins.lemur_acme.plugin.current_app') @@ -210,7 +220,8 @@ class TestAcme(unittest.TestCase): @patch('lemur.plugins.lemur_acme.plugin.current_app') @patch('lemur.plugins.lemur_acme.dyn.current_app') @patch('lemur.plugins.lemur_acme.cloudflare.current_app') - def test_get_dns_provider(self, mock_current_app_cloudflare, mock_current_app_dyn, mock_current_app): + @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service') + def test_get_dns_provider(self, mock_dns_provider_service, mock_current_app_cloudflare, mock_current_app_dyn, mock_current_app): provider = plugin.ACMEIssuerPlugin() route53 = provider.get_dns_provider("route53") assert route53 @@ -219,13 +230,13 @@ class TestAcme(unittest.TestCase): dyn = provider.get_dns_provider("dyn") assert dyn - @patch('lemur.plugins.lemur_acme.plugin.setup_acme_client') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client') @patch('lemur.plugins.lemur_acme.plugin.current_app') @patch('lemur.plugins.lemur_acme.plugin.authorization_service') @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service') - @patch('lemur.plugins.lemur_acme.plugin.get_authorizations') - @patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations') - @patch('lemur.plugins.lemur_acme.plugin.request_certificate') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate') def test_get_ordered_certificate( self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations, mock_dns_provider_service, mock_authorization_service, mock_current_app, mock_acme): @@ -248,13 +259,13 @@ class TestAcme(unittest.TestCase): } ) - @patch('lemur.plugins.lemur_acme.plugin.setup_acme_client') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client') @patch('lemur.plugins.lemur_acme.plugin.current_app') @patch('lemur.plugins.lemur_acme.plugin.authorization_service') @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service') - @patch('lemur.plugins.lemur_acme.plugin.get_authorizations') - @patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations') - @patch('lemur.plugins.lemur_acme.plugin.request_certificate') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate') def test_get_ordered_certificates( self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations, mock_dns_provider_service, mock_authorization_service, mock_current_app, mock_acme): @@ -275,12 +286,12 @@ class TestAcme(unittest.TestCase): self.assertEqual(result[0]['cert'], {'body': 'pem_certificate', 'chain': 'chain', 'external_id': '1'}) self.assertEqual(result[1]['cert'], {'body': 'pem_certificate', 'chain': 'chain', 'external_id': '2'}) - @patch('lemur.plugins.lemur_acme.plugin.setup_acme_client') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client') @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service') @patch('lemur.plugins.lemur_acme.plugin.current_app') - @patch('lemur.plugins.lemur_acme.plugin.get_authorizations') - @patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations') - @patch('lemur.plugins.lemur_acme.plugin.request_certificate') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations') + @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate') @patch('lemur.plugins.lemur_acme.plugin.authorization_service') def test_create_certificate(self, mock_authorization_service, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations, diff --git a/lemur/schemas.py b/lemur/schemas.py index ffdfe66f..4242c399 100644 --- a/lemur/schemas.py +++ b/lemur/schemas.py @@ -78,7 +78,7 @@ def fetch_objects(model, data, many=False): items = model.query.filter(getattr(model, attr).in_(values)).all() found = [getattr(i, attr) for i in items] diff = set(values).symmetric_difference(set(found)) - + AssociatedDnsProviderSchema if diff: raise ValidationError('Unable to locate {model} with {attr} {diff}'.format( model=model, diff --git a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html index fb74d208..9410f42d 100644 --- a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html @@ -107,6 +107,17 @@ +
+ +
+ The selected authority uses the ACME protocol and works differently than other authorities. + Your request will initially be created under the "pending certificates" section. Lemur will attempt to create the certificate for you, + and move the final certificate to the "certificates" section. Lemur performs validation by writing a DNS text record. You may choose a specific DNS provider, + or allow Lemur to automatically detect the correct provider for you. Requests may take up to ten minutes. +
+
diff --git a/requirements-dev.txt b/requirements-dev.txt index 9a3cd613..cb9be135 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # aspy.yaml==1.1.1 # via pre-commit cached-property==1.4.3 # via pre-commit -certifi==2018.4.16 # via requests +certifi==2018.8.13 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 diff --git a/requirements-docs.txt b/requirements-docs.txt index d2f8cb36..6361030c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,9 +15,9 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.66 -botocore==1.10.66 -certifi==2018.4.16 +boto3==1.7.75 +botocore==1.10.75 +certifi==2018.8.13 cffi==1.11.5 chardet==3.0.4 click==6.7 @@ -50,7 +50,7 @@ lockfile==0.12.2 mako==1.0.7 markupsafe==1.0 marshmallow-sqlalchemy==0.14.0 -marshmallow==2.15.3 +marshmallow==2.15.4 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==17.1 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index eeda5e13..38db62ae 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,10 +8,10 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.71 # via moto +boto3==1.7.75 # via moto boto==2.49.0 # via moto -botocore==1.10.71 # via boto3, moto, s3transfer -certifi==2018.4.16 # via requests +botocore==1.10.75 # via boto3, moto, s3transfer +certifi==2018.8.13 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==6.7 # via flask @@ -19,12 +19,14 @@ cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.3 # via moto docker-pycreds==0.3.0 # via docker -docker==3.4.1 # via moto +docker==3.5.0 # via moto docutils==0.14 # via botocore +ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==0.8.17 +faker==0.9.0 flask==1.0.2 # via pytest-flask freezegun==0.3.10 +future==0.16.0 # via python-jose idna==2.7 # via cryptography, requests itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto @@ -34,25 +36,27 @@ jsonpickle==0.9.6 # via aws-xray-sdk markupsafe==1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.3.0 # via pytest -moto==1.3.3 +moto==1.3.4 nose==1.3.7 pbr==4.2.0 # via mock pluggy==0.7.1 # via pytest py==1.5.4 # via pytest pyaml==17.12.1 # via moto pycparser==2.18 # via cffi +pycryptodome==3.6.5 # via python-jose pyflakes==2.0.0 pytest-flask==0.10.0 pytest-mock==1.10.0 pytest==3.7.1 -python-dateutil==2.6.1 # via botocore, faker, freezegun, moto +python-dateutil==2.7.3 # via botocore, faker, freezegun, moto +python-jose==2.0.2 # via moto pytz==2018.5 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.19.1 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 -six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client +six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.23 # via requests websocket-client==0.48.0 # via docker diff --git a/requirements.txt b/requirements.txt index bb566c62..47576f03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,9 +13,9 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.71 -botocore==1.10.71 # via boto3, s3transfer -certifi==2018.4.16 +boto3==1.7.75 +botocore==1.10.75 # via boto3, s3transfer +certifi==2018.8.13 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==6.7 # via flask From 5829794d82d140eefd46cef55e52f6fb0e81d283 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 13 Aug 2018 14:25:54 -0700 Subject: [PATCH 2/4] typo fix --- lemur/plugins/lemur_acme/plugin.py | 2 +- lemur/schemas.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 1dab1362..caa4ca45 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -468,7 +468,7 @@ class ACMEIssuerPlugin(IssuerPlugin): create_immediately = issuer_options.get('create_immediately', False) acme_client, registration = self.acme.setup_acme_client(authority) dns_provider = issuer_options.get('dns_provider', {}) - # TODO: IF NOT DNS PROVIDER, AUTODISCOVER + if dns_provider: dns_provider_options = dns_provider.options credentials = json.loads(dns_provider.credentials) diff --git a/lemur/schemas.py b/lemur/schemas.py index 4242c399..ffdfe66f 100644 --- a/lemur/schemas.py +++ b/lemur/schemas.py @@ -78,7 +78,7 @@ def fetch_objects(model, data, many=False): items = model.query.filter(getattr(model, attr).in_(values)).all() found = [getattr(i, attr) for i in items] diff = set(values).symmetric_difference(set(found)) - AssociatedDnsProviderSchema + if diff: raise ValidationError('Unable to locate {model} with {attr} {diff}'.format( model=model, From cc836433fb0e20da9acbddf292d4f0656743f499 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 13 Aug 2018 15:06:16 -0700 Subject: [PATCH 3/4] formatting --- lemur/plugins/lemur_acme/plugin.py | 5 +++-- lemur/plugins/lemur_acme/tests/conftest.py | 2 +- lemur/plugins/lemur_acme/tests/test_acme.py | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index caa4ca45..1604fcdc 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -500,8 +500,9 @@ class ACMEIssuerPlugin(IssuerPlugin): return None, None, dns_authorization.id authorizations = self.acme.get_authorizations(acme_client, account_number, domains, dns_provider_plugin, - dns_provider_options) - self.acme.finalize_authorizations(acme_client, account_number, dns_provider_plugin, authorizations, dns_provider_options) + dns_provider_options) + self.acme.finalize_authorizations(acme_client, account_number, dns_provider_plugin, authorizations, + dns_provider_options) pem_certificate, pem_certificate_chain = self.acme.request_certificate(acme_client, authorizations, csr) # TODO add external ID (if possible) return pem_certificate, pem_certificate_chain, None diff --git a/lemur/plugins/lemur_acme/tests/conftest.py b/lemur/plugins/lemur_acme/tests/conftest.py index 0e1cd89f..8b137891 100644 --- a/lemur/plugins/lemur_acme/tests/conftest.py +++ b/lemur/plugins/lemur_acme/tests/conftest.py @@ -1 +1 @@ -from lemur.tests.conftest import * # noqa + diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index a9b82698..0c406627 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -15,7 +15,8 @@ class TestAcme(unittest.TestCase): mock_dns_provider.name = "cloudflare" mock_dns_provider.credentials = "{}" mock_dns_provider.provider_type = "cloudflare" - self.acme.dns_providers_for_domain = {"www.test.com": [mock_dns_provider], "test.fakedomain.net": [mock_dns_provider]} + self.acme.dns_providers_for_domain = {"www.test.com": [mock_dns_provider], + "test.fakedomain.net": [mock_dns_provider]} @patch('lemur.plugins.lemur_acme.plugin.len', return_value=1) def test_find_dns_challenge(self, mock_len): @@ -221,7 +222,8 @@ class TestAcme(unittest.TestCase): @patch('lemur.plugins.lemur_acme.dyn.current_app') @patch('lemur.plugins.lemur_acme.cloudflare.current_app') @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service') - def test_get_dns_provider(self, mock_dns_provider_service, mock_current_app_cloudflare, mock_current_app_dyn, mock_current_app): + def test_get_dns_provider(self, mock_dns_provider_service, mock_current_app_cloudflare, mock_current_app_dyn, + mock_current_app): provider = plugin.ACMEIssuerPlugin() route53 = provider.get_dns_provider("route53") assert route53 From 1a5abe65509e22a18e393c0dfea747e7a7b35594 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 13 Aug 2018 15:11:57 -0700 Subject: [PATCH 4/4] fix lint --- lemur/plugins/lemur_acme/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/conftest.py b/lemur/plugins/lemur_acme/tests/conftest.py index 8b137891..e69de29b 100644 --- a/lemur/plugins/lemur_acme/tests/conftest.py +++ b/lemur/plugins/lemur_acme/tests/conftest.py @@ -1 +0,0 @@ -