Allow LetsEncrypt renewals and requesting certificates without specifying DNS provider

.gitignore

@@ -10,6 +10,7 @@
 *.db
 *.pid
 *.enc
+*.env
 MANIFEST
 test.conf
 pip-log.txt

lemur/certificates/models.py

@@ -101,7 +101,7 @@ class Certificate(db.Model):
     serial = Column(String(128))
     cn = Column(String(128))
     deleted = Column(Boolean, index=True)
-    dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='cascade'), nullable=True)
+    dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='CASCADE'), nullable=True)
 
     not_before = Column(ArrowType)
     not_after = Column(ArrowType)

lemur/certificates/schemas.py

@@ -10,6 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load
 from marshmallow.exceptions import ValidationError
 
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
+from lemur.dns_providers.schemas import DnsProvidersNestedOutputSchema
 from lemur.common import validators, missing
 from lemur.common.fields import ArrowDateTime, Hex
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
@@ -223,6 +224,7 @@ class CertificateOutputSchema(LemurOutputSchema):
     notifications = fields.Nested(NotificationNestedOutputSchema, many=True)
     replaces = fields.Nested(CertificateNestedOutputSchema, many=True)
     authority = fields.Nested(AuthorityNestedOutputSchema)
+    dns_provider = fields.Nested(DnsProvidersNestedOutputSchema)
     roles = fields.Nested(RoleNestedOutputSchema, many=True)
     endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
     replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')

lemur/dns_providers/cli.py

@@ -0,0 +1,27 @@
+from flask_script import Manager
+
+from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.dns_providers.service import get_all_dns_providers, set_domains
+from lemur.extensions import metrics
+from lemur.plugins.base import plugins
+
+manager = Manager(usage="Iterates through all DNS providers and sets DNS zones in the database.")
+
+
+@manager.command
+def get_all_zones():
+    """
+    Retrieves all DNS providers from the database. Refreshes the zones associated with each DNS provider
+    """
+    print("[+] Starting dns provider zone lookup and configuration.")
+    dns_providers = get_all_dns_providers()
+    acme_plugin = plugins.get("acme-issuer")
+
+    for dns_provider in dns_providers:
+        zones = acme_plugin.get_all_zones(dns_provider)
+        set_domains(dns_provider, zones)
+
+    status = SUCCESS_METRIC_STATUS
+
+    metrics.send('get_all_zones', 'counter', 1, metric_tags={'status': status})
+    print("[+] Done with dns provider zone lookup and configuration.")

lemur/dns_providers/models.py

@@ -1,5 +1,6 @@
 from sqlalchemy import Column, Integer, String, text, Text
 from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy.orm import relationship
 from sqlalchemy_utils import ArrowType
 
 from lemur.database import db
@@ -22,6 +23,7 @@ class DnsProvider(db.Model):
     status = Column(String(length=128), nullable=True)
     options = Column(JSON, nullable=True)
     domains = Column(JSON, nullable=True)
+    certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id')
 
     def __init__(self, name, description, provider_type, credentials):
         self.name = name

lemur/dns_providers/service.py

@@ -22,6 +22,15 @@ def get(dns_provider_id):
     return provider
 
 
+def get_all_dns_providers():
+    """
+    Retrieves all dns providers within Lemur.
+
+    :return:
+    """
+    return DnsProvider.query.all()
+
+
 def get_friendly(dns_provider_id):
     """
     Retrieves a dns provider by its lemur assigned ID.
@@ -96,6 +105,15 @@ def get_types():
     return provider_config
 
 
+def set_domains(dns_provider, domains):
+    """
+    Increments pending certificate attempt counter and updates it in the database.
+    """
+    dns_provider.domains = domains
+    database.update(dns_provider)
+    return dns_provider
+
+
 def create(data):
     provider_name = data.get("name")
 

lemur/manage.py

@@ -15,6 +15,7 @@ from flask_script import Manager, Command, Option, prompt_pass
 from flask_migrate import Migrate, MigrateCommand, stamp
 from flask_script.commands import ShowUrls, Clean, Server
 
+from lemur.dns_providers.cli import manager as dns_provider_manager
 from lemur.sources.cli import manager as source_manager
 from lemur.policies.cli import manager as policy_manager
 from lemur.reporting.cli import manager as report_manager
@@ -539,6 +540,7 @@ def main():
     manager.add_command("report", report_manager)
     manager.add_command("policy", policy_manager)
     manager.add_command("pending_certs", pending_certificate_manager)
+    manager.add_command("dns_providers", dns_provider_manager)
     manager.run()
 
 

lemur/plugins/lemur_acme/dyn.py

@@ -5,7 +5,7 @@ import dns.exception
 import dns.name
 import dns.query
 import dns.resolver
-from dyn.tm.errors import DynectGetError
+from dyn.tm.errors import DynectCreateError
 from dyn.tm.session import DynectSession
 from dyn.tm.zones import Node, Zone, get_all_zones
 from flask import current_app
@@ -49,6 +49,7 @@ def wait_for_dns_change(change_id, account_number=None):
             break
         time.sleep(20)
     if not status:
+        # TODO: Delete associated DNS text record here
         raise Exception("Unable to query DNS token for fqdn {}.".format(fqdn))
     return
 
@@ -70,6 +71,15 @@ def get_zone_name(domain):
     return zone_name
 
 
+def get_zones(account_number):
+    get_dynect_session()
+    zones = get_all_zones()
+    zone_list = []
+    for zone in zones:
+        zone_list.append(zone.name)
+    return zone_list
+
+
 def create_txt_record(domain, token, account_number):
     get_dynect_session()
     zone_name = get_zone_name(domain)
@@ -77,21 +87,20 @@ def create_txt_record(domain, token, account_number):
     node_name = '.'.join(domain.split('.')[:-zone_parts])
     fqdn = "{0}.{1}".format(node_name, zone_name)
     zone = Zone(zone_name)
+
     try:
-        # Delete all stale ACME TXT records
-        delete_acme_txt_records(domain)
-    except DynectGetError as e:
-        if (
-                "No such zone." in e.message or
-                "Host is not in this zone" in e.message or
-                "Host not found in this zone" in e.message
-        ):
-            current_app.logger.debug("Unable to delete ACME TXT records. They probably don't exist yet: {}".format(e))
-        else:
-            raise
         zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
         zone.publish()
-    current_app.logger.debug("TXT record created: {0}".format(fqdn))
+        current_app.logger.debug("TXT record created: {0}, token: {1}".format(fqdn, token))
+    except DynectCreateError as e:
+        if "Cannot duplicate existing record data" in e.message:
+            current_app.logger.debug(
+                "Unable to add record. Domain: {}. Token: {}. "
+                "Record already exists: {}".format(domain, token, e), exc_info=True
+            )
+        else:
+            raise
+
     change_id = (fqdn, token)
     return change_id
 

lemur/plugins/lemur_acme/plugin.py

@@ -33,15 +33,6 @@ from lemur.plugins.bases import IssuerPlugin
 from lemur.plugins.lemur_acme import cloudflare, dyn, route53
 
 
-def find_dns_challenge(authorizations):
-    dns_challenges = []
-    for authz in authorizations:
-        for combo in authz.body.challenges:
-            if isinstance(combo.chall, challenges.DNS01):
-                dns_challenges.append(combo)
-    return dns_challenges
-
-
 class AuthorizationRecord(object):
     def __init__(self, host, authz, dns_challenge, change_id):
         self.host = host
@@ -50,26 +41,37 @@ class AuthorizationRecord(object):
         self.change_id = change_id
 
 
-def maybe_remove_wildcard(host):
+class AcmeHandler(object):
+    def __init__(self):
+        self.dns_providers_for_domain = {}
+        self.all_dns_providers = dns_provider_service.get_all_dns_providers()
+
+    def find_dns_challenge(self, authorizations):
+        dns_challenges = []
+        for authz in authorizations:
+            for combo in authz.body.challenges:
+                if isinstance(combo.chall, challenges.DNS01):
+                    dns_challenges.append(combo)
+        return dns_challenges
+
+    def maybe_remove_wildcard(self, host):
         return host.replace("*.", "")
 
-
+    def maybe_add_extension(self, host, dns_provider_options):
-def maybe_add_extension(host, dns_provider_options):
         if dns_provider_options and dns_provider_options.get("acme_challenge_extension"):
             host = host + dns_provider_options.get("acme_challenge_extension")
         return host
 
-
+    def start_dns_challenge(self, acme_client, account_number, host, dns_provider, order, dns_provider_options):
-def start_dns_challenge(acme_client, account_number, host, dns_provider, order, dns_provider_options):
         current_app.logger.debug("Starting DNS challenge for {0}".format(host))
 
-    dns_challenges = find_dns_challenge(order.authorizations)
+        dns_challenges = self.find_dns_challenge(order.authorizations)
         change_ids = []
 
-    host_to_validate = maybe_remove_wildcard(host)
+        host_to_validate = self.maybe_remove_wildcard(host)
-    host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options)
+        host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
 
-    for dns_challenge in find_dns_challenge(order.authorizations):
+        for dns_challenge in self.find_dns_challenge(order.authorizations):
             change_id = dns_provider.create_txt_record(
                 dns_challenge.validation_domain_name(host_to_validate),
                 dns_challenge.validation(acme_client.client.net.key),
@@ -84,14 +86,21 @@ def start_dns_challenge(acme_client, account_number, host, dns_provider, order,
             change_ids
         )
 
-
+    def complete_dns_challenge(self, acme_client, authz_record):
-def complete_dns_challenge(acme_client, account_number, authz_record, dns_provider):
         current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.authz[0].body.identifier.value))
+        dns_providers = self.dns_providers_for_domain.get(authz_record.host)
+        if not dns_providers:
+            raise Exception("No DNS providers found for domain: {}".format(authz_record.host))
+
+        for dns_provider in dns_providers:
+            # Grab account number (For Route53)
+            dns_provider_options = json.loads(dns_provider.credentials)
+            account_number = dns_provider_options.get("account_id")
+            dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
             for change_id in authz_record.change_id:
-        dns_provider.wait_for_dns_change(change_id, account_number=account_number)
+                dns_provider_plugin.wait_for_dns_change(change_id, account_number=account_number)
 
             for dns_challenge in authz_record.dns_challenge:
-
                 response = dns_challenge.response(acme_client.client.net.key)
 
                 verified = response.simple_verify(
@@ -106,8 +115,7 @@ def complete_dns_challenge(acme_client, account_number, authz_record, dns_provid
                 time.sleep(5)
                 acme_client.answer_challenge(dns_challenge, response)
 
-
+    def request_certificate(self, acme_client, authorizations, order):
-def request_certificate(acme_client, authorizations, csr, order):
         for authorization in authorizations:
             for authz in authorization.authz:
                 authorization_resource, _ = acme_client.poll(authz)
@@ -128,8 +136,7 @@ def request_certificate(acme_client, authorizations, csr, order):
         current_app.logger.debug("{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)))
         return pem_certificate, pem_certificate_chain
 
-
+    def setup_acme_client(self, authority):
-def setup_acme_client(authority):
         if not authority.options:
             raise InvalidAuthority("Invalid authority. Options not set")
         options = {}
@@ -164,8 +171,7 @@ def setup_acme_client(authority):
 
         return client, registration
 
-
+    def get_domains(self, options):
-def get_domains(options):
         """
         Fetches all domains currently requested
         :param options:
@@ -181,25 +187,51 @@ def get_domains(options):
         current_app.logger.debug("Got these domains: {0}".format(domains))
         return domains
 
-
+    def get_authorizations(self, acme_client, order, order_info):
-def get_authorizations(acme_client, order, order_info, dns_provider, dns_provider_options):
         authorizations = []
+
         for domain in order_info.domains:
-        authz_record = start_dns_challenge(acme_client, order_info.account_number, domain, dns_provider, order,
+            if not self.dns_providers_for_domain.get(domain):
-                                           dns_provider_options)
+                raise Exception("No DNS providers found for domain: {}".format(domain))
+            for dns_provider in self.dns_providers_for_domain[domain]:
+                dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
+                dns_provider_options = json.loads(dns_provider.credentials)
+                account_number = dns_provider_options.get("account_id")
+                authz_record = self.start_dns_challenge(acme_client, account_number, domain,
+                                                        dns_provider_plugin,
+                                                        order,
+                                                        dns_provider.options)
                 authorizations.append(authz_record)
         return authorizations
 
+    def autodetect_dns_providers(self, domain):
+        """
+        Get DNS providers associated with a domain when it has not been provided for certificate creation.
+        :param domain:
+        :return: dns_providers: List of DNS providers that have the correct zone.
+        """
+        self.dns_providers_for_domain[domain] = []
+        for dns_provider in self.all_dns_providers:
+            for name in dns_provider.domains:
+                if domain.endswith(name):
+                    self.dns_providers_for_domain[domain].append(dns_provider)
+        return self.dns_providers_for_domain
 
-def finalize_authorizations(acme_client, account_number, dns_provider, authorizations, dns_provider_options):
+    def finalize_authorizations(self, acme_client, authorizations):
         for authz_record in authorizations:
-        complete_dns_challenge(acme_client, account_number, authz_record, dns_provider)
+            self.complete_dns_challenge(acme_client, authz_record)
         for authz_record in authorizations:
             dns_challenges = authz_record.dns_challenge
-        host_to_validate = maybe_remove_wildcard(authz_record.host)
-        host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options)
             for dns_challenge in dns_challenges:
-            dns_provider.delete_txt_record(
+                dns_providers = self.dns_providers_for_domain.get(authz_record.host)
+                for dns_provider in dns_providers:
+                    # Grab account number (For Route53)
+                    dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
+                    dns_provider_options = json.loads(dns_provider.credentials)
+                    account_number = dns_provider_options.get("account_id")
+                    host_to_validate = self.maybe_remove_wildcard(authz_record.host)
+                    host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
+                    dns_provider_plugin.delete_txt_record(
                         authz_record.change_id,
                         account_number,
                         dns_challenge.validation_domain_name(host_to_validate),
@@ -208,8 +240,7 @@ def finalize_authorizations(acme_client, account_number, dns_provider, authoriza
 
         return authorizations
 
-
+    def cleanup_dns_challenges(self, acme_client, authorizations):
-def cleanup_dns_challenges(acme_client, account_number, dns_provider, authorizations, dns_provider_options):
         """
         Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called
         on an exception
@@ -222,9 +253,14 @@ def cleanup_dns_challenges(acme_client, account_number, dns_provider, authorizat
         :return:
         """
         for authz_record in authorizations:
+            dns_providers = self.dns_providers_for_domain.get(authz_record.host)
+            for dns_provider in dns_providers:
+                # Grab account number (For Route53)
+                dns_provider_options = json.loads(dns_provider.credentials)
+                account_number = dns_provider_options.get("account_id")
                 dns_challenges = authz_record.dns_challenge
-        host_to_validate = maybe_remove_wildcard(authz_record.host)
+                host_to_validate = self.maybe_remove_wildcard(authz_record.host)
-        host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options)
+                host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
                 for dns_challenge in dns_challenges:
                     try:
                         dns_provider.delete_txt_record(
@@ -233,10 +269,22 @@ def cleanup_dns_challenges(acme_client, account_number, dns_provider, authorizat
                             dns_challenge.validation_domain_name(host_to_validate),
                             dns_challenge.validation(acme_client.client.net.key)
                         )
-            except Exception:
+                    except Exception as e:
-                # If this fails, it's most likely because the record doesn't exist or we're not authorized to modify it.
+                        # If this fails, it's most likely because the record doesn't exist (It was already cleaned up)
+                        # or we're not authorized to modify it.
                         pass
 
+    def get_dns_provider(self, type):
+        provider_types = {
+            'cloudflare': cloudflare,
+            'dyn': dyn,
+            'route53': route53,
+        }
+        provider = provider_types.get(type)
+        if not provider:
+            raise UnknownProvider("No such DNS provider: {}".format(type))
+        return provider
+
 
 class ACMEIssuerPlugin(IssuerPlugin):
     title = 'Acme'
@@ -279,6 +327,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
 
     def __init__(self, *args, **kwargs):
         super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
+        self.acme = AcmeHandler()
 
     def get_dns_provider(self, type):
         provider_types = {
@@ -291,22 +340,40 @@ class ACMEIssuerPlugin(IssuerPlugin):
             raise UnknownProvider("No such DNS provider: {}".format(type))
         return provider
 
+    def get_all_zones(self, dns_provider):
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+        dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
+        return dns_provider_plugin.get_zones(account_number=account_number)
+
     def get_ordered_certificate(self, pending_cert):
-        acme_client, registration = setup_acme_client(pending_cert.authority)
+        acme_client, registration = self.acme.setup_acme_client(pending_cert.authority)
         order_info = authorization_service.get(pending_cert.external_id)
+        if pending_cert.dns_provider_id:
             dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
-        dns_provider_options = dns_provider.options
+
-        dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
+            for domain in order_info.domains:
+                # Currently, we only support specifying one DNS provider per certificate, even if that
+                # certificate has multiple SANs that may belong to different providers.
+                self.acme.dns_providers_for_domain[domain] = [dns_provider]
+        else:
+            for domain in order_info.domains:
+                self.acme.autodetect_dns_providers(domain)
+
         try:
-            authorizations = get_authorizations(
+            order = acme_client.new_order(pending_cert.csr)
-                acme_client, order_info.account_number, order_info.domains, dns_provider_type, dns_provider_options)
+        except WildcardUnsupportedError:
+            raise Exception("The currently selected ACME CA endpoint does"
+                            " not support issuing wildcard certificates.")
+        try:
+            authorizations = self.acme.get_authorizations(acme_client, order, order_info)
         except ClientError:
             current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert.name), exc_info=True)
             return False
 
-        authorizations = finalize_authorizations(
+        authorizations = self.acme.finalize_authorizations(acme_client, authorizations)
-            acme_client, order_info.account_number, dns_provider_type, authorizations, dns_provider_options)
+        pem_certificate, pem_certificate_chain = self.acme.request_certificate(
-        pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr)
+            acme_client, authorizations, order)
         cert = {
             'body': "\n".join(str(pem_certificate).splitlines()),
             'chain': "\n".join(str(pem_certificate_chain).splitlines()),
@@ -319,28 +386,32 @@ class ACMEIssuerPlugin(IssuerPlugin):
         certs = []
         for pending_cert in pending_certs:
             try:
-                acme_client, registration = setup_acme_client(pending_cert.authority)
+                acme_client, registration = self.acme.setup_acme_client(pending_cert.authority)
                 order_info = authorization_service.get(pending_cert.external_id)
+                if pending_cert.dns_provider_id:
                     dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
-                dns_provider_options = dns_provider.options
+
-                dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
+                    for domain in order_info.domains:
+                        # Currently, we only support specifying one DNS provider per certificate, even if that
+                        # certificate has multiple SANs that may belong to different providers.
+                        self.acme.dns_providers_for_domain[domain] = [dns_provider]
+                else:
+                    for domain in order_info.domains:
+                        self.acme.autodetect_dns_providers(domain)
+
                 try:
                     order = acme_client.new_order(pending_cert.csr)
                 except WildcardUnsupportedError:
                     raise Exception("The currently selected ACME CA endpoint does"
                                     " not support issuing wildcard certificates.")
 
-                authorizations = get_authorizations(acme_client, order, order_info, dns_provider_type,
+                authorizations = self.acme.get_authorizations(acme_client, order, order_info)
-                                                    dns_provider_options)
 
                 pending.append({
                     "acme_client": acme_client,
-                    "account_number": order_info.account_number,
-                    "dns_provider_type": dns_provider_type,
                     "authorizations": authorizations,
                     "pending_cert": pending_cert,
                     "order": order,
-                    "dns_provider_options": dns_provider_options,
                 })
             except (ClientError, ValueError, Exception) as e:
                 current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
@@ -352,17 +423,13 @@ class ACMEIssuerPlugin(IssuerPlugin):
 
         for entry in pending:
             try:
-                entry["authorizations"] = finalize_authorizations(
+                entry["authorizations"] = self.acme.finalize_authorizations(
                     entry["acme_client"],
-                    entry["account_number"],
-                    entry["dns_provider_type"],
                     entry["authorizations"],
-                    entry["dns_provider_options"],
                 )
-                pem_certificate, pem_certificate_chain = request_certificate(
+                pem_certificate, pem_certificate_chain = self.acme.request_certificate(
                     entry["acme_client"],
                     entry["authorizations"],
-                    entry["pending_cert"].csr,
                     entry["order"]
                 )
 
@@ -383,12 +450,9 @@ class ACMEIssuerPlugin(IssuerPlugin):
                     "last_error": e,
                 })
                 # Ensure DNS records get deleted
-                cleanup_dns_challenges(
+                self.acme.cleanup_dns_challenges(
                     entry["acme_client"],
-                    entry["account_number"],
-                    entry["dns_provider_type"],
                     entry["authorizations"],
-                    entry["dns_provider_options"],
                 )
         return certs
 
@@ -402,21 +466,25 @@ class ACMEIssuerPlugin(IssuerPlugin):
         """
         authority = issuer_options.get('authority')
         create_immediately = issuer_options.get('create_immediately', False)
-        acme_client, registration = setup_acme_client(authority)
+        acme_client, registration = self.acme.setup_acme_client(authority)
-        dns_provider = issuer_options.get('dns_provider')
+        dns_provider = issuer_options.get('dns_provider', {})
+        # TODO: IF NOT DNS PROVIDER, AUTODISCOVER
+        if dns_provider:
             dns_provider_options = dns_provider.options
-        if not dns_provider:
-            raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.")
             credentials = json.loads(dns_provider.credentials)
-
             current_app.logger.debug("Using DNS provider: {0}".format(dns_provider.provider_type))
-        dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
+            dns_provider_plugin = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
             account_number = credentials.get("account_id")
             if dns_provider.provider_type == 'route53' and not account_number:
                 error = "Route53 DNS Provider {} does not have an account number configured.".format(dns_provider.name)
                 current_app.logger.error(error)
                 raise InvalidConfiguration(error)
-        domains = get_domains(issuer_options)
+        else:
+            dns_provider = {}
+            dns_provider_options = None
+            account_number = None
+
+        domains = self.acme.get_domains(issuer_options)
         if not create_immediately:
             # Create pending authorizations that we'll need to do the creation
             authz_domains = []
@@ -426,14 +494,15 @@ class ACMEIssuerPlugin(IssuerPlugin):
                 else:
                     authz_domains.append(d.value)
 
-            dns_authorization = authorization_service.create(account_number, authz_domains, dns_provider.provider_type)
+            dns_authorization = authorization_service.create(account_number, authz_domains,
+                                                             dns_provider.get("provider_type"))
             # Return id of the DNS Authorization
             return None, None, dns_authorization.id
 
-        authorizations = get_authorizations(acme_client, account_number, domains, dns_provider_type,
+        authorizations = self.acme.get_authorizations(acme_client, account_number, domains, dns_provider_plugin,
                                             dns_provider_options)
-        finalize_authorizations(acme_client, account_number, dns_provider_type, authorizations, dns_provider_options)
+        self.acme.finalize_authorizations(acme_client, account_number, dns_provider_plugin, authorizations, dns_provider_options)
-        pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, csr)
+        pem_certificate, pem_certificate_chain = self.acme.request_certificate(acme_client, authorizations, csr)
         # TODO add external ID (if possible)
         return pem_certificate, pem_certificate_chain, None
 

lemur/plugins/lemur_acme/route53.py

@@ -31,6 +31,16 @@ def find_zone_id(domain, client=None):
     return zones[0][1]
 
 
+@sts_client('route53')
+def get_zones(client=None):
+    paginator = client.get_paginator("list_hosted_zones")
+    zones = []
+    for page in paginator.paginate():
+        for zone in page["HostedZones"]:
+            zones.append(zone["Name"][:-1])  # We need [:-1] to strip out the trailing dot.
+    return zones
+
+
 @sts_client('route53')
 def change_txt_record(action, zone_id, domain, value, client=None):
     current_txt_records = []
@@ -50,6 +60,12 @@ def change_txt_record(action, zone_id, domain, value, client=None):
             raise
     # For some reason TXT records need to be
     # manually quoted.
+    seen = False
+    for record in current_txt_records:
+        for k, v in record.items():
+            if '"{}"'.format(value) == v:
+                seen = True
+    if not seen:
         current_txt_records.append({"Value": '"{}"'.format(value)})
 
     if action == "DELETE" and len(current_txt_records) > 1:
@@ -95,6 +111,7 @@ def create_txt_record(host, value, account_number):
 def delete_txt_record(change_ids, account_number, host, value):
     for change_id in change_ids:
         zone_id, _ = change_id
+        try:
             change_txt_record(
                 "DELETE",
                 zone_id,
@@ -102,3 +119,9 @@ def delete_txt_record(change_ids, account_number, host, value):
                 value,
                 account_number=account_number
             )
+        except Exception as e:
+            if "but it was not found" in e.response.get("Error", {}).get("Message"):
+                # We tried to delete a record that doesn't exist. We'll ignore this error.
+                pass
+            else:
+                raise

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -7,8 +7,15 @@ from lemur.plugins.lemur_acme import plugin
 
 class TestAcme(unittest.TestCase):
 
-    def setUp(self):
+    @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
+    def setUp(self, mock_dns_provider_service):
         self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin()
+        self.acme = plugin.AcmeHandler()
+        mock_dns_provider = Mock()
+        mock_dns_provider.name = "cloudflare"
+        mock_dns_provider.credentials = "{}"
+        mock_dns_provider.provider_type = "cloudflare"
+        self.acme.dns_providers_for_domain = {"www.test.com": [mock_dns_provider], "test.fakedomain.net": [mock_dns_provider]}
 
     @patch('lemur.plugins.lemur_acme.plugin.len', return_value=1)
     def test_find_dns_challenge(self, mock_len):
@@ -22,7 +29,7 @@ class TestAcme(unittest.TestCase):
         mock_entry = Mock()
         mock_entry.chall = c
         mock_authz.body.resolved_combinations.append(mock_entry)
-        result = yield plugin.find_dns_challenge(mock_authz)
+        result = yield self.acme.find_dns_challenge(mock_authz)
         self.assertEqual(result, mock_entry)
 
     def test_authz_record(self):
@@ -32,7 +39,7 @@ class TestAcme(unittest.TestCase):
     @patch('acme.client.Client')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
     @patch('lemur.plugins.lemur_acme.plugin.len', return_value=1)
-    @patch('lemur.plugins.lemur_acme.plugin.find_dns_challenge')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge')
     def test_start_dns_challenge(self, mock_find_dns_challenge, mock_len, mock_app, mock_acme):
         assert mock_len
         mock_order = Mock()
@@ -42,7 +49,7 @@ class TestAcme(unittest.TestCase):
         mock_entry = MagicMock()
         from acme import challenges
         c = challenges.DNS01()
-        mock_entry.chall = c
+        mock_entry.chall = TestAcme.test_complete_dns_challenge_fail
         mock_authz.body.resolved_combinations.append(mock_entry)
         mock_acme.request_domain_challenges = Mock(return_value=mock_authz)
         mock_dns_provider = Mock()
@@ -52,19 +59,20 @@ class TestAcme(unittest.TestCase):
         iterable = mock_find_dns_challenge.return_value
         iterator = iter(values)
         iterable.__iter__.return_value = iterator
-        result = plugin.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order, {})
+        result = self.acme.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order, {})
         self.assertEqual(type(result), plugin.AuthorizationRecord)
 
     @patch('acme.client.Client')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
-    def test_complete_dns_challenge_success(self, mock_current_app, mock_acme):
+    @patch('lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change')
+    def test_complete_dns_challenge_success(self, mock_wait_for_dns_change, mock_current_app, mock_acme):
         mock_dns_provider = Mock()
         mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
-
         mock_authz = Mock()
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True)
         mock_authz.authz = []
+        mock_authz.host = "www.test.com"
         mock_authz_record = Mock()
         mock_authz_record.body.identifier.value = "test"
         mock_authz.authz.append(mock_authz_record)
@@ -73,11 +81,12 @@ class TestAcme(unittest.TestCase):
         mock_authz.dns_challenge = []
         dns_challenge = Mock()
         mock_authz.dns_challenge.append(dns_challenge)
-        plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
+        self.acme.complete_dns_challenge(mock_acme, mock_authz)
 
     @patch('acme.client.Client')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
-    def test_complete_dns_challenge_fail(self, mock_current_app, mock_acme):
+    @patch('lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change')
+    def test_complete_dns_challenge_fail(self, mock_wait_for_dns_change, mock_current_app, mock_acme):
         mock_dns_provider = Mock()
         mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
 
@@ -85,6 +94,7 @@ class TestAcme(unittest.TestCase):
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False)
         mock_authz.authz = []
+        mock_authz.host = "www.test.com"
         mock_authz_record = Mock()
         mock_authz_record.body.identifier.value = "test"
         mock_authz.authz.append(mock_authz_record)
@@ -95,13 +105,13 @@ class TestAcme(unittest.TestCase):
         mock_authz.dns_challenge.append(dns_challenge)
         self.assertRaises(
             ValueError,
-            plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
+            self.acme.complete_dns_challenge(mock_acme, mock_authz)
         )
 
     @patch('acme.client.Client')
     @patch('OpenSSL.crypto', return_value="mock_cert")
     @patch('josepy.util.ComparableX509')
-    @patch('lemur.plugins.lemur_acme.plugin.find_dns_challenge')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
     def test_request_certificate(self, mock_current_app, mock_find_dns_challenge, mock_jose, mock_crypto, mock_acme):
         mock_cert_response = Mock()
@@ -115,13 +125,13 @@ class TestAcme(unittest.TestCase):
         mock_acme.fetch_chain = Mock(return_value="mock_chain")
         mock_crypto.dump_certificate = Mock(return_value=b'chain')
         mock_order = Mock()
-        plugin.request_certificate(mock_acme, [], "mock_csr", mock_order)
+        self.acme.request_certificate(mock_acme, [], mock_order)
 
     def test_setup_acme_client_fail(self):
         mock_authority = Mock()
         mock_authority.options = []
         with self.assertRaises(Exception):
-            plugin.setup_acme_client(mock_authority)
+            self.acme.setup_acme_client(mock_authority)
 
     @patch('lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
@@ -135,7 +145,7 @@ class TestAcme(unittest.TestCase):
         mock_client.agree_to_tos = Mock(return_value=True)
         mock_acme.return_value = mock_client
         mock_current_app.config = {}
-        result_client, result_registration = plugin.setup_acme_client(mock_authority)
+        result_client, result_registration = self.acme.setup_acme_client(mock_authority)
         assert result_client
         assert result_registration
 
@@ -144,7 +154,7 @@ class TestAcme(unittest.TestCase):
         options = {
             "common_name": "test.netflix.net"
         }
-        result = plugin.get_domains(options)
+        result = self.acme.get_domains(options)
         self.assertEqual(result, [options["common_name"]])
 
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
@@ -160,10 +170,10 @@ class TestAcme(unittest.TestCase):
                 }
             }
         }
-        result = plugin.get_domains(options)
+        result = self.acme.get_domains(options)
         self.assertEqual(result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"])
 
-    @patch('lemur.plugins.lemur_acme.plugin.start_dns_challenge', return_value="test")
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge', return_value="test")
     def test_get_authorizations(self, mock_start_dns_challenge):
         mock_order = Mock()
         mock_order.body.identifiers = []
@@ -172,10 +182,10 @@ class TestAcme(unittest.TestCase):
         mock_order_info = Mock()
         mock_order_info.account_number = 1
         mock_order_info.domains = ["test.fakedomain.net"]
-        result = plugin.get_authorizations("acme_client", mock_order, mock_order_info, "dns_provider", {})
+        result = self.acme.get_authorizations("acme_client", mock_order, mock_order_info)
         self.assertEqual(result, ["test"])
 
-    @patch('lemur.plugins.lemur_acme.plugin.complete_dns_challenge', return_value="test")
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge', return_value="test")
     def test_finalize_authorizations(self, mock_complete_dns_challenge):
         mock_authz = []
         mock_authz_record = MagicMock()
@@ -188,7 +198,7 @@ class TestAcme(unittest.TestCase):
         mock_dns_provider.delete_txt_record = Mock()
 
         mock_acme_client = Mock()
-        result = plugin.finalize_authorizations(mock_acme_client, "account_number", mock_dns_provider, mock_authz, {})
+        result = self.acme.finalize_authorizations(mock_acme_client, mock_authz)
         self.assertEqual(result, mock_authz)
 
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
@@ -210,7 +220,8 @@ class TestAcme(unittest.TestCase):
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
     @patch('lemur.plugins.lemur_acme.dyn.current_app')
     @patch('lemur.plugins.lemur_acme.cloudflare.current_app')
-    def test_get_dns_provider(self, mock_current_app_cloudflare, mock_current_app_dyn, mock_current_app):
+    @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
+    def test_get_dns_provider(self, mock_dns_provider_service, mock_current_app_cloudflare, mock_current_app_dyn, mock_current_app):
         provider = plugin.ACMEIssuerPlugin()
         route53 = provider.get_dns_provider("route53")
         assert route53
@@ -219,13 +230,13 @@ class TestAcme(unittest.TestCase):
         dyn = provider.get_dns_provider("dyn")
         assert dyn
 
-    @patch('lemur.plugins.lemur_acme.plugin.setup_acme_client')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
     @patch('lemur.plugins.lemur_acme.plugin.authorization_service')
     @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
-    @patch('lemur.plugins.lemur_acme.plugin.get_authorizations')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations')
-    @patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations')
-    @patch('lemur.plugins.lemur_acme.plugin.request_certificate')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate')
     def test_get_ordered_certificate(
             self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations,
             mock_dns_provider_service, mock_authorization_service, mock_current_app, mock_acme):
@@ -248,13 +259,13 @@ class TestAcme(unittest.TestCase):
             }
         )
 
-    @patch('lemur.plugins.lemur_acme.plugin.setup_acme_client')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
     @patch('lemur.plugins.lemur_acme.plugin.authorization_service')
     @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
-    @patch('lemur.plugins.lemur_acme.plugin.get_authorizations')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations')
-    @patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations')
-    @patch('lemur.plugins.lemur_acme.plugin.request_certificate')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate')
     def test_get_ordered_certificates(
             self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations,
             mock_dns_provider_service, mock_authorization_service, mock_current_app, mock_acme):
@@ -275,12 +286,12 @@ class TestAcme(unittest.TestCase):
         self.assertEqual(result[0]['cert'], {'body': 'pem_certificate', 'chain': 'chain', 'external_id': '1'})
         self.assertEqual(result[1]['cert'], {'body': 'pem_certificate', 'chain': 'chain', 'external_id': '2'})
 
-    @patch('lemur.plugins.lemur_acme.plugin.setup_acme_client')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client')
     @patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
     @patch('lemur.plugins.lemur_acme.plugin.current_app')
-    @patch('lemur.plugins.lemur_acme.plugin.get_authorizations')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations')
-    @patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations')
-    @patch('lemur.plugins.lemur_acme.plugin.request_certificate')
+    @patch('lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate')
     @patch('lemur.plugins.lemur_acme.plugin.authorization_service')
     def test_create_certificate(self, mock_authorization_service, mock_request_certificate,
                                 mock_finalize_authorizations, mock_get_authorizations,

lemur/schemas.py

@@ -78,7 +78,7 @@ def fetch_objects(model, data, many=False):
         items = model.query.filter(getattr(model, attr).in_(values)).all()
         found = [getattr(i, attr) for i in items]
         diff = set(values).symmetric_difference(set(found))
-
+        AssociatedDnsProviderSchema
         if diff:
             raise ValidationError('Unable to locate {model} with {attr} {diff}'.format(
                 model=model,

lemur/static/app/angular/certificates/certificate/tracking.tpl.html

@@ -106,6 +106,17 @@
           </ui-select-choices>
         </ui-select>
       </div>
+    </div>
+       <div class="form-group" ng-show="certificate.authority.plugin.slug == 'acme-issuer'">
+           <label class="control-label col-sm-2">
+            Note:
+            </label>
+           <div class="col-sm-10">
+               The selected authority uses the <a target="_blank" href="https://letsencrypt.org/how-it-works/">ACME protocol</a> and works differently than other authorities.
+               Your request will initially be created under the "pending certificates" section. Lemur will attempt to create the certificate for you,
+               and move the final certificate to the "certificates" section. Lemur performs validation by writing a DNS text record. You may choose a specific DNS provider,
+               or allow Lemur to automatically detect the correct provider for you. Requests may take up to ten minutes.
+           </div>
        </div>
        <div class="form-group" ng-show="certificate.authority.plugin.slug == 'acme-issuer'">
         <label class="control-label col-sm-2">
@@ -114,7 +125,7 @@
 
         <div class="col-sm-10">
           <select class="form-control" ng-model="certificate.dnsProvider" ng-options="item as item.name for item in dnsProviders.items track by item.id">
-              <option value="">- choose an entry. Neded for ACME Providers -</option>
+              <option value="">Automatically select for me</option>
           </select>
         </div>
       </div>

requirements-dev.txt

@@ -6,7 +6,7 @@
 #
 aspy.yaml==1.1.1          # via pre-commit
 cached-property==1.4.3    # via pre-commit
-certifi==2018.4.16        # via requests
+certifi==2018.8.13        # via requests
 cfgv==1.1.0               # via pre-commit
 chardet==3.0.4            # via requests
 flake8==3.5.0

requirements-docs.txt

@@ -15,9 +15,9 @@ asyncpool==1.0
 babel==2.6.0              # via sphinx
 bcrypt==3.1.4
 blinker==1.4
-boto3==1.7.66
+boto3==1.7.75
-botocore==1.10.66
+botocore==1.10.75
-certifi==2018.4.16
+certifi==2018.8.13
 cffi==1.11.5
 chardet==3.0.4
 click==6.7
@@ -50,7 +50,7 @@ lockfile==0.12.2
 mako==1.0.7
 markupsafe==1.0
 marshmallow-sqlalchemy==0.14.0
-marshmallow==2.15.3
+marshmallow==2.15.4
 mock==2.0.0
 ndg-httpsclient==0.5.1
 packaging==17.1           # via sphinx

requirements-tests.txt

@@ -8,10 +8,10 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.1.5       # via pytest
 attrs==18.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.7.71             # via moto
+boto3==1.7.75             # via moto
 boto==2.49.0              # via moto
-botocore==1.10.71         # via boto3, moto, s3transfer
+botocore==1.10.75         # via boto3, moto, s3transfer
-certifi==2018.4.16        # via requests
+certifi==2018.8.13        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
 click==6.7                # via flask
@@ -19,12 +19,14 @@ cookies==2.2.1            # via moto, responses
 coverage==4.5.1
 cryptography==2.3         # via moto
 docker-pycreds==0.3.0     # via docker
-docker==3.4.1             # via moto
+docker==3.5.0             # via moto
 docutils==0.14            # via botocore
+ecdsa==0.13               # via python-jose
 factory-boy==2.11.1
-faker==0.8.17
+faker==0.9.0
 flask==1.0.2              # via pytest-flask
 freezegun==0.3.10
+future==0.16.0            # via python-jose
 idna==2.7                 # via cryptography, requests
 itsdangerous==0.24        # via flask
 jinja2==2.10              # via flask, moto
@@ -34,25 +36,27 @@ jsonpickle==0.9.6         # via aws-xray-sdk
 markupsafe==1.0           # via jinja2
 mock==2.0.0               # via moto
 more-itertools==4.3.0     # via pytest
-moto==1.3.3
+moto==1.3.4
 nose==1.3.7
 pbr==4.2.0                # via mock
 pluggy==0.7.1             # via pytest
 py==1.5.4                 # via pytest
 pyaml==17.12.1            # via moto
 pycparser==2.18           # via cffi
+pycryptodome==3.6.5       # via python-jose
 pyflakes==2.0.0
 pytest-flask==0.10.0
 pytest-mock==1.10.0
 pytest==3.7.1
-python-dateutil==2.6.1    # via botocore, faker, freezegun, moto
+python-dateutil==2.7.3    # via botocore, faker, freezegun, moto
+python-jose==2.0.2        # via moto
 pytz==2018.5              # via moto
 pyyaml==3.13              # via pyaml
 requests-mock==1.5.2
 requests==2.19.1          # via aws-xray-sdk, docker, moto, requests-mock, responses
 responses==0.9.0          # via moto
 s3transfer==0.1.13        # via boto3
-six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client
+six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
 text-unidecode==1.2       # via faker
 urllib3==1.23             # via requests
 websocket-client==0.48.0  # via docker

requirements.txt

@@ -13,9 +13,9 @@ asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.7.71
+boto3==1.7.75
-botocore==1.10.71         # via boto3, s3transfer
+botocore==1.10.75         # via boto3, s3transfer
-certifi==2018.4.16
+certifi==2018.8.13
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
 click==6.7                # via flask