Merge branch 'master' into acme_validation_dns_provider_option

lemur/plugins/lemur_acme/dyn.py

@@ -5,11 +5,10 @@ import dns.exception
 import dns.name
 import dns.query
 import dns.resolver
-from dyn.tm.errors import DynectCreateError
+from dyn.tm.errors import DynectGetError
 from dyn.tm.session import DynectSession
-from dyn.tm.zones import Node, Zone
+from dyn.tm.zones import Node, Zone, get_all_zones
 from flask import current_app
-from tld import get_tld
 
 
 def get_dynect_session():
@@ -42,28 +41,55 @@ def _has_dns_propagated(name, token):
 
 def wait_for_dns_change(change_id, account_number=None):
     fqdn, token = change_id
-    while True:
+    number_of_attempts = 10
+    for attempts in range(0, number_of_attempts):
         status = _has_dns_propagated(fqdn, token)
         current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
         if status:
             break
         time.sleep(20)
+    if not status:
+        raise Exception("Unable to query DNS token for fqdn {}.".format(fqdn))
     return
 
 
+def get_zone_name(domain):
+    zones = get_all_zones()
+
+    zone_name = ""
+
+    for z in zones:
+        if domain.endswith(z.name):
+            # Find the most specific zone possible for the domain
+            # Ex: If fqdn is a.b.c.com, there is a zone for c.com,
+            # and a zone for b.c.com, we want to use b.c.com.
+            if z.name.count(".") > zone_name.count("."):
+                zone_name = z.name
+    if not zone_name:
+        raise Exception("No Dyn zone found for domain: {}".format(domain))
+    return zone_name
+
+
 def create_txt_record(domain, token, account_number):
     get_dynect_session()
-    zone_name = get_tld('http://' + domain)
+    zone_name = get_zone_name(domain)
     zone_parts = len(zone_name.split('.'))
     node_name = '.'.join(domain.split('.')[:-zone_parts])
     fqdn = "{0}.{1}".format(node_name, zone_name)
     zone = Zone(zone_name)
     try:
-        zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
+        # Delete all stale ACME TXT records
-    except DynectCreateError:
+        delete_acme_txt_records(domain)
-        delete_txt_record(None, None, domain, token)
+    except DynectGetError as e:
-        zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
+        if (
-    node = zone.get_node(node_name)
+                "No such zone." in e.message or
+                "Host is not in this zone" in e.message or
+                "Host not found in this zone" in e.message
+        ):
+            current_app.logger.debug("Unable to delete ACME TXT records. They probably don't exist yet: {}".format(e))
+        else:
+            raise
+    zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
     zone.publish()
     current_app.logger.debug("TXT record created: {0}".format(fqdn))
     change_id = (fqdn, token)
@@ -76,7 +102,7 @@ def delete_txt_record(change_id, account_number, domain, token):
         current_app.logger.debug("delete_txt_record: No domain passed")
         return
 
-    zone_name = get_tld('http://' + domain)
+    zone_name = get_zone_name(domain)
     zone_parts = len(zone_name.split('.'))
     node_name = '.'.join(domain.split('.')[:-zone_parts])
     fqdn = "{0}.{1}".format(node_name, zone_name)
@@ -92,40 +118,70 @@ def delete_txt_record(change_id, account_number, domain, token):
     zone.publish()
 
 
+def delete_acme_txt_records(domain):
+    get_dynect_session()
+    if not domain:
+        current_app.logger.debug("delete_acme_txt_records: No domain passed")
+        return
+    acme_challenge_string = "_acme-challenge"
+    if not domain.startswith(acme_challenge_string):
+        current_app.logger.debug(
+            "delete_acme_txt_records: Domain {} doesn't start with string {}. "
+            "Cowardly refusing to delete TXT records".format(domain, acme_challenge_string))
+        return
+
+    zone_name = get_zone_name(domain)
+    zone_parts = len(zone_name.split('.'))
+    node_name = '.'.join(domain.split('.')[:-zone_parts])
+    fqdn = "{0}.{1}".format(node_name, zone_name)
+
+    zone = Zone(zone_name)
+    node = Node(zone_name, fqdn)
+
+    all_txt_records = node.get_all_records_by_type('TXT')
+    for txt_record in all_txt_records:
+        current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn))
+        txt_record.delete()
+    zone.publish()
+
+
 def get_authoritative_nameserver(domain):
-    n = dns.name.from_text(domain)
+    if current_app.config.get('ACME_DYN_GET_AUTHORATATIVE_NAMESERVER'):
+        n = dns.name.from_text(domain)
 
-    depth = 2
+        depth = 2
-    default = dns.resolver.get_default_resolver()
+        default = dns.resolver.get_default_resolver()
-    nameserver = default.nameservers[0]
+        nameserver = default.nameservers[0]
 
-    last = False
+        last = False
-    while not last:
+        while not last:
-        s = n.split(depth)
+            s = n.split(depth)
 
-        last = s[0].to_unicode() == u'@'
+            last = s[0].to_unicode() == u'@'
-        sub = s[1]
+            sub = s[1]
 
-        query = dns.message.make_query(sub, dns.rdatatype.NS)
+            query = dns.message.make_query(sub, dns.rdatatype.NS)
-        response = dns.query.udp(query, nameserver)
+            response = dns.query.udp(query, nameserver)
 
-        rcode = response.rcode()
+            rcode = response.rcode()
-        if rcode != dns.rcode.NOERROR:
+            if rcode != dns.rcode.NOERROR:
-            if rcode == dns.rcode.NXDOMAIN:
+                if rcode == dns.rcode.NXDOMAIN:
-                raise Exception('%s does not exist.' % sub)
+                    raise Exception('%s does not exist.' % sub)
+                else:
+                    raise Exception('Error %s' % dns.rcode.to_text(rcode))
+
+            if len(response.authority) > 0:
+                rrset = response.authority[0]
             else:
-                raise Exception('Error %s' % dns.rcode.to_text(rcode))
+                rrset = response.answer[0]
 
-        if len(response.authority) > 0:
+            rr = rrset[0]
-            rrset = response.authority[0]
+            if rr.rdtype != dns.rdatatype.SOA:
-        else:
+                authority = rr.target
-            rrset = response.answer[0]
+                nameserver = default.query(authority).rrset[0].to_text()
 
-        rr = rrset[0]
+            depth += 1
-        if rr.rdtype != dns.rdatatype.SOA:
-            authority = rr.target
-            nameserver = default.query(authority).rrset[0].to_text()
 
-        depth += 1
+        return nameserver
-
+    else:
-    return nameserver
+        return "8.8.8.8"

lemur/plugins/lemur_acme/plugin.py

@@ -116,8 +116,8 @@ def request_certificate(acme_client, authorizations, csr, order):
 
     try:
         orderr = acme_client.finalize_order(order, deadline)
-    except:
+    except AcmeError:
-        current_app.logger.error("Unable to finalize ACME order: {}".format(order), exc_info=True)
+        current_app.logger.error("Unable to resolve Acme order: {}".format(order), exc_info=True)
         raise
 
     pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
@@ -407,3 +407,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
             if option.get('name') == 'certificate':
                 acme_root = option.get('value')
         return acme_root, "", [role]
+
+    def cancel_ordered_certificate(self, pending_cert, **kwargs):
+        # Needed to override issuer function.
+        pass

requirements-docs.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
-acme==0.24.0
+acme==0.25.1
 alabaster==0.7.11         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==0.9.9
@@ -15,8 +15,8 @@ asyncpool==1.0
 babel==2.6.0              # via sphinx
 bcrypt==3.1.4
 blinker==1.4
-boto3==1.7.32
+boto3==1.7.39
-botocore==1.10.32
+botocore==1.10.37
 certifi==2018.4.16
 cffi==1.11.5
 click==6.7
@@ -27,7 +27,7 @@ dnspython==1.15.0
 docutils==0.14
 dyn==1.8.1
 flask-bcrypt==0.7.1
-flask-cors==3.0.4
+flask-cors==3.0.6
 flask-mail==0.9.1
 flask-migrate==2.1.1
 flask-principal==0.4.0
@@ -37,7 +37,7 @@ flask-sqlalchemy==2.3.2
 flask==0.12
 future==0.16.0
 gunicorn==19.8.1
-idna==2.6
+idna==2.7
 imagesize==1.0.0          # via sphinx
 inflection==0.3.1
 itsdangerous==0.24
@@ -54,7 +54,7 @@ mock==2.0.0
 ndg-httpsclient==0.5.0
 packaging==17.1           # via sphinx
 paramiko==2.4.1
-pbr==4.0.3
+pbr==4.0.4
 pem==17.1.0
 psycopg2==2.7.4
 pyasn1-modules==0.2.1
@@ -65,12 +65,13 @@ pyjwt==1.6.4
 pynacl==1.2.1
 pyopenssl==17.2.0
 pyparsing==2.2.0          # via packaging
-pyrfc3339==1.0
+pyrfc3339==1.1
 python-dateutil==2.7.3
 python-editor==1.0.3
 pytz==2018.4
 pyyaml==3.12
 raven[flask]==6.9.0
+requests-toolbelt==0.8.0
 requests[security]==2.11.1
 retrying==1.3.3
 s3transfer==0.1.13
@@ -83,6 +84,5 @@ sphinxcontrib-websupport==1.1.0  # via sphinx
 sqlalchemy-utils==0.33.3
 sqlalchemy==1.2.8
 tabulate==0.8.2
-tld==0.7.10
 werkzeug==0.14.1
 xmltodict==0.11.0

requirements.in

@@ -39,5 +39,4 @@ retrying
 six
 SQLAlchemy-Utils
 tabulate
-tld
 xmltodict