dyn support

lemur/authorities/service.py

@@ -111,7 +111,7 @@ def create(**kwargs):
     cert = upload(**kwargs)
     kwargs['authority_certificate'] = cert
     if kwargs.get('plugin', {}).get('plugin_options', []):
-        kwargs['options'] = json.dumps(kwargs.get('plugin', {}).get('plugin_options', []))
+        kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options'])
 
     authority = Authority(**kwargs)
     authority = database.create(authority)

lemur/exceptions.py

@@ -34,3 +34,7 @@ class AttrNotFound(LemurException):
 
 class InvalidConfiguration(Exception):
     pass
+
+
+class InvalidAuthority(Exception):
+    pass

lemur/pending_certificates/cli.py

@@ -5,14 +5,12 @@
 .. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
 """
 from flask_script import Manager
-from multiprocessing import Pool
 
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.users import service as user_service
 
 manager = Manager(usage="Handles pending certificate related tasks.")
-agents = 20
 
 
 # Need to call this multiple times and store status of the cert in DB. If it is being worked on by a worker, and which
@@ -55,31 +53,32 @@ def fetch(ids):
     )
 
 
-def fetch_all():
+@manager.command
+def fetch_all_acme():
     """
-    Attempt to get full certificates for each pending certificate listed.
+    Attempt to get full certificates for each pending certificate listed for ACME.
-
-    Args:
-        ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
-             `python manager.py pending_certs fetch -i 123 321 all`
     """
     pending_certs = pending_certificate_service.get_pending_certs('all')
     user = user_service.get_by_username('lemur')
     new = 0
     failed = 0
-    certs = authority.get_ordered_certificates(pending_certs)
+    authority = plugins.get("acme-issuer")
-    for cert in certs:
+    resolved_certs = authority.get_ordered_certificates(pending_certs)
-        authority = plugins.get(cert.authority.plugin_name)
+
-        real_cert = authority.get_ordered_certificate(cert)
+    for cert in resolved_certs:
+        real_cert = cert.get("cert")
+        # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
+        pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
+
         if real_cert:
             # If a real certificate was returned from issuer, then create it in Lemur and delete
             # the pending certificate
-            pending_certificate_service.create_certificate(cert, real_cert, user)
+            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
-            pending_certificate_service.delete(cert)
+            pending_certificate_service.delete_by_id(pending_cert.id)
             # add metrics to metrics extension
             new += 1
         else:
-            pending_certificate_service.increment_attempt(cert)
+            pending_certificate_service.increment_attempt(pending_cert)
             failed += 1
     print(
         "[+] Certificates: New: {new} Failed: {failed}".format(

lemur/pending_certificates/service.py

@@ -59,6 +59,10 @@ def delete(pending_certificate):
     database.delete(pending_certificate)
 
 
+def delete_by_id(id):
+    database.delete(get(id))
+
+
 def get_pending_certs(pending_ids):
     """
     Retrieve a list of pending certs given a list of ids

lemur/plugins/lemur_acme/dyn.py

@@ -0,0 +1,80 @@
+import dns.exception
+import dns.resolver
+import time
+
+from dyn.tm.session import DynectSession
+from dyn.tm.zones import Node, Zone
+from flask import current_app
+from tld import get_tld
+
+current_app.logger.debug("Logging in to Dyn API")
+
+dynect_session = DynectSession(
+    current_app.config.get('ACME_DYN_CUSTOMER_NAME', ''),
+    current_app.config.get('ACME_DYN_USERNAME', ''),
+    current_app.config.get('ACME_DYN_PASSWORD', ''),
+)
+
+
+def _has_dns_propagated(name, token):
+    txt_records = []
+    try:
+        dns_resolver = dns.resolver.Resolver()
+        dns_resolver.nameservers = ['8.8.8.8']
+        dns_response = dns_resolver.query(name, 'TXT')
+        for rdata in dns_response:
+            for txt_record in rdata.strings:
+                txt_records.append(txt_record.decode("utf-8"))
+    except dns.exception.DNSException:
+        return False
+
+    for txt_record in txt_records:
+        if txt_record == token:
+            return True
+
+    return False
+
+
+def wait_for_dns_change(change_id, account_number=None):
+    fqdn, token = change_id
+    while True:
+        status = _has_dns_propagated(fqdn, token)
+        current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
+        if status:
+            break
+        time.sleep(20)
+    return
+
+
+def create_txt_record(domain, token, account_number):
+    zone_name = get_tld('http://' + domain)
+    zone_parts = len(zone_name.split('.'))
+    node_name = '.'.join(domain.split('.')[:-zone_parts])
+    fqdn = "{0}.{1}".format(node_name, zone_name)
+    zone = Zone(zone_name)
+    zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
+    node = zone.get_node(node_name)
+    zone.publish()
+    current_app.logger.debug("TXT record created: {0}".format(fqdn))
+    change_id = (fqdn, token)
+    return change_id
+
+
+def delete_txt_record(change_id, account_number, domain, token):
+    if not domain:
+        current_app.logger.debug("delete_txt_record: No domain passed")
+        return
+
+    zone_name = get_tld('http://' + domain)
+    zone_parts = len(zone_name.split('.'))
+    node_name = '.'.join(domain.split('.')[:-zone_parts])
+    fqdn = "{0}.{1}".format(node_name, zone_name)
+
+    zone = Zone(zone_name)
+    node = Node(zone_name, fqdn)
+    all_txt_records = node.get_all_records_by_type('TXT')
+    for txt_record in all_txt_records:
+        if txt_record.txtdata == ("{}".format(token)):
+            current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn))
+            txt_record.delete()
+    zone.publish()

lemur/plugins/lemur_acme/plugin.py

@@ -17,8 +17,9 @@ import json
 from flask import current_app
 
 from acme.client import Client
-from acme import messages
+from acme import challenges, messages
-from acme import challenges
+from acme.errors import PollError
+from botocore.exceptions import ClientError
 
 from lemur.common.utils import generate_private_key
 
@@ -26,6 +27,7 @@ import OpenSSL.crypto
 
 from lemur.authorizations import service as authorization_service
 from lemur.dns_providers import service as dns_provider_service
+from lemur.exceptions import InvalidAuthority, InvalidConfiguration
 from lemur.plugins.bases import IssuerPlugin
 from lemur.plugins import lemur_acme as acme
 
@@ -68,6 +70,7 @@ def start_dns_challenge(acme_client, account_number, host, dns_provider):
 
 
 def complete_dns_challenge(acme_client, account_number, authz_record, dns_provider):
+    current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.host))
     dns_provider.wait_for_dns_change(authz_record.change_id, account_number=account_number)
 
     response = authz_record.dns_challenge.response(acme_client.key)
@@ -93,6 +96,8 @@ def request_certificate(acme_client, authorizations, csr):
             )
         ),
         authzrs=[authz_record.authz for authz_record in authorizations],
+        mintime=60,
+        max_attempts=10,
     )
 
     pem_certificate = OpenSSL.crypto.dump_certificate(
@@ -111,11 +116,11 @@ def request_certificate(acme_client, authorizations, csr):
 
 def setup_acme_client(authority):
     if not authority.options:
-        raise Exception("Invalid authority. Options not set")
+        raise InvalidAuthority("Invalid authority. Options not set")
     options = {}
 
     for option in json.loads(authority.options):
-        options[option.get("name")] = option.get("value")
+        options[option["name"]] = option.get("value")
     email = options.get('email', current_app.config.get('ACME_EMAIL'))
     tel = options.get('telephone', current_app.config.get('ACME_TEL'))
     directory_url = options.get('acme_url', current_app.config.get('ACME_DIRECTORY_URL'))
@@ -219,15 +224,29 @@ class ACMEIssuerPlugin(IssuerPlugin):
     def __init__(self, *args, **kwargs):
         super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
 
+    def get_dns_provider(self, type):
+        from lemur.plugins.lemur_acme import cloudflare, dyn, route53
+        provider_types = {
+            'cloudflare': cloudflare,
+            'dyn': dyn,
+            'route53': route53,
+        }
+        return provider_types[type]
+
     def get_ordered_certificate(self, pending_cert):
         acme_client, registration = setup_acme_client(pending_cert.authority)
         order_info = authorization_service.get(pending_cert.external_id)
         dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
-        dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
+        dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
+        try:
             authorizations = get_authorizations(
                 acme_client, order_info.account_number, order_info.domains, dns_provider_type)
+        except ClientError:
+            current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert.name), exc_info=True)
+            return False
 
-        finalize_authorizations(acme_client, order_info.account_number, dns_provider_type, authorizations)
+        authorizations = finalize_authorizations(
+            acme_client, order_info.account_number, dns_provider_type, authorizations)
         pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr)
         cert = {
             'body': "\n".join(str(pem_certificate).splitlines()),
@@ -238,11 +257,13 @@ class ACMEIssuerPlugin(IssuerPlugin):
 
     def get_ordered_certificates(self, pending_certs):
         pending = []
+        certs = []
         for pending_cert in pending_certs:
             acme_client, registration = setup_acme_client(pending_cert.authority)
             order_info = authorization_service.get(pending_cert.external_id)
             dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
-            dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
+            dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
+            try:
                 authorizations = get_authorizations(
                     acme_client, order_info.account_number, order_info.domains, dns_provider_type)
                 pending.append({
@@ -252,30 +273,42 @@ class ACMEIssuerPlugin(IssuerPlugin):
                     "authorizations": authorizations,
                     "pending_cert": pending_cert,
                 })
-
+            except (ClientError, ValueError):
-        certs = []
+                current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
+                certs.append({
+                    "cert": False,
+                    "pending_cert": pending_cert,
+                })
 
         for entry in pending:
-            finalize_authorizations(
+            entry["authorizations"] = finalize_authorizations(
-                pending["acme_client"],
+                entry["acme_client"],
-                pending["account_number"],
+                entry["account_number"],
-                pending["dns_provider_type"],
+                entry["dns_provider_type"],
-                pending["authorizations"]
+                entry["authorizations"]
             )
+
+            try:
                 pem_certificate, pem_certificate_chain = request_certificate(
-                pending["acme_client"],
+                    entry["acme_client"],
-                pending["authorizations"],
+                    entry["authorizations"],
-                pending["pending_cert"].csr
+                    entry["pending_cert"].csr
                 )
 
                 cert = {
                     'body': "\n".join(str(pem_certificate).splitlines()),
                     'chain': "\n".join(str(pem_certificate_chain).splitlines()),
-                'external_id': str(pending_cert.external_id)
+                    'external_id': str(entry["pending_cert"].external_id)
                 }
                 certs.append({
                     "cert": cert,
-                "pending_cert": pending_cert,
+                    "pending_cert": entry["pending_cert"],
+                })
+            except PollError:
+                current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
+                certs.append({
+                    "cert": False,
+                    "pending_cert": entry["pending_cert"],
                 })
         return certs
 
@@ -292,7 +325,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
         acme_client, registration = setup_acme_client(authority)
         dns_provider_d = issuer_options.get('dns_provider')
         if not dns_provider_d:
-            raise Exception("DNS Provider setting is required for ACME certificates.")
+            raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.")
         dns_provider = dns_provider_service.get(dns_provider_d.get("id"))
         credentials = json.loads(dns_provider.credentials)
 
@@ -300,9 +333,9 @@ class ACMEIssuerPlugin(IssuerPlugin):
         dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
         account_number = credentials.get("account_id")
         if dns_provider.provider_type == 'route53' and not account_number:
-            error = "DNS Provider {} does not have an account number configured.".format(dns_provider.name)
+            error = "Route53 DNS Provider {} does not have an account number configured.".format(dns_provider.name)
             current_app.logger.error(error)
-            raise Exception(error)
+            raise InvalidConfiguration(error)
         domains = get_domains(issuer_options)
         if not create_immediately:
             # Create pending authorizations that we'll need to do the creation
@@ -333,7 +366,11 @@ class ACMEIssuerPlugin(IssuerPlugin):
         :return:
         """
         role = {'username': '', 'password': '', 'name': 'acme'}
-        plugin_options = options.get('plugin').get('plugin_options')
+        plugin_options = options.get('plugin', {}).get('plugin_options')
+        if not plugin_options:
+            error = "Invalid options for lemur_acme plugin: {}".format(options)
+            current_app.logger.error(error)
+            raise InvalidConfiguration(error)
         # Define static acme_root based off configuration variable by default. However, if user has passed a
         # certificate, use this certificate as the root.
         acme_root = current_app.config.get('ACME_ROOT')

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -54,7 +54,8 @@ class TestAcme(unittest.TestCase):
         self.assertEqual(type(result), plugin.AuthorizationRecord)
 
     @patch('acme.client.Client')
-    def test_complete_dns_challenge_success(self, mock_acme):
+    @patch('lemur.plugins.lemur_acme.plugin.current_app')
+    def test_complete_dns_challenge_success(self, mock_current_app, mock_acme):
         mock_dns_provider = Mock()
         mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
 
@@ -65,7 +66,8 @@ class TestAcme(unittest.TestCase):
         plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
 
     @patch('acme.client.Client')
-    def test_complete_dns_challenge_fail(self, mock_acme):
+    @patch('lemur.plugins.lemur_acme.plugin.current_app')
+    def test_complete_dns_challenge_fail(self, mock_current_app, mock_acme):
         mock_dns_provider = Mock()
         mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
 

requirements-dev.txt

@@ -12,7 +12,7 @@ chardet==3.0.4            # via requests
 flake8==3.5.0
 identify==1.0.13          # via pre-commit
 idna==2.6                 # via requests
-invoke==0.22.1
+invoke==0.23.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.0
 pkginfo==1.4.2            # via twine
@@ -23,7 +23,7 @@ pyyaml==3.12              # via aspy.yaml, pre-commit
 requests-toolbelt==0.8.0  # via twine
 requests==2.18.4          # via requests-toolbelt, twine
 six==1.11.0               # via cfgv, pre-commit
-tqdm==4.23.1              # via twine
+tqdm==4.23.2              # via twine
 twine==1.11.0
 urllib3==1.22             # via requests
 virtualenv==15.2.0        # via pre-commit

requirements-docs.txt

@@ -15,13 +15,15 @@ asyncpool==1.0
 babel==2.5.3              # via sphinx
 bcrypt==3.1.4
 blinker==1.4
-boto3==1.7.10
+boto3==1.7.11
-botocore==1.10.10
+botocore==1.10.11
+certifi==2018.4.16
 cffi==1.11.5
 click==6.7
 cloudflare==2.1.0
 cryptography==2.2.2
 docutils==0.14
+dyn==1.8.1
 flask-bcrypt==0.7.1
 flask-cors==3.0.4
 flask-mail==0.9.1
@@ -34,11 +36,12 @@ flask==0.12
 future==0.16.0
 gevent==1.2.2
 greenlet==0.4.13
-gunicorn==19.7.1
+gunicorn==19.8.1
 idna==2.6
 imagesize==1.0.0          # via sphinx
 inflection==0.3.1
 itsdangerous==0.24
+janus==0.3.1
 jinja2==2.10
 jmespath==0.9.3
 josepy==1.1.0
@@ -74,11 +77,11 @@ retrying==1.3.3
 s3transfer==0.1.13
 six==1.11.0
 snowballstemmer==1.2.1    # via sphinx
-sphinx-rtd-theme==0.3.0
+sphinx-rtd-theme==0.3.1
 sphinx==1.7.4
 sphinxcontrib-httpdomain==1.6.1
 sphinxcontrib-websupport==1.0.1  # via sphinx
-sqlalchemy-utils==0.33.2
+sqlalchemy-utils==0.33.3
 sqlalchemy==1.2.7
 tabulate==0.8.2
 werkzeug==0.14.1

requirements-tests.txt

@@ -7,9 +7,9 @@
 asn1crypto==0.24.0        # via cryptography
 attrs==17.4.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.7.10             # via moto
+boto3==1.7.11             # via moto
 boto==2.48.0              # via moto
-botocore==1.10.10         # via boto3, moto, s3transfer
+botocore==1.10.11         # via boto3, moto, s3transfer
 certifi==2018.4.16        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -22,7 +22,7 @@ docker==3.3.0             # via moto
 docutils==0.14            # via botocore
 factory-boy==2.10.0
 faker==0.8.13
-flask==1.0                # via pytest-flask
+flask==1.0.2              # via pytest-flask
 freezegun==0.3.10
 idna==2.6                 # via cryptography, requests
 itsdangerous==0.24        # via flask
@@ -42,7 +42,7 @@ pyaml==17.12.1            # via moto
 pycparser==2.18           # via cffi
 pyflakes==1.6.0
 pytest-flask==0.10.0
-pytest-mock==1.9.0
+pytest-mock==1.10.0
 pytest==3.5.1
 python-dateutil==2.6.1    # via botocore, faker, freezegun, moto
 pytz==2018.4              # via moto

requirements.in

@@ -5,8 +5,11 @@ alembic-autogenerate-enums
 arrow
 asyncpool
 boto3
+certifi
 CloudFlare
 cryptography
+dnspython3
+dyn
 Flask-Bcrypt==0.7.1
 Flask-Mail==0.9.1
 Flask-Migrate==2.1.1
@@ -38,4 +41,5 @@ retrying
 six
 SQLAlchemy-Utils
 tabulate
+tld
 xmltodict

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --no-index --output-file requirements.txt requirements.in
 #
-acme==0.23.0
+acme==0.24.0
 alembic-autogenerate-enums==0.0.2
 alembic==0.9.9            # via flask-migrate
 aniso8601==3.0.0          # via flask-restful
@@ -13,13 +13,17 @@ asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.7.10
+boto3==1.7.11
-botocore==1.10.10         # via boto3, s3transfer
+botocore==1.10.11         # via boto3, s3transfer
+certifi==2018.4.16
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 click==6.7                # via flask
 cloudflare==2.1.0
 cryptography==2.2.2
+dnspython3==1.15.0
+dnspython==1.15.0         # via dnspython3
 docutils==0.14            # via botocore
+dyn==1.8.1
 flask-bcrypt==0.7.1
 flask-cors==3.0.4
 flask-mail==0.9.1
@@ -32,7 +36,7 @@ flask==0.12
 future==0.16.0
 gevent==1.2.2
 greenlet==0.4.13          # via gevent
-gunicorn==19.7.1
+gunicorn==19.8.1
 idna==2.6                 # via cryptography
 inflection==0.3.1
 itsdangerous==0.24        # via flask
@@ -69,8 +73,9 @@ requests[security]==2.11.1
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.11.0
-sqlalchemy-utils==0.33.2
+sqlalchemy-utils==0.33.3
 sqlalchemy==1.2.7         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
+tld==0.7.10
 werkzeug==0.14.1          # via flask
 xmltodict==0.11.0