From 1be3f8368f3288641121b3f58d4ad1e7a1f91af8 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 4 May 2018 15:00:43 -0700 Subject: [PATCH] dyn support --- lemur/authorities/service.py | 2 +- lemur/certificates/schemas.py | 2 +- lemur/exceptions.py | 4 + lemur/pending_certificates/cli.py | 29 +++-- lemur/pending_certificates/service.py | 4 + lemur/plugins/lemur_acme/dyn.py | 80 +++++++++++++ lemur/plugins/lemur_acme/plugin.py | 123 +++++++++++++------- lemur/plugins/lemur_acme/tests/test_acme.py | 6 +- requirements-dev.txt | 4 +- requirements-docs.txt | 13 ++- requirements-tests.txt | 8 +- requirements.in | 4 + requirements.txt | 15 ++- 13 files changed, 216 insertions(+), 78 deletions(-) create mode 100644 lemur/plugins/lemur_acme/dyn.py diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 8c80757d..fe90faea 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -111,7 +111,7 @@ def create(**kwargs): cert = upload(**kwargs) kwargs['authority_certificate'] = cert if kwargs.get('plugin', {}).get('plugin_options', []): - kwargs['options'] = json.dumps(kwargs.get('plugin', {}).get('plugin_options', [])) + kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options']) authority = Authority(**kwargs) authority = database.create(authority) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 6924674c..ac02e7e0 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -78,7 +78,7 @@ class CertificateInputSchema(CertificateCreationSchema): key_type = fields.String( validate=validate.OneOf( ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1', - 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1','ECCSECT163K1', 'ECCSECT233K1', + 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1', 'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2']), missing='RSA2048') diff --git a/lemur/exceptions.py b/lemur/exceptions.py index fea14e10..a9909e87 100644 --- a/lemur/exceptions.py +++ b/lemur/exceptions.py @@ -34,3 +34,7 @@ class AttrNotFound(LemurException): class InvalidConfiguration(Exception): pass + + +class InvalidAuthority(Exception): + pass diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index 5b48559a..5dad96b9 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -5,14 +5,12 @@ .. moduleauthor:: Curtis Castrapel """ from flask_script import Manager -from multiprocessing import Pool from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.users import service as user_service manager = Manager(usage="Handles pending certificate related tasks.") -agents = 20 # Need to call this multiple times and store status of the cert in DB. If it is being worked on by a worker, and which @@ -55,31 +53,32 @@ def fetch(ids): ) -def fetch_all(): +@manager.command +def fetch_all_acme(): """ - Attempt to get full certificates for each pending certificate listed. - - Args: - ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI) - `python manager.py pending_certs fetch -i 123 321 all` + Attempt to get full certificates for each pending certificate listed for ACME. """ pending_certs = pending_certificate_service.get_pending_certs('all') user = user_service.get_by_username('lemur') new = 0 failed = 0 - certs = authority.get_ordered_certificates(pending_certs) - for cert in certs: - authority = plugins.get(cert.authority.plugin_name) - real_cert = authority.get_ordered_certificate(cert) + authority = plugins.get("acme-issuer") + resolved_certs = authority.get_ordered_certificates(pending_certs) + + for cert in resolved_certs: + real_cert = cert.get("cert") + # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 + pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) + if real_cert: # If a real certificate was returned from issuer, then create it in Lemur and delete # the pending certificate - pending_certificate_service.create_certificate(cert, real_cert, user) - pending_certificate_service.delete(cert) + pending_certificate_service.create_certificate(pending_cert, real_cert, user) + pending_certificate_service.delete_by_id(pending_cert.id) # add metrics to metrics extension new += 1 else: - pending_certificate_service.increment_attempt(cert) + pending_certificate_service.increment_attempt(pending_cert) failed += 1 print( "[+] Certificates: New: {new} Failed: {failed}".format( diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 9046e0c8..f58871b7 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -59,6 +59,10 @@ def delete(pending_certificate): database.delete(pending_certificate) +def delete_by_id(id): + database.delete(get(id)) + + def get_pending_certs(pending_ids): """ Retrieve a list of pending certs given a list of ids diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py new file mode 100644 index 00000000..5d7abf66 --- /dev/null +++ b/lemur/plugins/lemur_acme/dyn.py @@ -0,0 +1,80 @@ +import dns.exception +import dns.resolver +import time + +from dyn.tm.session import DynectSession +from dyn.tm.zones import Node, Zone +from flask import current_app +from tld import get_tld + +current_app.logger.debug("Logging in to Dyn API") + +dynect_session = DynectSession( + current_app.config.get('ACME_DYN_CUSTOMER_NAME', ''), + current_app.config.get('ACME_DYN_USERNAME', ''), + current_app.config.get('ACME_DYN_PASSWORD', ''), +) + + +def _has_dns_propagated(name, token): + txt_records = [] + try: + dns_resolver = dns.resolver.Resolver() + dns_resolver.nameservers = ['8.8.8.8'] + dns_response = dns_resolver.query(name, 'TXT') + for rdata in dns_response: + for txt_record in rdata.strings: + txt_records.append(txt_record.decode("utf-8")) + except dns.exception.DNSException: + return False + + for txt_record in txt_records: + if txt_record == token: + return True + + return False + + +def wait_for_dns_change(change_id, account_number=None): + fqdn, token = change_id + while True: + status = _has_dns_propagated(fqdn, token) + current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) + if status: + break + time.sleep(20) + return + + +def create_txt_record(domain, token, account_number): + zone_name = get_tld('http://' + domain) + zone_parts = len(zone_name.split('.')) + node_name = '.'.join(domain.split('.')[:-zone_parts]) + fqdn = "{0}.{1}".format(node_name, zone_name) + zone = Zone(zone_name) + zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5) + node = zone.get_node(node_name) + zone.publish() + current_app.logger.debug("TXT record created: {0}".format(fqdn)) + change_id = (fqdn, token) + return change_id + + +def delete_txt_record(change_id, account_number, domain, token): + if not domain: + current_app.logger.debug("delete_txt_record: No domain passed") + return + + zone_name = get_tld('http://' + domain) + zone_parts = len(zone_name.split('.')) + node_name = '.'.join(domain.split('.')[:-zone_parts]) + fqdn = "{0}.{1}".format(node_name, zone_name) + + zone = Zone(zone_name) + node = Node(zone_name, fqdn) + all_txt_records = node.get_all_records_by_type('TXT') + for txt_record in all_txt_records: + if txt_record.txtdata == ("{}".format(token)): + current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn)) + txt_record.delete() + zone.publish() diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f5b4b12a..e95fe3b7 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -17,8 +17,9 @@ import json from flask import current_app from acme.client import Client -from acme import messages -from acme import challenges +from acme import challenges, messages +from acme.errors import PollError +from botocore.exceptions import ClientError from lemur.common.utils import generate_private_key @@ -26,6 +27,7 @@ import OpenSSL.crypto from lemur.authorizations import service as authorization_service from lemur.dns_providers import service as dns_provider_service +from lemur.exceptions import InvalidAuthority, InvalidConfiguration from lemur.plugins.bases import IssuerPlugin from lemur.plugins import lemur_acme as acme @@ -68,6 +70,7 @@ def start_dns_challenge(acme_client, account_number, host, dns_provider): def complete_dns_challenge(acme_client, account_number, authz_record, dns_provider): + current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.host)) dns_provider.wait_for_dns_change(authz_record.change_id, account_number=account_number) response = authz_record.dns_challenge.response(acme_client.key) @@ -93,6 +96,8 @@ def request_certificate(acme_client, authorizations, csr): ) ), authzrs=[authz_record.authz for authz_record in authorizations], + mintime=60, + max_attempts=10, ) pem_certificate = OpenSSL.crypto.dump_certificate( @@ -111,11 +116,11 @@ def request_certificate(acme_client, authorizations, csr): def setup_acme_client(authority): if not authority.options: - raise Exception("Invalid authority. Options not set") + raise InvalidAuthority("Invalid authority. Options not set") options = {} for option in json.loads(authority.options): - options[option.get("name")] = option.get("value") + options[option["name"]] = option.get("value") email = options.get('email', current_app.config.get('ACME_EMAIL')) tel = options.get('telephone', current_app.config.get('ACME_TEL')) directory_url = options.get('acme_url', current_app.config.get('ACME_DIRECTORY_URL')) @@ -219,15 +224,29 @@ class ACMEIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) + def get_dns_provider(self, type): + from lemur.plugins.lemur_acme import cloudflare, dyn, route53 + provider_types = { + 'cloudflare': cloudflare, + 'dyn': dyn, + 'route53': route53, + } + return provider_types[type] + def get_ordered_certificate(self, pending_cert): acme_client, registration = setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) - dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1) - authorizations = get_authorizations( - acme_client, order_info.account_number, order_info.domains, dns_provider_type) + dns_provider_type = self.get_dns_provider(dns_provider.provider_type) + try: + authorizations = get_authorizations( + acme_client, order_info.account_number, order_info.domains, dns_provider_type) + except ClientError: + current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert.name), exc_info=True) + return False - finalize_authorizations(acme_client, order_info.account_number, dns_provider_type, authorizations) + authorizations = finalize_authorizations( + acme_client, order_info.account_number, dns_provider_type, authorizations) pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr) cert = { 'body': "\n".join(str(pem_certificate).splitlines()), @@ -238,45 +257,59 @@ class ACMEIssuerPlugin(IssuerPlugin): def get_ordered_certificates(self, pending_certs): pending = [] + certs = [] for pending_cert in pending_certs: acme_client, registration = setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) - dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1) - authorizations = get_authorizations( - acme_client, order_info.account_number, order_info.domains, dns_provider_type) - pending.append({ - "acme_client": acme_client, - "account_number": order_info.account_number, - "dns_provider_type": dns_provider_type, - "authorizations": authorizations, - "pending_cert": pending_cert, - }) - - certs = [] + dns_provider_type = self.get_dns_provider(dns_provider.provider_type) + try: + authorizations = get_authorizations( + acme_client, order_info.account_number, order_info.domains, dns_provider_type) + pending.append({ + "acme_client": acme_client, + "account_number": order_info.account_number, + "dns_provider_type": dns_provider_type, + "authorizations": authorizations, + "pending_cert": pending_cert, + }) + except (ClientError, ValueError): + current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) + certs.append({ + "cert": False, + "pending_cert": pending_cert, + }) for entry in pending: - finalize_authorizations( - pending["acme_client"], - pending["account_number"], - pending["dns_provider_type"], - pending["authorizations"] - ) - pem_certificate, pem_certificate_chain = request_certificate( - pending["acme_client"], - pending["authorizations"], - pending["pending_cert"].csr + entry["authorizations"] = finalize_authorizations( + entry["acme_client"], + entry["account_number"], + entry["dns_provider_type"], + entry["authorizations"] ) - cert = { - 'body': "\n".join(str(pem_certificate).splitlines()), - 'chain': "\n".join(str(pem_certificate_chain).splitlines()), - 'external_id': str(pending_cert.external_id) - } - certs.append({ - "cert": cert, - "pending_cert": pending_cert, - }) + try: + pem_certificate, pem_certificate_chain = request_certificate( + entry["acme_client"], + entry["authorizations"], + entry["pending_cert"].csr + ) + + cert = { + 'body': "\n".join(str(pem_certificate).splitlines()), + 'chain': "\n".join(str(pem_certificate_chain).splitlines()), + 'external_id': str(entry["pending_cert"].external_id) + } + certs.append({ + "cert": cert, + "pending_cert": entry["pending_cert"], + }) + except PollError: + current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) + certs.append({ + "cert": False, + "pending_cert": entry["pending_cert"], + }) return certs def create_certificate(self, csr, issuer_options): @@ -292,7 +325,7 @@ class ACMEIssuerPlugin(IssuerPlugin): acme_client, registration = setup_acme_client(authority) dns_provider_d = issuer_options.get('dns_provider') if not dns_provider_d: - raise Exception("DNS Provider setting is required for ACME certificates.") + raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.") dns_provider = dns_provider_service.get(dns_provider_d.get("id")) credentials = json.loads(dns_provider.credentials) @@ -300,9 +333,9 @@ class ACMEIssuerPlugin(IssuerPlugin): dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1) account_number = credentials.get("account_id") if dns_provider.provider_type == 'route53' and not account_number: - error = "DNS Provider {} does not have an account number configured.".format(dns_provider.name) + error = "Route53 DNS Provider {} does not have an account number configured.".format(dns_provider.name) current_app.logger.error(error) - raise Exception(error) + raise InvalidConfiguration(error) domains = get_domains(issuer_options) if not create_immediately: # Create pending authorizations that we'll need to do the creation @@ -333,7 +366,11 @@ class ACMEIssuerPlugin(IssuerPlugin): :return: """ role = {'username': '', 'password': '', 'name': 'acme'} - plugin_options = options.get('plugin').get('plugin_options') + plugin_options = options.get('plugin', {}).get('plugin_options') + if not plugin_options: + error = "Invalid options for lemur_acme plugin: {}".format(options) + current_app.logger.error(error) + raise InvalidConfiguration(error) # Define static acme_root based off configuration variable by default. However, if user has passed a # certificate, use this certificate as the root. acme_root = current_app.config.get('ACME_ROOT') diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 45758a68..b148f1c7 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -54,7 +54,8 @@ class TestAcme(unittest.TestCase): self.assertEqual(type(result), plugin.AuthorizationRecord) @patch('acme.client.Client') - def test_complete_dns_challenge_success(self, mock_acme): + @patch('lemur.plugins.lemur_acme.plugin.current_app') + def test_complete_dns_challenge_success(self, mock_current_app, mock_acme): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -65,7 +66,8 @@ class TestAcme(unittest.TestCase): plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider) @patch('acme.client.Client') - def test_complete_dns_challenge_fail(self, mock_acme): + @patch('lemur.plugins.lemur_acme.plugin.current_app') + def test_complete_dns_challenge_fail(self, mock_current_app, mock_acme): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7471ccbc..7047b699 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,7 +12,7 @@ chardet==3.0.4 # via requests flake8==3.5.0 identify==1.0.13 # via pre-commit idna==2.6 # via requests -invoke==0.22.1 +invoke==0.23.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.0 pkginfo==1.4.2 # via twine @@ -23,7 +23,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.18.4 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -tqdm==4.23.1 # via twine +tqdm==4.23.2 # via twine twine==1.11.0 urllib3==1.22 # via requests virtualenv==15.2.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 10193b47..17b86edb 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,13 +15,15 @@ asyncpool==1.0 babel==2.5.3 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.10 -botocore==1.10.10 +boto3==1.7.11 +botocore==1.10.11 +certifi==2018.4.16 cffi==1.11.5 click==6.7 cloudflare==2.1.0 cryptography==2.2.2 docutils==0.14 +dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.4 flask-mail==0.9.1 @@ -34,11 +36,12 @@ flask==0.12 future==0.16.0 gevent==1.2.2 greenlet==0.4.13 -gunicorn==19.7.1 +gunicorn==19.8.1 idna==2.6 imagesize==1.0.0 # via sphinx inflection==0.3.1 itsdangerous==0.24 +janus==0.3.1 jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 @@ -74,11 +77,11 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.3.0 +sphinx-rtd-theme==0.3.1 sphinx==1.7.4 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx -sqlalchemy-utils==0.33.2 +sqlalchemy-utils==0.33.3 sqlalchemy==1.2.7 tabulate==0.8.2 werkzeug==0.14.1 diff --git a/requirements-tests.txt b/requirements-tests.txt index 8d04ddcf..54ae9226 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,9 +7,9 @@ asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.10 # via moto +boto3==1.7.11 # via moto boto==2.48.0 # via moto -botocore==1.10.10 # via boto3, moto, s3transfer +botocore==1.10.11 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -22,7 +22,7 @@ docker==3.3.0 # via moto docutils==0.14 # via botocore factory-boy==2.10.0 faker==0.8.13 -flask==1.0 # via pytest-flask +flask==1.0.2 # via pytest-flask freezegun==0.3.10 idna==2.6 # via cryptography, requests itsdangerous==0.24 # via flask @@ -42,7 +42,7 @@ pyaml==17.12.1 # via moto pycparser==2.18 # via cffi pyflakes==1.6.0 pytest-flask==0.10.0 -pytest-mock==1.9.0 +pytest-mock==1.10.0 pytest==3.5.1 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.4 # via moto diff --git a/requirements.in b/requirements.in index 1bd54d20..5acccc62 100644 --- a/requirements.in +++ b/requirements.in @@ -5,8 +5,11 @@ alembic-autogenerate-enums arrow asyncpool boto3 +certifi CloudFlare cryptography +dnspython3 +dyn Flask-Bcrypt==0.7.1 Flask-Mail==0.9.1 Flask-Migrate==2.1.1 @@ -38,4 +41,5 @@ retrying six SQLAlchemy-Utils tabulate +tld xmltodict diff --git a/requirements.txt b/requirements.txt index 5351ee50..e880bb02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.23.0 +acme==0.24.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -13,13 +13,17 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.10 -botocore==1.10.10 # via boto3, s3transfer +boto3==1.7.11 +botocore==1.10.11 # via boto3, s3transfer +certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask cloudflare==2.1.0 cryptography==2.2.2 +dnspython3==1.15.0 +dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore +dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.4 flask-mail==0.9.1 @@ -32,7 +36,7 @@ flask==0.12 future==0.16.0 gevent==1.2.2 greenlet==0.4.13 # via gevent -gunicorn==19.7.1 +gunicorn==19.8.1 idna==2.6 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask @@ -69,8 +73,9 @@ requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.2 +sqlalchemy-utils==0.33.3 sqlalchemy==1.2.7 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 +tld==0.7.10 werkzeug==0.14.1 # via flask xmltodict==0.11.0