From 28614b579380811a9a3a794a9df791ce4c883a2e Mon Sep 17 00:00:00 2001 From: Dmitry Zykov Date: Wed, 4 Apr 2018 14:49:25 +0300 Subject: [PATCH 01/18] remove linuxdst plugin --- docs/developer/internals/lemur.rst | 1 - lemur/plugins/lemur_linuxdst/__init__.py | 5 -- lemur/plugins/lemur_linuxdst/plugin.py | 83 --------------------- lemur/plugins/lemur_linuxdst/readme.txt | 8 -- lemur/plugins/lemur_linuxdst/remote_host.py | 73 ------------------ 5 files changed, 170 deletions(-) delete mode 100644 lemur/plugins/lemur_linuxdst/__init__.py delete mode 100644 lemur/plugins/lemur_linuxdst/plugin.py delete mode 100644 lemur/plugins/lemur_linuxdst/readme.txt delete mode 100644 lemur/plugins/lemur_linuxdst/remote_host.py diff --git a/docs/developer/internals/lemur.rst b/docs/developer/internals/lemur.rst index 7faacfac..b6517e4b 100644 --- a/docs/developer/internals/lemur.rst +++ b/docs/developer/internals/lemur.rst @@ -110,6 +110,5 @@ Subpackages lemur.plugins.lemur_digicert lemur.plugins.lemur_java lemur.plugins.lemur_kubernetes - lemur.plugins.lemur_linuxdst lemur.plugins.lemur_openssl lemur.plugins.lemur_slack diff --git a/lemur/plugins/lemur_linuxdst/__init__.py b/lemur/plugins/lemur_linuxdst/__init__.py deleted file mode 100644 index 8ce5a7f3..00000000 --- a/lemur/plugins/lemur_linuxdst/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -try: - VERSION = __import__('pkg_resources') \ - .get_distribution(__name__).version -except Exception as e: - VERSION = 'unknown' diff --git a/lemur/plugins/lemur_linuxdst/plugin.py b/lemur/plugins/lemur_linuxdst/plugin.py deleted file mode 100644 index 71b32551..00000000 --- a/lemur/plugins/lemur_linuxdst/plugin.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/python -from lemur.plugins.bases import DestinationPlugin -from lemur.plugins.lemur_linuxdst import remote_host - - -class LinuxDstPlugin(DestinationPlugin): - title = 'Linux Destination Plugin' - slug = 'linux-destination' - description = 'Allow the distribution of certificates to a Linux host' - version = 1 - - author = 'Rick Breidenstein ' - author_url = 'https://github.com/RickB17/' - - options = [ - { - 'name': 'dstHost', - 'type': 'str', - 'required': True, - 'helpMessage': 'This is the host you will be sending the certificate to', - }, - { - 'name': 'dstPort', - 'type': 'int', - 'required': True, - 'helpMessage': 'This is the port SSHD is running on', - 'default': '22' - }, - { - 'name': 'dstUser', - 'type': 'str', - 'required': True, - 'helpMessage': 'The user name to use on the remote host. Hopefully not root.', - 'default': 'root', - }, - { - 'name': 'dstPriv', - 'type': 'str', - 'required': True, - 'helpMessage': 'The private key to use for auth', - 'default': '/root/.ssh/id_rsa', - }, - { - 'name': 'dstPrivKey', - 'type': 'str', - 'required': False, - 'helpMessage': 'The password for the destination private key', - 'default': 'somethingsecret', - }, - { - 'name': 'dstDir', - 'type': 'str', - 'required': True, - 'helpMessage': 'This is the directory on the host you want to send the certificate to', - 'default': '/etc/nginx/certs/' - }, - { - "available": [ - "NGINX", - "3File" - ], - "name": "exportType", - "required": True, - "value": "NGINX", - "helpMessage": "Reference the docs for an explanation of each export type", - "type": "select" - } - ] - requires_key = False - - def upload(self, name, body, private_key, cert_chain, options, **kwargs): - export_type = self.get_option('exportType', options) - dst_host = self.get_option('dstHost', options) - dst_host_port = self.get_option('dstPort', options) - dst_user = self.get_option('dstUser', options) - dst_priv = self.get_option('dstPriv', options) - dst_priv_key = self.get_option('dstPrivKey', options) - - if dst_priv_key: - dst_priv_key = None - - dst_dir = self.get_option('dstDir', options) - remote_host.create_cert(name, dst_dir, export_type, dst_user, dst_priv, dst_priv_key, dst_host, int(dst_host_port)) diff --git a/lemur/plugins/lemur_linuxdst/readme.txt b/lemur/plugins/lemur_linuxdst/readme.txt deleted file mode 100644 index 944cda9a..00000000 --- a/lemur/plugins/lemur_linuxdst/readme.txt +++ /dev/null @@ -1,8 +0,0 @@ -NOTE: AS OF 4/25/2017 DATA IN THIS PLUGIN IS NOT ENCRYPTED AT REST -example: If you input a password for a private key, it is stored in clear text - -add this to the lemur.plugins entry_points in /www/lemur/setup.py - -'linux_destination = lemur.plugins.lemur_linuxdst.plugin:LinuxDstPlugin', - -note: DO NOT FORGET TO EXECUTE 'make release' and restart your service after modifying the setup.py diff --git a/lemur/plugins/lemur_linuxdst/remote_host.py b/lemur/plugins/lemur_linuxdst/remote_host.py deleted file mode 100644 index b0f2cdb0..00000000 --- a/lemur/plugins/lemur_linuxdst/remote_host.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python -from lemur.certificates import service -import paramiko -import stat - - -def copy_cert(cert_cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_port, dst_dir, dst_file, dst_data): - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # include the private key password if required - if dst_priv_key is None: - priv_key = paramiko.RSAKey.from_private_key_file(dst_priv) - else: - priv_key = paramiko.RSAKey.from_private_key_file(dst_priv, dst_priv_key) - - # open the sftp connection - ssh.connect(dst_host, username=dst_user, port=dst_port, pkey=priv_key) - sftp = ssh.open_sftp() - - # make the directory on the destination server - # files will be in a folder based on the cert_cn - # example: - # destination folder: /etc/nginx/certs/ - # files will go in: /etc/nginx/certs/your.cn.com/cert.pem - try: - sftp.mkdir(dst_dir) - except IOError: - pass - try: - dst_dir_cn = dst_dir + '/' + cert_cn - sftp.mkdir(dst_dir_cn) - except IOError: - pass - - cert_out = sftp.open(dst_dir_cn + '/' + dst_file, 'w') - cert_out.write(dst_data) - cert_out.close() - sftp.chmod(dst_dir_cn + '/' + dst_file, (stat.S_IRUSR)) - ssh.close() - - -def create_cert(name, dst_dir, export_type, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port): - lem_cert = service.get_by_name(name) - dst_file = 'cert.pem' - chain_req = False - - if export_type == 'NGINX': - # This process will result in a cert.pem file with the body and chain in a single file - if lem_cert.chain is None: - dst_data = lem_cert.body - else: - dst_data = lem_cert.body + '\n' + lem_cert.chain - chain_req = False - - elif export_type == '3File': - # This process will results in three files. cert.pem, priv.key, chain.pem - dst_data = lem_cert.body - chain_req = True - - else: - dst_data = lem_cert.body - - copy_cert(lem_cert.cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port, dst_dir, dst_file, dst_data) - - if chain_req is True: - dst_file = 'chain.pem' - dst_data = lem_cert.chain_req - copy_cert(lem_cert.cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port, dst_dir, dst_file, dst_data) - - dst_file = 'priv.key' - dst_data = lem_cert.private_key - copy_cert(lem_cert.cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port, dst_dir, dst_file, dst_data) From 5e964fad3915e65c5f078ffe7dcf662611f11103 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 12 Jun 2018 07:35:46 -0700 Subject: [PATCH 02/18] update requirements --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 6 +++--- requirements-tests.txt | 8 ++++---- requirements.txt | 15 ++++++++------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0fd765d0..58499408 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,13 +10,13 @@ certifi==2018.4.16 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.18 # via pre-commit +identify==1.1.0 # via pre-commit idna==2.6 # via requests invoke==1.0.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.0 pkginfo==1.4.2 # via twine -pre-commit==1.10.1 +pre-commit==1.10.2 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.12 # via aspy.yaml, pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 4e96643b..8795a511 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.31 -botocore==1.10.31 +boto3==1.7.32 +botocore==1.10.32 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -76,7 +76,7 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.3.1 +sphinx-rtd-theme==0.4.0 sphinx==1.7.5 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.1.0 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index bca2a199..efcfeac3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.32 # via moto +boto3==1.7.36 # via moto boto==2.48.0 # via moto -botocore==1.10.32 # via boto3, moto, s3transfer +botocore==1.10.36 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -18,7 +18,7 @@ click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto -docker-pycreds==0.2.3 # via docker +docker-pycreds==0.3.0 # via docker docker==3.3.0 # via moto docutils==0.14 # via botocore factory-boy==2.11.1 @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.txt b/requirements.txt index 9e8ec438..712a88f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.24.0 +acme==0.25.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.32 -botocore==1.10.32 # via boto3, s3transfer +boto3==1.7.36 +botocore==1.10.36 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -25,7 +25,7 @@ dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.4 +flask-cors==3.0.6 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -35,7 +35,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.8.1 -idna==2.6 # via cryptography +idna==2.7 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 @@ -50,7 +50,7 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 # via python-ldap @@ -59,13 +59,14 @@ pycparser==2.18 # via cffi pyjwt==1.6.4 pynacl==1.2.1 # via paramiko pyopenssl==17.2.0 -pyrfc3339==1.0 # via acme +pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 pytz==2018.4 # via acme, flask-restful, pyrfc3339 pyyaml==3.12 # via cloudflare raven[flask]==6.9.0 +requests-toolbelt==0.8.0 # via acme requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 From b31c7357ed06b56b16dfae109740ee242509e15a Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 12 Jun 2018 07:35:46 -0700 Subject: [PATCH 03/18] update requirements --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 6 +++--- requirements-tests.txt | 8 ++++---- requirements.txt | 15 ++++++++------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0fd765d0..58499408 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,13 +10,13 @@ certifi==2018.4.16 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.18 # via pre-commit +identify==1.1.0 # via pre-commit idna==2.6 # via requests invoke==1.0.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.0 pkginfo==1.4.2 # via twine -pre-commit==1.10.1 +pre-commit==1.10.2 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.12 # via aspy.yaml, pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 4e96643b..8795a511 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.31 -botocore==1.10.31 +boto3==1.7.32 +botocore==1.10.32 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -76,7 +76,7 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.3.1 +sphinx-rtd-theme==0.4.0 sphinx==1.7.5 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.1.0 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index bca2a199..efcfeac3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.32 # via moto +boto3==1.7.36 # via moto boto==2.48.0 # via moto -botocore==1.10.32 # via boto3, moto, s3transfer +botocore==1.10.36 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -18,7 +18,7 @@ click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto -docker-pycreds==0.2.3 # via docker +docker-pycreds==0.3.0 # via docker docker==3.3.0 # via moto docutils==0.14 # via botocore factory-boy==2.11.1 @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.txt b/requirements.txt index 9e8ec438..712a88f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.24.0 +acme==0.25.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.32 -botocore==1.10.32 # via boto3, s3transfer +boto3==1.7.36 +botocore==1.10.36 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -25,7 +25,7 @@ dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.4 +flask-cors==3.0.6 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -35,7 +35,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.8.1 -idna==2.6 # via cryptography +idna==2.7 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 @@ -50,7 +50,7 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 # via python-ldap @@ -59,13 +59,14 @@ pycparser==2.18 # via cffi pyjwt==1.6.4 pynacl==1.2.1 # via paramiko pyopenssl==17.2.0 -pyrfc3339==1.0 # via acme +pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 pytz==2018.4 # via acme, flask-restful, pyrfc3339 pyyaml==3.12 # via cloudflare raven[flask]==6.9.0 +requests-toolbelt==0.8.0 # via acme requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 From d72792ff3702b81ed4260b187b0eb5da4898841c Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 13 Jun 2018 14:08:39 -0700 Subject: [PATCH 04/18] Fix unique dyn situation where zone does not match tld, and there's a deeper zone --- lemur/plugins/lemur_acme/dyn.py | 127 ++++++++++++++++++++++---------- requirements-dev.txt | 8 +- requirements-docs.txt | 15 ++-- requirements-tests.txt | 10 +-- requirements.in | 1 - requirements.txt | 7 +- 6 files changed, 110 insertions(+), 58 deletions(-) diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 28832cbc..09a53884 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -5,11 +5,10 @@ import dns.exception import dns.name import dns.query import dns.resolver -from dyn.tm.errors import DynectCreateError +from dyn.tm.errors import DynectCreateError, DynectGetError from dyn.tm.session import DynectSession -from dyn.tm.zones import Node, Zone +from dyn.tm.zones import Node, Zone, get_all_zones from flask import current_app -from tld import get_tld def get_dynect_session(): @@ -51,19 +50,43 @@ def wait_for_dns_change(change_id, account_number=None): return +def get_zone_name(domain): + zones = get_all_zones() + + zone_name = "" + + for z in zones: + if domain.endswith(z.name): + # Find the most specific zone possible for the domain + # Ex: If fqdn is a.b.c.com, there is a zone for c.com, + # and a zone for b.c.com, we want to use b.c.com. + if z.name.count(".") > zone_name.count("."): + zone_name = z.name + if not zone_name: + raise Exception("No Dyn zone found for domain: {}".format(domain)) + return zone_name + + def create_txt_record(domain, token, account_number): get_dynect_session() - zone_name = get_tld('http://' + domain) + zone_name = get_zone_name(domain) zone_parts = len(zone_name.split('.')) node_name = '.'.join(domain.split('.')[:-zone_parts]) fqdn = "{0}.{1}".format(node_name, zone_name) zone = Zone(zone_name) try: - zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5) - except DynectCreateError: - delete_txt_record(None, None, domain, token) - zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5) - node = zone.get_node(node_name) + # Delete all stale ACME TXT records + delete_acme_txt_records(domain) + except DynectGetError as e: + if ( + "No such zone." in e.message or + "Host is not in this zone" in e.message or + "Host not found in this zone" in e.message + ): + current_app.logger.debug("Unable to delete ACME TXT records. They probably don't exist yet: {}".format(e)) + else: + raise + zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5) zone.publish() current_app.logger.debug("TXT record created: {0}".format(fqdn)) change_id = (fqdn, token) @@ -76,7 +99,7 @@ def delete_txt_record(change_id, account_number, domain, token): current_app.logger.debug("delete_txt_record: No domain passed") return - zone_name = get_tld('http://' + domain) + zone_name = get_zone_name(domain) zone_parts = len(zone_name.split('.')) node_name = '.'.join(domain.split('.')[:-zone_parts]) fqdn = "{0}.{1}".format(node_name, zone_name) @@ -92,40 +115,70 @@ def delete_txt_record(change_id, account_number, domain, token): zone.publish() +def delete_acme_txt_records(domain): + get_dynect_session() + if not domain: + current_app.logger.debug("delete_acme_txt_records: No domain passed") + return + acme_challenge_string = "_acme-challenge" + if not domain.startswith(acme_challenge_string): + current_app.logger.debug( + "delete_acme_txt_records: Domain {} doesn't start with string {}. " + "Cowardly refusing to delete TXT records".format(domain, acme_challenge_string)) + return + + zone_name = get_zone_name(domain) + zone_parts = len(zone_name.split('.')) + node_name = '.'.join(domain.split('.')[:-zone_parts]) + fqdn = "{0}.{1}".format(node_name, zone_name) + + zone = Zone(zone_name) + node = Node(zone_name, fqdn) + + all_txt_records = node.get_all_records_by_type('TXT') + for txt_record in all_txt_records: + current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn)) + txt_record.delete() + zone.publish() + + def get_authoritative_nameserver(domain): - n = dns.name.from_text(domain) + if current_app.config.get('ACME_DYN_GET_AUTHORATATIVE_NAMESERVER'): + n = dns.name.from_text(domain) - depth = 2 - default = dns.resolver.get_default_resolver() - nameserver = default.nameservers[0] + depth = 2 + default = dns.resolver.get_default_resolver() + nameserver = default.nameservers[0] - last = False - while not last: - s = n.split(depth) + last = False + while not last: + s = n.split(depth) - last = s[0].to_unicode() == u'@' - sub = s[1] + last = s[0].to_unicode() == u'@' + sub = s[1] - query = dns.message.make_query(sub, dns.rdatatype.NS) - response = dns.query.udp(query, nameserver) + query = dns.message.make_query(sub, dns.rdatatype.NS) + response = dns.query.udp(query, nameserver) - rcode = response.rcode() - if rcode != dns.rcode.NOERROR: - if rcode == dns.rcode.NXDOMAIN: - raise Exception('%s does not exist.' % sub) + rcode = response.rcode() + if rcode != dns.rcode.NOERROR: + if rcode == dns.rcode.NXDOMAIN: + raise Exception('%s does not exist.' % sub) + else: + raise Exception('Error %s' % dns.rcode.to_text(rcode)) + + if len(response.authority) > 0: + rrset = response.authority[0] else: - raise Exception('Error %s' % dns.rcode.to_text(rcode)) + rrset = response.answer[0] - if len(response.authority) > 0: - rrset = response.authority[0] - else: - rrset = response.answer[0] + rr = rrset[0] + if rr.rdtype != dns.rdatatype.SOA: + authority = rr.target + nameserver = default.query(authority).rrset[0].to_text() - rr = rrset[0] - if rr.rdtype != dns.rdatatype.SOA: - authority = rr.target - nameserver = default.query(authority).rrset[0].to_text() + depth += 1 - depth += 1 - - return nameserver + return nameserver + else: + return "8.8.8.8" diff --git a/requirements-dev.txt b/requirements-dev.txt index 58499408..a66b38af 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,20 +11,20 @@ cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 identify==1.1.0 # via pre-commit -idna==2.6 # via requests +idna==2.7 # via requests invoke==1.0.0 mccabe==0.6.1 # via flake8 -nodeenv==1.3.0 +nodeenv==1.3.1 pkginfo==1.4.2 # via twine pre-commit==1.10.2 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine -requests==2.18.4 # via requests-toolbelt, twine +requests==2.19.0 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit toml==0.9.4 # via pre-commit tqdm==4.23.4 # via twine twine==1.11.0 -urllib3==1.22 # via requests +urllib3==1.23 # via requests virtualenv==16.0.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 8795a511..58646db7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.24.0 +acme==0.25.0 alabaster==0.7.10 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==0.9.9 @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.32 -botocore==1.10.32 +boto3==1.7.36 +botocore==1.10.36 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -27,7 +27,7 @@ dnspython==1.15.0 docutils==0.14 dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.4 +flask-cors==3.0.6 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -37,7 +37,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.8.1 -idna==2.6 +idna==2.7 imagesize==1.0.0 # via sphinx inflection==0.3.1 itsdangerous==0.24 @@ -54,7 +54,7 @@ mock==2.0.0 ndg-httpsclient==0.5.0 packaging==17.1 # via sphinx paramiko==2.4.1 -pbr==4.0.3 +pbr==4.0.4 pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 @@ -65,12 +65,13 @@ pyjwt==1.6.4 pynacl==1.2.1 pyopenssl==17.2.0 pyparsing==2.2.0 # via packaging -pyrfc3339==1.0 +pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 pytz==2018.4 pyyaml==3.12 raven[flask]==6.9.0 +requests-toolbelt==0.8.0 requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 diff --git a/requirements-tests.txt b/requirements-tests.txt index efcfeac3..ccfad67f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.36 # via moto +boto3==1.7.37 # via moto boto==2.48.0 # via moto -botocore==1.10.36 # via boto3, moto, s3transfer +botocore==1.10.37 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -25,7 +25,7 @@ factory-boy==2.11.1 faker==0.8.15 flask==1.0.2 # via pytest-flask freezegun==0.3.10 -idna==2.6 # via cryptography, requests +idna==2.7 # via cryptography, requests itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore @@ -49,12 +49,12 @@ python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.4 # via moto pyyaml==3.12 # via pyaml requests-mock==1.5.0 -requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses +requests==2.19.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.22 # via requests +urllib3==1.23 # via requests websocket-client==0.48.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.10.11 # via aws-xray-sdk diff --git a/requirements.in b/requirements.in index 4d287c0d..2a5051a7 100644 --- a/requirements.in +++ b/requirements.in @@ -39,5 +39,4 @@ retrying six SQLAlchemy-Utils tabulate -tld xmltodict diff --git a/requirements.txt b/requirements.txt index 712a88f6..d10c92ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.25.0 +acme==0.25.1 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.36 -botocore==1.10.36 # via boto3, s3transfer +boto3==1.7.37 +botocore==1.10.37 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -74,6 +74,5 @@ six==1.11.0 sqlalchemy-utils==0.33.3 sqlalchemy==1.2.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 -tld==0.7.10 werkzeug==0.14.1 # via flask xmltodict==0.11.0 From 065e0edc5f4f6ad10e3cc768d93f1e200f1feffe Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 13 Jun 2018 14:22:45 -0700 Subject: [PATCH 05/18] lint --- lemur/plugins/lemur_acme/dyn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 09a53884..763436a1 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -5,7 +5,7 @@ import dns.exception import dns.name import dns.query import dns.resolver -from dyn.tm.errors import DynectCreateError, DynectGetError +from dyn.tm.errors import DynectGetError from dyn.tm.session import DynectSession from dyn.tm.zones import Node, Zone, get_all_zones from flask import current_app From 76621e497fb839c48013b879e1ccfb861ee0697e Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 13 Jun 2018 14:27:50 -0700 Subject: [PATCH 06/18] boto update. They updated between this and the last change --- requirements-docs.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 58646db7..9312142a 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.25.0 +acme==0.25.1 alabaster==0.7.10 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==0.9.9 @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.36 -botocore==1.10.36 +boto3==1.7.37 +botocore==1.10.37 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -84,6 +84,5 @@ sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 sqlalchemy==1.2.8 tabulate==0.8.2 -tld==0.7.10 werkzeug==0.14.1 xmltodict==0.11.0 From 135f2b710c2471612e5e3555f0730298f5d54718 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 13 Jun 2018 15:14:48 -0700 Subject: [PATCH 07/18] Limit dns queries to 10 attempts --- lemur/plugins/lemur_acme/dyn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 763436a1..6ce117bf 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -41,12 +41,15 @@ def _has_dns_propagated(name, token): def wait_for_dns_change(change_id, account_number=None): fqdn, token = change_id - while True: + number_of_attempts = 10 + for attempts in range(0, number_of_attempts): status = _has_dns_propagated(fqdn, token) current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) if status: break time.sleep(20) + if not status: + raise Exception("Unable to query DNS token for fqdn {}.".format(fqdn)) return From b1ce4d630dc3bc2b30769eb40ec577322b610c90 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 12 Jun 2018 07:35:46 -0700 Subject: [PATCH 08/18] update requirements --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 6 +++--- requirements-tests.txt | 8 ++++---- requirements.txt | 15 ++++++++------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0fd765d0..58499408 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,13 +10,13 @@ certifi==2018.4.16 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.18 # via pre-commit +identify==1.1.0 # via pre-commit idna==2.6 # via requests invoke==1.0.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.0 pkginfo==1.4.2 # via twine -pre-commit==1.10.1 +pre-commit==1.10.2 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.12 # via aspy.yaml, pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 4e96643b..8795a511 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.31 -botocore==1.10.31 +boto3==1.7.32 +botocore==1.10.32 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -76,7 +76,7 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.3.1 +sphinx-rtd-theme==0.4.0 sphinx==1.7.5 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.1.0 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index bca2a199..efcfeac3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.32 # via moto +boto3==1.7.36 # via moto boto==2.48.0 # via moto -botocore==1.10.32 # via boto3, moto, s3transfer +botocore==1.10.36 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -18,7 +18,7 @@ click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto -docker-pycreds==0.2.3 # via docker +docker-pycreds==0.3.0 # via docker docker==3.3.0 # via moto docutils==0.14 # via botocore factory-boy==2.11.1 @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.txt b/requirements.txt index 9e8ec438..712a88f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.24.0 +acme==0.25.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.32 -botocore==1.10.32 # via boto3, s3transfer +boto3==1.7.36 +botocore==1.10.36 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -25,7 +25,7 @@ dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.4 +flask-cors==3.0.6 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -35,7 +35,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.8.1 -idna==2.6 # via cryptography +idna==2.7 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 @@ -50,7 +50,7 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 # via python-ldap @@ -59,13 +59,14 @@ pycparser==2.18 # via cffi pyjwt==1.6.4 pynacl==1.2.1 # via paramiko pyopenssl==17.2.0 -pyrfc3339==1.0 # via acme +pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 pytz==2018.4 # via acme, flask-restful, pyrfc3339 pyyaml==3.12 # via cloudflare raven[flask]==6.9.0 +requests-toolbelt==0.8.0 # via acme requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 From b99aad743beb7e0288fd142aeeaed93c2899128c Mon Sep 17 00:00:00 2001 From: Dmitry Zykov Date: Wed, 4 Apr 2018 14:49:25 +0300 Subject: [PATCH 09/18] remove linuxdst plugin --- docs/developer/internals/lemur.rst | 1 - lemur/plugins/lemur_linuxdst/__init__.py | 5 -- lemur/plugins/lemur_linuxdst/plugin.py | 83 --------------------- lemur/plugins/lemur_linuxdst/readme.txt | 8 -- lemur/plugins/lemur_linuxdst/remote_host.py | 73 ------------------ 5 files changed, 170 deletions(-) delete mode 100644 lemur/plugins/lemur_linuxdst/__init__.py delete mode 100644 lemur/plugins/lemur_linuxdst/plugin.py delete mode 100644 lemur/plugins/lemur_linuxdst/readme.txt delete mode 100644 lemur/plugins/lemur_linuxdst/remote_host.py diff --git a/docs/developer/internals/lemur.rst b/docs/developer/internals/lemur.rst index 7faacfac..b6517e4b 100644 --- a/docs/developer/internals/lemur.rst +++ b/docs/developer/internals/lemur.rst @@ -110,6 +110,5 @@ Subpackages lemur.plugins.lemur_digicert lemur.plugins.lemur_java lemur.plugins.lemur_kubernetes - lemur.plugins.lemur_linuxdst lemur.plugins.lemur_openssl lemur.plugins.lemur_slack diff --git a/lemur/plugins/lemur_linuxdst/__init__.py b/lemur/plugins/lemur_linuxdst/__init__.py deleted file mode 100644 index 8ce5a7f3..00000000 --- a/lemur/plugins/lemur_linuxdst/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -try: - VERSION = __import__('pkg_resources') \ - .get_distribution(__name__).version -except Exception as e: - VERSION = 'unknown' diff --git a/lemur/plugins/lemur_linuxdst/plugin.py b/lemur/plugins/lemur_linuxdst/plugin.py deleted file mode 100644 index 71b32551..00000000 --- a/lemur/plugins/lemur_linuxdst/plugin.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/python -from lemur.plugins.bases import DestinationPlugin -from lemur.plugins.lemur_linuxdst import remote_host - - -class LinuxDstPlugin(DestinationPlugin): - title = 'Linux Destination Plugin' - slug = 'linux-destination' - description = 'Allow the distribution of certificates to a Linux host' - version = 1 - - author = 'Rick Breidenstein ' - author_url = 'https://github.com/RickB17/' - - options = [ - { - 'name': 'dstHost', - 'type': 'str', - 'required': True, - 'helpMessage': 'This is the host you will be sending the certificate to', - }, - { - 'name': 'dstPort', - 'type': 'int', - 'required': True, - 'helpMessage': 'This is the port SSHD is running on', - 'default': '22' - }, - { - 'name': 'dstUser', - 'type': 'str', - 'required': True, - 'helpMessage': 'The user name to use on the remote host. Hopefully not root.', - 'default': 'root', - }, - { - 'name': 'dstPriv', - 'type': 'str', - 'required': True, - 'helpMessage': 'The private key to use for auth', - 'default': '/root/.ssh/id_rsa', - }, - { - 'name': 'dstPrivKey', - 'type': 'str', - 'required': False, - 'helpMessage': 'The password for the destination private key', - 'default': 'somethingsecret', - }, - { - 'name': 'dstDir', - 'type': 'str', - 'required': True, - 'helpMessage': 'This is the directory on the host you want to send the certificate to', - 'default': '/etc/nginx/certs/' - }, - { - "available": [ - "NGINX", - "3File" - ], - "name": "exportType", - "required": True, - "value": "NGINX", - "helpMessage": "Reference the docs for an explanation of each export type", - "type": "select" - } - ] - requires_key = False - - def upload(self, name, body, private_key, cert_chain, options, **kwargs): - export_type = self.get_option('exportType', options) - dst_host = self.get_option('dstHost', options) - dst_host_port = self.get_option('dstPort', options) - dst_user = self.get_option('dstUser', options) - dst_priv = self.get_option('dstPriv', options) - dst_priv_key = self.get_option('dstPrivKey', options) - - if dst_priv_key: - dst_priv_key = None - - dst_dir = self.get_option('dstDir', options) - remote_host.create_cert(name, dst_dir, export_type, dst_user, dst_priv, dst_priv_key, dst_host, int(dst_host_port)) diff --git a/lemur/plugins/lemur_linuxdst/readme.txt b/lemur/plugins/lemur_linuxdst/readme.txt deleted file mode 100644 index 944cda9a..00000000 --- a/lemur/plugins/lemur_linuxdst/readme.txt +++ /dev/null @@ -1,8 +0,0 @@ -NOTE: AS OF 4/25/2017 DATA IN THIS PLUGIN IS NOT ENCRYPTED AT REST -example: If you input a password for a private key, it is stored in clear text - -add this to the lemur.plugins entry_points in /www/lemur/setup.py - -'linux_destination = lemur.plugins.lemur_linuxdst.plugin:LinuxDstPlugin', - -note: DO NOT FORGET TO EXECUTE 'make release' and restart your service after modifying the setup.py diff --git a/lemur/plugins/lemur_linuxdst/remote_host.py b/lemur/plugins/lemur_linuxdst/remote_host.py deleted file mode 100644 index b0f2cdb0..00000000 --- a/lemur/plugins/lemur_linuxdst/remote_host.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python -from lemur.certificates import service -import paramiko -import stat - - -def copy_cert(cert_cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_port, dst_dir, dst_file, dst_data): - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # include the private key password if required - if dst_priv_key is None: - priv_key = paramiko.RSAKey.from_private_key_file(dst_priv) - else: - priv_key = paramiko.RSAKey.from_private_key_file(dst_priv, dst_priv_key) - - # open the sftp connection - ssh.connect(dst_host, username=dst_user, port=dst_port, pkey=priv_key) - sftp = ssh.open_sftp() - - # make the directory on the destination server - # files will be in a folder based on the cert_cn - # example: - # destination folder: /etc/nginx/certs/ - # files will go in: /etc/nginx/certs/your.cn.com/cert.pem - try: - sftp.mkdir(dst_dir) - except IOError: - pass - try: - dst_dir_cn = dst_dir + '/' + cert_cn - sftp.mkdir(dst_dir_cn) - except IOError: - pass - - cert_out = sftp.open(dst_dir_cn + '/' + dst_file, 'w') - cert_out.write(dst_data) - cert_out.close() - sftp.chmod(dst_dir_cn + '/' + dst_file, (stat.S_IRUSR)) - ssh.close() - - -def create_cert(name, dst_dir, export_type, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port): - lem_cert = service.get_by_name(name) - dst_file = 'cert.pem' - chain_req = False - - if export_type == 'NGINX': - # This process will result in a cert.pem file with the body and chain in a single file - if lem_cert.chain is None: - dst_data = lem_cert.body - else: - dst_data = lem_cert.body + '\n' + lem_cert.chain - chain_req = False - - elif export_type == '3File': - # This process will results in three files. cert.pem, priv.key, chain.pem - dst_data = lem_cert.body - chain_req = True - - else: - dst_data = lem_cert.body - - copy_cert(lem_cert.cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port, dst_dir, dst_file, dst_data) - - if chain_req is True: - dst_file = 'chain.pem' - dst_data = lem_cert.chain_req - copy_cert(lem_cert.cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port, dst_dir, dst_file, dst_data) - - dst_file = 'priv.key' - dst_data = lem_cert.private_key - copy_cert(lem_cert.cn, dst_user, dst_priv, dst_priv_key, dst_host, dst_host_port, dst_dir, dst_file, dst_data) From 4e72cb96c9648132fb16061b8b3027c2aba2d712 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 14 Jun 2018 08:02:34 -0700 Subject: [PATCH 10/18] Graceful cancellation of pending cert and order details in log for acme failure --- lemur/plugins/lemur_acme/plugin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index a3f9af00..712164d5 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -104,7 +104,11 @@ def request_certificate(acme_client, authorizations, csr, order): authorization_resource, _ = acme_client.poll(authz) deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) - orderr = acme_client.finalize_order(order, deadline) + try: + orderr = acme_client.finalize_order(order, deadline) + except AcmeError: + current_app.logger.error("Unable to resolve Acme order: {}".format(order), exc_info=True) + raise pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, orderr.fullchain_pem)).decode() @@ -382,3 +386,7 @@ class ACMEIssuerPlugin(IssuerPlugin): if option.get('name') == 'certificate': acme_root = option.get('value') return acme_root, "", [role] + + def cancel_ordered_certificate(self, pending_cert, **kwargs): + # Needed to override issuer function. + pass From 9c5140006b42d33918b704f5d8b0e0e7c8d5bbfe Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 14 Jun 2018 08:04:35 -0700 Subject: [PATCH 11/18] update requirements while we're at it --- requirements-dev.txt | 8 ++++---- requirements-docs.txt | 15 ++++++++------- requirements-tests.txt | 10 +++++----- requirements.txt | 8 ++++---- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 58499408..983067a4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,20 +11,20 @@ cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 identify==1.1.0 # via pre-commit -idna==2.6 # via requests +idna==2.7 # via requests invoke==1.0.0 mccabe==0.6.1 # via flake8 -nodeenv==1.3.0 +nodeenv==1.3.1 pkginfo==1.4.2 # via twine pre-commit==1.10.2 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine -requests==2.18.4 # via requests-toolbelt, twine +requests==2.19.1 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit toml==0.9.4 # via pre-commit tqdm==4.23.4 # via twine twine==1.11.0 -urllib3==1.22 # via requests +urllib3==1.23 # via requests virtualenv==16.0.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 8795a511..58646db7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.24.0 +acme==0.25.0 alabaster==0.7.10 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==0.9.9 @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.32 -botocore==1.10.32 +boto3==1.7.36 +botocore==1.10.36 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -27,7 +27,7 @@ dnspython==1.15.0 docutils==0.14 dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.4 +flask-cors==3.0.6 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -37,7 +37,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.8.1 -idna==2.6 +idna==2.7 imagesize==1.0.0 # via sphinx inflection==0.3.1 itsdangerous==0.24 @@ -54,7 +54,7 @@ mock==2.0.0 ndg-httpsclient==0.5.0 packaging==17.1 # via sphinx paramiko==2.4.1 -pbr==4.0.3 +pbr==4.0.4 pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 @@ -65,12 +65,13 @@ pyjwt==1.6.4 pynacl==1.2.1 pyopenssl==17.2.0 pyparsing==2.2.0 # via packaging -pyrfc3339==1.0 +pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 pytz==2018.4 pyyaml==3.12 raven[flask]==6.9.0 +requests-toolbelt==0.8.0 requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 diff --git a/requirements-tests.txt b/requirements-tests.txt index efcfeac3..94104d37 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.36 # via moto +boto3==1.7.38 # via moto boto==2.48.0 # via moto -botocore==1.10.36 # via boto3, moto, s3transfer +botocore==1.10.38 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -25,7 +25,7 @@ factory-boy==2.11.1 faker==0.8.15 flask==1.0.2 # via pytest-flask freezegun==0.3.10 -idna==2.6 # via cryptography, requests +idna==2.7 # via cryptography, requests itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore @@ -49,12 +49,12 @@ python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.4 # via moto pyyaml==3.12 # via pyaml requests-mock==1.5.0 -requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses +requests==2.19.1 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.22 # via requests +urllib3==1.23 # via requests websocket-client==0.48.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.10.11 # via aws-xray-sdk diff --git a/requirements.txt b/requirements.txt index 712a88f6..769f3364 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.25.0 +acme==0.25.1 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.36 -botocore==1.10.36 # via boto3, s3transfer +boto3==1.7.38 +botocore==1.10.38 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -74,6 +74,6 @@ six==1.11.0 sqlalchemy-utils==0.33.3 sqlalchemy==1.2.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 -tld==0.7.10 +tld==0.9 werkzeug==0.14.1 # via flask xmltodict==0.11.0 From e835fa607317e6a659191d2f886d4b2dc009513b Mon Sep 17 00:00:00 2001 From: Doppins Date: Thu, 14 Jun 2018 23:37:07 +0000 Subject: [PATCH 12/18] Upgrade dependency boto3 to ==1.7.39 --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 9312142a..1a92297a 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,7 +15,7 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.37 +boto3==1.7.39 botocore==1.10.37 certifi==2018.4.16 cffi==1.11.5 From a141b8c5ea6c584bd767833f5ae8daebc0ef15c3 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 19 Jun 2018 16:27:58 -0700 Subject: [PATCH 13/18] Support concurrent issuance in Route53 for LetsEncrypt --- lemur/plugins/lemur_acme/plugin.py | 49 ++++++++++++++++++++++------- lemur/plugins/lemur_acme/route53.py | 29 ++++++++++++++--- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index a3f9af00..4d150e76 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -54,15 +54,24 @@ def maybe_remove_wildcard(host): return host.replace("*.", "") -def start_dns_challenge(acme_client, account_number, host, dns_provider, order): +def maybe_add_extension(host, dns_provider_options): + if dns_provider_options and dns_provider_options.get("acme_challenge_extension"): + host = host + dns_provider_options.get("acme_challenge_extension") + return host + + +def start_dns_challenge(acme_client, account_number, host, dns_provider, order, dns_provider_options): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) dns_challenges = find_dns_challenge(order.authorizations) change_ids = [] + host_to_validate = maybe_remove_wildcard(host) + host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options) + for dns_challenge in find_dns_challenge(order.authorizations): change_id = dns_provider.create_txt_record( - dns_challenge.validation_domain_name(maybe_remove_wildcard(host)), + dns_challenge.validation_domain_name(host_to_validate), dns_challenge.validation(acme_client.client.net.key), account_number ) @@ -104,7 +113,13 @@ def request_certificate(acme_client, authorizations, csr, order): authorization_resource, _ = acme_client.poll(authz) deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) - orderr = acme_client.finalize_order(order, deadline) + + try: + orderr = acme_client.finalize_order(order, deadline) + except: + current_app.logger.error("Unable to finalize ACME order: {}".format(order), exc_info=True) + raise + pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, orderr.fullchain_pem)).decode() @@ -154,24 +169,27 @@ def get_domains(options): return domains -def get_authorizations(acme_client, order, order_info, dns_provider): +def get_authorizations(acme_client, order, order_info, dns_provider, dns_provider_options): authorizations = [] for domain in order_info.domains: - authz_record = start_dns_challenge(acme_client, order_info.account_number, domain, dns_provider, order) + authz_record = start_dns_challenge(acme_client, order_info.account_number, domain, dns_provider, order, + dns_provider_options) authorizations.append(authz_record) return authorizations -def finalize_authorizations(acme_client, account_number, dns_provider, authorizations): +def finalize_authorizations(acme_client, account_number, dns_provider, authorizations, dns_provider_options): for authz_record in authorizations: complete_dns_challenge(acme_client, account_number, authz_record, dns_provider) for authz_record in authorizations: dns_challenges = authz_record.dns_challenge + host_to_validate = maybe_remove_wildcard(authz_record.host) + host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options) for dns_challenge in dns_challenges: dns_provider.delete_txt_record( authz_record.change_id, account_number, - dns_challenge.validation_domain_name(maybe_remove_wildcard(authz_record.host)), + dns_challenge.validation_domain_name(host_to_validate), dns_challenge.validation(acme_client.client.net.key) ) @@ -235,16 +253,17 @@ class ACMEIssuerPlugin(IssuerPlugin): acme_client, registration = setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) + dns_provider_options = dns_provider.options dns_provider_type = self.get_dns_provider(dns_provider.provider_type) try: authorizations = get_authorizations( - acme_client, order_info.account_number, order_info.domains, dns_provider_type) + acme_client, order_info.account_number, order_info.domains, dns_provider_type, dns_provider_options) except ClientError: current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert.name), exc_info=True) return False authorizations = finalize_authorizations( - acme_client, order_info.account_number, dns_provider_type, authorizations) + acme_client, order_info.account_number, dns_provider_type, authorizations, dns_provider_options) pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr) cert = { 'body': "\n".join(str(pem_certificate).splitlines()), @@ -261,6 +280,7 @@ class ACMEIssuerPlugin(IssuerPlugin): acme_client, registration = setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) + dns_provider_options = dns_provider.options dns_provider_type = self.get_dns_provider(dns_provider.provider_type) try: order = acme_client.new_order(pending_cert.csr) @@ -268,7 +288,8 @@ class ACMEIssuerPlugin(IssuerPlugin): raise Exception("The currently selected ACME CA endpoint does" " not support issuing wildcard certificates.") - authorizations = get_authorizations(acme_client, order, order_info, dns_provider_type) + authorizations = get_authorizations(acme_client, order, order_info, dns_provider_type, + dns_provider_options) pending.append({ "acme_client": acme_client, @@ -277,6 +298,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "authorizations": authorizations, "pending_cert": pending_cert, "order": order, + "dns_provider_options": dns_provider_options, }) except (ClientError, ValueError, Exception): current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) @@ -292,6 +314,7 @@ class ACMEIssuerPlugin(IssuerPlugin): entry["account_number"], entry["dns_provider_type"], entry["authorizations"], + entry["dns_provider_options"], ) pem_certificate, pem_certificate_chain = request_certificate( entry["acme_client"], @@ -329,6 +352,7 @@ class ACMEIssuerPlugin(IssuerPlugin): create_immediately = issuer_options.get('create_immediately', False) acme_client, registration = setup_acme_client(authority) dns_provider = issuer_options.get('dns_provider') + dns_provider_options = dns_provider.options if not dns_provider: raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.") credentials = json.loads(dns_provider.credentials) @@ -354,8 +378,9 @@ class ACMEIssuerPlugin(IssuerPlugin): # Return id of the DNS Authorization return None, None, dns_authorization.id - authorizations = get_authorizations(acme_client, account_number, domains, dns_provider_type) - finalize_authorizations(acme_client, account_number, dns_provider_type, authorizations) + authorizations = get_authorizations(acme_client, account_number, domains, dns_provider_type, + dns_provider_options) + finalize_authorizations(acme_client, account_number, dns_provider_type, authorizations, dns_provider_options) pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, csr) # TODO add external ID (if possible) return pem_certificate, pem_certificate_chain, None diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index b55d215b..989e8dd2 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -33,6 +33,29 @@ def find_zone_id(domain, client=None): @sts_client('route53') def change_txt_record(action, zone_id, domain, value, client=None): + current_txt_records = [] + try: + current_txt_records = client.list_resource_record_sets( + HostedZoneId=zone_id, + StartRecordName=domain, + StartRecordType='TXT', + MaxItems="1")["ResourceRecordSets"][0]["ResourceRecords"] + except Exception as e: + # Current Resource Record does not exist + if "NoSuchHostedZone" not in str(type(e)): + raise + # For some reason TXT records need to be + # manually quoted. + current_txt_records.append({"Value": '"{}"'.format(value)}) + + if action == "DELETE" and len(current_txt_records) > 1: + # If we want to delete one record out of many, we'll update the record to not include the deleted value instead. + # This allows us to support concurrent issuance. + current_txt_records = [ + record for record in current_txt_records if not (record.get('Value') == '"{}"'.format(value)) + ] + action = "UPSERT" + response = client.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ @@ -43,11 +66,7 @@ def change_txt_record(action, zone_id, domain, value, client=None): "Name": domain, "Type": "TXT", "TTL": 300, - "ResourceRecords": [ - # For some reason TXT records need to be - # manually quoted. - {"Value": '"{}"'.format(value)} - ], + "ResourceRecords": current_txt_records, } } ] From 665a0bcffe7f1ce29e6cdc2ec5f92570f17857f5 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 19 Jun 2018 16:38:05 -0700 Subject: [PATCH 14/18] Update requirements --- requirements-dev.txt | 14 +++++++------- requirements-docs.txt | 8 ++++---- requirements-tests.txt | 18 +++++++++--------- requirements.txt | 21 +++++++++++---------- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0fd765d0..4f6d3603 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,26 +5,26 @@ # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # aspy.yaml==1.1.1 # via pre-commit -cached-property==1.4.2 # via pre-commit +cached-property==1.4.3 # via pre-commit certifi==2018.4.16 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.18 # via pre-commit -idna==2.6 # via requests +identify==1.1.0 # via pre-commit +idna==2.7 # via requests invoke==1.0.0 mccabe==0.6.1 # via flake8 -nodeenv==1.3.0 +nodeenv==1.3.1 pkginfo==1.4.2 # via twine -pre-commit==1.10.1 +pre-commit==1.10.2 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine -requests==2.18.4 # via requests-toolbelt, twine +requests==2.19.1 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit toml==0.9.4 # via pre-commit tqdm==4.23.4 # via twine twine==1.11.0 -urllib3==1.22 # via requests +urllib3==1.23 # via requests virtualenv==16.0.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 4e96643b..b4c12d4d 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -5,7 +5,7 @@ # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # acme==0.24.0 -alabaster==0.7.10 # via sphinx +alabaster==0.7.11 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==0.9.9 aniso8601==3.0.0 @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.31 -botocore==1.10.31 +boto3==1.7.32 +botocore==1.10.32 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -76,7 +76,7 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.3.1 +sphinx-rtd-theme==0.4.0 sphinx==1.7.5 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.1.0 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index bca2a199..4e19f638 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.32 # via moto +boto3==1.7.41 # via moto boto==2.48.0 # via moto -botocore==1.10.32 # via boto3, moto, s3transfer +botocore==1.10.41 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -18,14 +18,14 @@ click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto -docker-pycreds==0.2.3 # via docker -docker==3.3.0 # via moto +docker-pycreds==0.3.0 # via docker +docker==3.4.0 # via moto docutils==0.14 # via botocore factory-boy==2.11.1 -faker==0.8.15 +faker==0.8.16 flask==1.0.2 # via pytest-flask freezegun==0.3.10 -idna==2.6 # via cryptography, requests +idna==2.7 # via cryptography, requests itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto @@ -49,12 +49,12 @@ python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.4 # via moto pyyaml==3.12 # via pyaml requests-mock==1.5.0 -requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses +requests==2.19.1 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.22 # via requests +urllib3==1.23 # via requests websocket-client==0.48.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.10.11 # via aws-xray-sdk diff --git a/requirements.txt b/requirements.txt index 9e8ec438..46839bda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,17 +4,17 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.24.0 +acme==0.25.1 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate -aniso8601==3.0.0 # via flask-restful +aniso8601==3.0.2 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.32 -botocore==1.10.32 # via boto3, s3transfer +boto3==1.7.41 +botocore==1.10.41 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -25,7 +25,7 @@ dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.4 +flask-cors==3.0.6 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -35,7 +35,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.8.1 -idna==2.6 # via cryptography +idna==2.7 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 @@ -50,22 +50,23 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.0.3 # via mock +pbr==4.0.4 # via mock pem==17.1.0 -psycopg2==2.7.4 +psycopg2==2.7.5 pyasn1-modules==0.2.1 # via python-ldap pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap, requests pycparser==2.18 # via cffi pyjwt==1.6.4 pynacl==1.2.1 # via paramiko pyopenssl==17.2.0 -pyrfc3339==1.0 # via acme +pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 pytz==2018.4 # via acme, flask-restful, pyrfc3339 pyyaml==3.12 # via cloudflare raven[flask]==6.9.0 +requests-toolbelt==0.8.0 # via acme requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 @@ -73,6 +74,6 @@ six==1.11.0 sqlalchemy-utils==0.33.3 sqlalchemy==1.2.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 -tld==0.7.10 +tld==0.9 werkzeug==0.14.1 # via flask xmltodict==0.11.0 From 2d33d3e2b8b335c2343d7ee224c97cf790783147 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 19 Jun 2018 20:32:11 -0700 Subject: [PATCH 15/18] lint --- lemur/plugins/lemur_acme/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 43f5244b..977b6ad8 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -130,7 +130,7 @@ def request_certificate(acme_client, authorizations, csr, order): def setup_acme_client(authority): - if not authority.options: + if not authority.options: raise InvalidAuthority("Invalid authority. Options not set") options = {} From dda7f54a1679aa82724bae2bfe9dc2ff945475d2 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 19 Jun 2018 20:58:00 -0700 Subject: [PATCH 16/18] lint --- lemur/plugins/lemur_acme/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 977b6ad8..43f5244b 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -130,7 +130,7 @@ def request_certificate(acme_client, authorizations, csr, order): def setup_acme_client(authority): - if not authority.options: + if not authority.options: raise InvalidAuthority("Invalid authority. Options not set") options = {} From 3efc709e030e433e4d54392c395bb353b0525cca Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 19 Jun 2018 21:16:35 -0700 Subject: [PATCH 17/18] tests --- lemur/plugins/lemur_acme/tests/test_acme.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index c80079af..f8b58f46 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -52,7 +52,7 @@ class TestAcme(unittest.TestCase): iterable = mock_find_dns_challenge.return_value iterator = iter(values) iterable.__iter__.return_value = iterator - result = plugin.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order) + result = plugin.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order, {}) self.assertEqual(type(result), plugin.AuthorizationRecord) @patch('acme.client.Client') @@ -171,7 +171,7 @@ class TestAcme(unittest.TestCase): mock_order_info = Mock() mock_order_info.account_number = 1 mock_order_info.domains = ["test.fakedomain.net"] - result = plugin.get_authorizations("acme_client", mock_order, mock_order_info, "dns_provider") + result = plugin.get_authorizations("acme_client", mock_order, mock_order_info, "dns_provider", {}) self.assertEqual(result, ["test"]) @patch('lemur.plugins.lemur_acme.plugin.complete_dns_challenge', return_value="test") @@ -187,7 +187,7 @@ class TestAcme(unittest.TestCase): mock_dns_provider.delete_txt_record = Mock() mock_acme_client = Mock() - result = plugin.finalize_authorizations(mock_acme_client, "account_number", mock_dns_provider, mock_authz) + result = plugin.finalize_authorizations(mock_acme_client, "account_number", mock_dns_provider, mock_authz, {}) self.assertEqual(result, mock_authz) @patch('lemur.plugins.lemur_acme.plugin.current_app') From 3397fb65607f421be8b5d81957925e47470a0b94 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 20 Jun 2018 10:33:35 -0700 Subject: [PATCH 18/18] R53: Extend only TXT records --- lemur/plugins/lemur_acme/route53.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index 989e8dd2..94823d80 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -35,11 +35,15 @@ def find_zone_id(domain, client=None): def change_txt_record(action, zone_id, domain, value, client=None): current_txt_records = [] try: - current_txt_records = client.list_resource_record_sets( + current_records = client.list_resource_record_sets( HostedZoneId=zone_id, StartRecordName=domain, StartRecordType='TXT', - MaxItems="1")["ResourceRecordSets"][0]["ResourceRecords"] + MaxItems="1")["ResourceRecordSets"] + + for record in current_records: + if record.get('Type') == 'TXT': + current_txt_records.extend(record.get("ResourceRecords", [])) except Exception as e: # Current Resource Record does not exist if "NoSuchHostedZone" not in str(type(e)):