Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

dyn support

This commit is contained in:
Curtis Castrapel 2018-05-04 15:00:43 -07:00
parent 3e64dd4653
commit 1be3f8368f
13 changed files with 216 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lemur/authorities/service.py
View File

@ -111,7 +111,7 @@ def create(**kwargs):
cert = upload(**kwargs)
kwargs['authority_certificate'] = cert
if kwargs.get('plugin', {}).get('plugin_options', []):
kwargs['options'] = json.dumps(kwargs.get('plugin', {}).get('plugin_options', []))
kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options'])
authority = Authority(**kwargs)
authority = database.create(authority)

2
lemur/certificates/schemas.py
View File

@ -78,7 +78,7 @@ class CertificateInputSchema(CertificateCreationSchema):
key_type = fields.String(
validate=validate.OneOf(
['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1',
'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1','ECCSECT163K1', 'ECCSECT233K1',
'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1', 'ECCSECT163K1', 'ECCSECT233K1',
'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1',
'ECCSECT409R1', 'ECCSECT571R2']),
missing='RSA2048')

4
lemur/exceptions.py
View File

@ -34,3 +34,7 @@ class AttrNotFound(LemurException):
class InvalidConfiguration(Exception):
pass
class InvalidAuthority(Exception):
pass

29
lemur/pending_certificates/cli.py
View File

@ -5,14 +5,12 @@
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
"""
from flask_script import Manager
from multiprocessing import Pool
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service
manager = Manager(usage="Handles pending certificate related tasks.")
agents = 20
# Need to call this multiple times and store status of the cert in DB. If it is being worked on by a worker, and which
@ -55,31 +53,32 @@ def fetch(ids):
)
def fetch_all():
@manager.command
def fetch_all_acme():
"""
Attempt to get full certificates for each pending certificate listed.
Args:
ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
`python manager.py pending_certs fetch -i 123 321 all`
Attempt to get full certificates for each pending certificate listed for ACME.
"""
pending_certs = pending_certificate_service.get_pending_certs('all')
user = user_service.get_by_username('lemur')
new = 0
failed = 0
certs = authority.get_ordered_certificates(pending_certs)
for cert in certs:
authority = plugins.get(cert.authority.plugin_name)
real_cert = authority.get_ordered_certificate(cert)
authority = plugins.get("acme-issuer")
resolved_certs = authority.get_ordered_certificates(pending_certs)
for cert in resolved_certs:
real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
pending_certificate_service.create_certificate(cert, real_cert, user)
pending_certificate_service.delete(cert)
pending_certificate_service.create_certificate(pending_cert, real_cert, user)
pending_certificate_service.delete_by_id(pending_cert.id)
# add metrics to metrics extension
new += 1
else:
pending_certificate_service.increment_attempt(cert)
pending_certificate_service.increment_attempt(pending_cert)
failed += 1
print(
"[+] Certificates: New: {new} Failed: {failed}".format(

4
lemur/pending_certificates/service.py
View File

@ -59,6 +59,10 @@ def delete(pending_certificate):
database.delete(pending_certificate)
def delete_by_id(id):
database.delete(get(id))
def get_pending_certs(pending_ids):
"""
Retrieve a list of pending certs given a list of ids

80
lemur/plugins/lemur_acme/dyn.py Normal file
View File

@ -0,0 +1,80 @@
import dns.exception
import dns.resolver
import time
from dyn.tm.session import DynectSession
from dyn.tm.zones import Node, Zone
from flask import current_app
from tld import get_tld
current_app.logger.debug("Logging in to Dyn API")
dynect_session = DynectSession(
current_app.config.get('ACME_DYN_CUSTOMER_NAME', ''),
current_app.config.get('ACME_DYN_USERNAME', ''),
current_app.config.get('ACME_DYN_PASSWORD', ''),
)
def _has_dns_propagated(name, token):
txt_records = []
try:
dns_resolver = dns.resolver.Resolver()
dns_resolver.nameservers = ['8.8.8.8']
dns_response = dns_resolver.query(name, 'TXT')
for rdata in dns_response:
for txt_record in rdata.strings:
txt_records.append(txt_record.decode("utf-8"))
except dns.exception.DNSException:
return False
for txt_record in txt_records:
if txt_record == token:
return True
return False
def wait_for_dns_change(change_id, account_number=None):
fqdn, token = change_id
while True:
status = _has_dns_propagated(fqdn, token)
current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
if status:
break
time.sleep(20)
return
def create_txt_record(domain, token, account_number):
zone_name = get_tld('http://' + domain)
zone_parts = len(zone_name.split('.'))
node_name = '.'.join(domain.split('.')[:-zone_parts])
fqdn = "{0}.{1}".format(node_name, zone_name)
zone = Zone(zone_name)
zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
node = zone.get_node(node_name)
zone.publish()
current_app.logger.debug("TXT record created: {0}".format(fqdn))
change_id = (fqdn, token)
return change_id
def delete_txt_record(change_id, account_number, domain, token):
if not domain:
current_app.logger.debug("delete_txt_record: No domain passed")
return
zone_name = get_tld('http://' + domain)
zone_parts = len(zone_name.split('.'))
node_name = '.'.join(domain.split('.')[:-zone_parts])
fqdn = "{0}.{1}".format(node_name, zone_name)
zone = Zone(zone_name)
node = Node(zone_name, fqdn)
all_txt_records = node.get_all_records_by_type('TXT')
for txt_record in all_txt_records:
if txt_record.txtdata == ("{}".format(token)):
current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn))
txt_record.delete()
zone.publish()

123
lemur/plugins/lemur_acme/plugin.py
View File

@ -17,8 +17,9 @@ import json
from flask import current_app
from acme.client import Client
from acme import messages
from acme import challenges
from acme import challenges, messages
from acme.errors import PollError
from botocore.exceptions import ClientError
from lemur.common.utils import generate_private_key
@ -26,6 +27,7 @@ import OpenSSL.crypto
from lemur.authorizations import service as authorization_service
from lemur.dns_providers import service as dns_provider_service
from lemur.exceptions import InvalidAuthority, InvalidConfiguration
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins import lemur_acme as acme
@ -68,6 +70,7 @@ def start_dns_challenge(acme_client, account_number, host, dns_provider):
def complete_dns_challenge(acme_client, account_number, authz_record, dns_provider):
current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.host))
dns_provider.wait_for_dns_change(authz_record.change_id, account_number=account_number)
response = authz_record.dns_challenge.response(acme_client.key)
@ -93,6 +96,8 @@ def request_certificate(acme_client, authorizations, csr):
)
),
authzrs=[authz_record.authz for authz_record in authorizations],
mintime=60,
max_attempts=10,
)
pem_certificate = OpenSSL.crypto.dump_certificate(
@ -111,11 +116,11 @@ def request_certificate(acme_client, authorizations, csr):
def setup_acme_client(authority):
if not authority.options:
raise Exception("Invalid authority. Options not set")
raise InvalidAuthority("Invalid authority. Options not set")
options = {}
for option in json.loads(authority.options):
options[option.get("name")] = option.get("value")
options[option["name"]] = option.get("value")
email = options.get('email', current_app.config.get('ACME_EMAIL'))
tel = options.get('telephone', current_app.config.get('ACME_TEL'))
directory_url = options.get('acme_url', current_app.config.get('ACME_DIRECTORY_URL'))
@ -219,15 +224,29 @@ class ACMEIssuerPlugin(IssuerPlugin):
def __init__(self, *args, **kwargs):
super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
def get_dns_provider(self, type):
from lemur.plugins.lemur_acme import cloudflare, dyn, route53
provider_types = {
'cloudflare': cloudflare,
'dyn': dyn,
'route53': route53,
}
return provider_types[type]
def get_ordered_certificate(self, pending_cert):
acme_client, registration = setup_acme_client(pending_cert.authority)
order_info = authorization_service.get(pending_cert.external_id)
dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
authorizations = get_authorizations(
acme_client, order_info.account_number, order_info.domains, dns_provider_type)
dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
try:
authorizations = get_authorizations(
acme_client, order_info.account_number, order_info.domains, dns_provider_type)
except ClientError:
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert.name), exc_info=True)
return False
finalize_authorizations(acme_client, order_info.account_number, dns_provider_type, authorizations)
authorizations = finalize_authorizations(
acme_client, order_info.account_number, dns_provider_type, authorizations)
pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr)
cert = {
'body': "\n".join(str(pem_certificate).splitlines()),
@ -238,45 +257,59 @@ class ACMEIssuerPlugin(IssuerPlugin):
def get_ordered_certificates(self, pending_certs):
pending = []
certs = []
for pending_cert in pending_certs:
acme_client, registration = setup_acme_client(pending_cert.authority)
order_info = authorization_service.get(pending_cert.external_id)
dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
authorizations = get_authorizations(
acme_client, order_info.account_number, order_info.domains, dns_provider_type)
pending.append({
"acme_client": acme_client,
"account_number": order_info.account_number,
"dns_provider_type": dns_provider_type,
"authorizations": authorizations,
"pending_cert": pending_cert,
})
certs = []
dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
try:
authorizations = get_authorizations(
acme_client, order_info.account_number, order_info.domains, dns_provider_type)
pending.append({
"acme_client": acme_client,
"account_number": order_info.account_number,
"dns_provider_type": dns_provider_type,
"authorizations": authorizations,
"pending_cert": pending_cert,
})
except (ClientError, ValueError):
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
certs.append({
"cert": False,
"pending_cert": pending_cert,
})
for entry in pending:
finalize_authorizations(
pending["acme_client"],
pending["account_number"],
pending["dns_provider_type"],
pending["authorizations"]
)
pem_certificate, pem_certificate_chain = request_certificate(
pending["acme_client"],
pending["authorizations"],
pending["pending_cert"].csr
entry["authorizations"] = finalize_authorizations(
entry["acme_client"],
entry["account_number"],
entry["dns_provider_type"],
entry["authorizations"]
)
cert = {
'body': "\n".join(str(pem_certificate).splitlines()),
'chain': "\n".join(str(pem_certificate_chain).splitlines()),
'external_id': str(pending_cert.external_id)
}
certs.append({
"cert": cert,
"pending_cert": pending_cert,
})
try:
pem_certificate, pem_certificate_chain = request_certificate(
entry["acme_client"],
entry["authorizations"],
entry["pending_cert"].csr
)
cert = {
'body': "\n".join(str(pem_certificate).splitlines()),
'chain': "\n".join(str(pem_certificate_chain).splitlines()),
'external_id': str(entry["pending_cert"].external_id)
}
certs.append({
"cert": cert,
"pending_cert": entry["pending_cert"],
})
except PollError:
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
certs.append({
"cert": False,
"pending_cert": entry["pending_cert"],
})
return certs
def create_certificate(self, csr, issuer_options):
@ -292,7 +325,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
acme_client, registration = setup_acme_client(authority)
dns_provider_d = issuer_options.get('dns_provider')
if not dns_provider_d:
raise Exception("DNS Provider setting is required for ACME certificates.")
raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.")
dns_provider = dns_provider_service.get(dns_provider_d.get("id"))
credentials = json.loads(dns_provider.credentials)
@ -300,9 +333,9 @@ class ACMEIssuerPlugin(IssuerPlugin):
dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
account_number = credentials.get("account_id")
if dns_provider.provider_type == 'route53' and not account_number:
error = "DNS Provider {} does not have an account number configured.".format(dns_provider.name)
error = "Route53 DNS Provider {} does not have an account number configured.".format(dns_provider.name)
current_app.logger.error(error)
raise Exception(error)
raise InvalidConfiguration(error)
domains = get_domains(issuer_options)
if not create_immediately:
# Create pending authorizations that we'll need to do the creation
@ -333,7 +366,11 @@ class ACMEIssuerPlugin(IssuerPlugin):
:return:
"""
role = {'username': '', 'password': '', 'name': 'acme'}
plugin_options = options.get('plugin').get('plugin_options')
plugin_options = options.get('plugin', {}).get('plugin_options')
if not plugin_options:
error = "Invalid options for lemur_acme plugin: {}".format(options)
current_app.logger.error(error)
raise InvalidConfiguration(error)
# Define static acme_root based off configuration variable by default. However, if user has passed a
# certificate, use this certificate as the root.
acme_root = current_app.config.get('ACME_ROOT')

6
lemur/plugins/lemur_acme/tests/test_acme.py
View File

@ -54,7 +54,8 @@ class TestAcme(unittest.TestCase):
self.assertEqual(type(result), plugin.AuthorizationRecord)
@patch('acme.client.Client')
def test_complete_dns_challenge_success(self, mock_acme):
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_complete_dns_challenge_success(self, mock_current_app, mock_acme):
mock_dns_provider = Mock()
mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
@ -65,7 +66,8 @@ class TestAcme(unittest.TestCase):
plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
@patch('acme.client.Client')
def test_complete_dns_challenge_fail(self, mock_acme):
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_complete_dns_challenge_fail(self, mock_current_app, mock_acme):
mock_dns_provider = Mock()
mock_dns_provider.wait_for_dns_change = Mock(return_value=True)

4
requirements-dev.txt
View File

@ -12,7 +12,7 @@ chardet==3.0.4 # via requests
flake8==3.5.0
identify==1.0.13 # via pre-commit
idna==2.6 # via requests
invoke==0.22.1
invoke==0.23.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.0
pkginfo==1.4.2 # via twine
@ -23,7 +23,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit
requests-toolbelt==0.8.0 # via twine
requests==2.18.4 # via requests-toolbelt, twine
six==1.11.0 # via cfgv, pre-commit
tqdm==4.23.1 # via twine
tqdm==4.23.2 # via twine
twine==1.11.0
urllib3==1.22 # via requests
virtualenv==15.2.0 # via pre-commit

13
requirements-docs.txt
View File

@ -15,13 +15,15 @@ asyncpool==1.0
babel==2.5.3 # via sphinx
bcrypt==3.1.4
blinker==1.4
boto3==1.7.10
botocore==1.10.10
boto3==1.7.11
botocore==1.10.11
certifi==2018.4.16
cffi==1.11.5
click==6.7
cloudflare==2.1.0
cryptography==2.2.2
docutils==0.14
dyn==1.8.1
flask-bcrypt==0.7.1
flask-cors==3.0.4
flask-mail==0.9.1
@ -34,11 +36,12 @@ flask==0.12
future==0.16.0
gevent==1.2.2
greenlet==0.4.13
gunicorn==19.7.1
gunicorn==19.8.1
idna==2.6
imagesize==1.0.0 # via sphinx
inflection==0.3.1
itsdangerous==0.24
janus==0.3.1
jinja2==2.10
jmespath==0.9.3
josepy==1.1.0
@ -74,11 +77,11 @@ retrying==1.3.3
s3transfer==0.1.13
six==1.11.0
snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.3.0
sphinx-rtd-theme==0.3.1
sphinx==1.7.4
sphinxcontrib-httpdomain==1.6.1
sphinxcontrib-websupport==1.0.1 # via sphinx
sqlalchemy-utils==0.33.2
sqlalchemy-utils==0.33.3
sqlalchemy==1.2.7
tabulate==0.8.2
werkzeug==0.14.1

8
requirements-tests.txt
View File

@ -7,9 +7,9 @@
asn1crypto==0.24.0 # via cryptography
attrs==17.4.0 # via pytest
aws-xray-sdk==0.95 # via moto
boto3==1.7.10 # via moto
boto3==1.7.11 # via moto
boto==2.48.0 # via moto
botocore==1.10.10 # via boto3, moto, s3transfer
botocore==1.10.11 # via boto3, moto, s3transfer
certifi==2018.4.16 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
@ -22,7 +22,7 @@ docker==3.3.0 # via moto
docutils==0.14 # via botocore
factory-boy==2.10.0
faker==0.8.13
flask==1.0 # via pytest-flask
flask==1.0.2 # via pytest-flask
freezegun==0.3.10
idna==2.6 # via cryptography, requests
itsdangerous==0.24 # via flask
@ -42,7 +42,7 @@ pyaml==17.12.1 # via moto
pycparser==2.18 # via cffi
pyflakes==1.6.0
pytest-flask==0.10.0
pytest-mock==1.9.0
pytest-mock==1.10.0
pytest==3.5.1
python-dateutil==2.6.1 # via botocore, faker, freezegun, moto
pytz==2018.4 # via moto

4
requirements.in
View File

@ -5,8 +5,11 @@ alembic-autogenerate-enums
arrow
asyncpool
boto3
certifi
CloudFlare
cryptography
dnspython3
dyn
Flask-Bcrypt==0.7.1
Flask-Mail==0.9.1
Flask-Migrate==2.1.1
@ -38,4 +41,5 @@ retrying
six
SQLAlchemy-Utils
tabulate
tld
xmltodict

15
requirements.txt
View File

@ -4,7 +4,7 @@
#
# pip-compile --no-index --output-file requirements.txt requirements.in
#
acme==0.23.0
acme==0.24.0
alembic-autogenerate-enums==0.0.2
alembic==0.9.9 # via flask-migrate
aniso8601==3.0.0 # via flask-restful
@ -13,13 +13,17 @@ asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
bcrypt==3.1.4 # via flask-bcrypt, paramiko
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.7.10
botocore==1.10.10 # via boto3, s3transfer
boto3==1.7.11
botocore==1.10.11 # via boto3, s3transfer
certifi==2018.4.16
cffi==1.11.5 # via bcrypt, cryptography, pynacl
click==6.7 # via flask
cloudflare==2.1.0
cryptography==2.2.2
dnspython3==1.15.0
dnspython==1.15.0 # via dnspython3
docutils==0.14 # via botocore
dyn==1.8.1
flask-bcrypt==0.7.1
flask-cors==3.0.4
flask-mail==0.9.1
@ -32,7 +36,7 @@ flask==0.12
future==0.16.0
gevent==1.2.2
greenlet==0.4.13 # via gevent
gunicorn==19.7.1
gunicorn==19.8.1
idna==2.6 # via cryptography
inflection==0.3.1
itsdangerous==0.24 # via flask
@ -69,8 +73,9 @@ requests[security]==2.11.1
retrying==1.3.3
s3transfer==0.1.13 # via boto3
six==1.11.0
sqlalchemy-utils==0.33.2
sqlalchemy-utils==0.33.3
sqlalchemy==1.2.7 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2
tld==0.7.10
werkzeug==0.14.1 # via flask
xmltodict==0.11.0