From 3e64dd465387d119629ff3862eed250ff5118e83 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 30 Apr 2018 10:48:48 -0700 Subject: [PATCH] Additional work --- lemur/certificates/schemas.py | 9 +++++- lemur/pending_certificates/cli.py | 49 +++++++++++++++++++++++++++--- lemur/plugins/lemur_acme/plugin.py | 43 ++++++++++++++++++++++++++ requirements-dev.txt | 2 +- requirements-docs.txt | 18 +++++++---- requirements-tests.txt | 10 +++--- requirements.in | 3 ++ requirements.txt | 12 +++++--- 8 files changed, 125 insertions(+), 21 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index cce4aa33..6924674c 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -74,7 +74,14 @@ class CertificateInputSchema(CertificateCreationSchema): dns_provider = fields.Nested(DnsProviderSchema, missing={}, required=False, allow_none=True) csr = fields.String(validate=validators.csr) - key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048') + + key_type = fields.String( + validate=validate.OneOf( + ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1', + 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1','ECCSECT163K1', 'ECCSECT233K1', + 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', + 'ECCSECT409R1', 'ECCSECT571R2']), + missing='RSA2048') notify = fields.Boolean(default=True) rotation = fields.Boolean() diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index ae69bc0d..5b48559a 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -2,17 +2,24 @@ .. module: lemur.pending_certificates.cli .. moduleauthor:: James Chuong +.. moduleauthor:: Curtis Castrapel """ - from flask_script import Manager +from multiprocessing import Pool from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.users import service as user_service manager = Manager(usage="Handles pending certificate related tasks.") +agents = 20 +# Need to call this multiple times and store status of the cert in DB. If it is being worked on by a worker, and which +# worker. +# Then open up an arbitrary number of copies of this? every minute?? +# Or instead how about you send in a list of all pending certificates, make all the dns changes at once, then loop +# through and wait for each one to complete? @manager.option('-i', dest='ids', action='append', help='IDs of pending certificates to fetch') def fetch(ids): """ @@ -22,10 +29,10 @@ def fetch(ids): ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI) `python manager.py pending_certs fetch -i 123 321 all` """ - new = 0 - failed = 0 pending_certs = pending_certificate_service.get_pending_certs(ids) user = user_service.get_by_username('lemur') + new = 0 + failed = 0 for cert in pending_certs: authority = plugins.get(cert.authority.plugin_name) @@ -43,6 +50,40 @@ def fetch(ids): print( "[+] Certificates: New: {new} Failed: {failed}".format( new=new, - failed=failed + failed=failed, + ) + ) + + +def fetch_all(): + """ + Attempt to get full certificates for each pending certificate listed. + + Args: + ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI) + `python manager.py pending_certs fetch -i 123 321 all` + """ + pending_certs = pending_certificate_service.get_pending_certs('all') + user = user_service.get_by_username('lemur') + new = 0 + failed = 0 + certs = authority.get_ordered_certificates(pending_certs) + for cert in certs: + authority = plugins.get(cert.authority.plugin_name) + real_cert = authority.get_ordered_certificate(cert) + if real_cert: + # If a real certificate was returned from issuer, then create it in Lemur and delete + # the pending certificate + pending_certificate_service.create_certificate(cert, real_cert, user) + pending_certificate_service.delete(cert) + # add metrics to metrics extension + new += 1 + else: + pending_certificate_service.increment_attempt(cert) + failed += 1 + print( + "[+] Certificates: New: {new} Failed: {failed}".format( + new=new, + failed=failed, ) ) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f07106db..f5b4b12a 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -236,6 +236,49 @@ class ACMEIssuerPlugin(IssuerPlugin): } return cert + def get_ordered_certificates(self, pending_certs): + pending = [] + for pending_cert in pending_certs: + acme_client, registration = setup_acme_client(pending_cert.authority) + order_info = authorization_service.get(pending_cert.external_id) + dns_provider = dns_provider_service.get(pending_cert.dns_provider_id) + dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1) + authorizations = get_authorizations( + acme_client, order_info.account_number, order_info.domains, dns_provider_type) + pending.append({ + "acme_client": acme_client, + "account_number": order_info.account_number, + "dns_provider_type": dns_provider_type, + "authorizations": authorizations, + "pending_cert": pending_cert, + }) + + certs = [] + + for entry in pending: + finalize_authorizations( + pending["acme_client"], + pending["account_number"], + pending["dns_provider_type"], + pending["authorizations"] + ) + pem_certificate, pem_certificate_chain = request_certificate( + pending["acme_client"], + pending["authorizations"], + pending["pending_cert"].csr + ) + + cert = { + 'body': "\n".join(str(pem_certificate).splitlines()), + 'chain': "\n".join(str(pem_certificate_chain).splitlines()), + 'external_id': str(pending_cert.external_id) + } + certs.append({ + "cert": cert, + "pending_cert": pending_cert, + }) + return certs + def create_certificate(self, csr, issuer_options): """ Creates an ACME certificate. diff --git a/requirements-dev.txt b/requirements-dev.txt index 356db421..7471ccbc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.18.4 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -tqdm==4.23.0 # via twine +tqdm==4.23.1 # via twine twine==1.11.0 urllib3==1.22 # via requests virtualenv==15.2.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 9d388183..10193b47 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -11,17 +11,19 @@ alembic==0.9.9 aniso8601==3.0.0 arrow==0.12.1 asn1crypto==0.24.0 +asyncpool==1.0 babel==2.5.3 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.6 -botocore==1.10.6 +boto3==1.7.10 +botocore==1.10.10 cffi==1.11.5 click==6.7 +cloudflare==2.1.0 cryptography==2.2.2 docutils==0.14 flask-bcrypt==0.7.1 -flask-cors==3.0.3 +flask-cors==3.0.4 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -30,6 +32,8 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 +gevent==1.2.2 +greenlet==0.4.13 gunicorn==19.7.1 idna==2.6 imagesize==1.0.0 # via sphinx @@ -38,13 +42,14 @@ itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 +jsonlines==1.2.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.0 marshmallow-sqlalchemy==0.13.2 -marshmallow==2.15.0 +marshmallow==2.15.1 mock==2.0.0 -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.0 packaging==17.1 # via sphinx paramiko==2.4.1 pbr==4.0.2 @@ -62,6 +67,7 @@ pyrfc3339==1.0 python-dateutil==2.7.2 python-editor==1.0.3 pytz==2018.4 +pyyaml==3.12 raven[flask]==6.7.0 requests[security]==2.11.1 retrying==1.3.3 @@ -69,7 +75,7 @@ s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.3.0 -sphinx==1.7.3 +sphinx==1.7.4 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx sqlalchemy-utils==0.33.2 diff --git a/requirements-tests.txt b/requirements-tests.txt index ebdd7ec5..8d04ddcf 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,9 +7,9 @@ asn1crypto==0.24.0 # via cryptography attrs==17.4.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.8 # via moto +boto3==1.7.10 # via moto boto==2.48.0 # via moto -botocore==1.10.8 # via boto3, moto, s3transfer +botocore==1.10.10 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -17,12 +17,12 @@ click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto -docker-pycreds==0.2.2 # via docker -docker==3.2.1 # via moto +docker-pycreds==0.2.3 # via docker +docker==3.3.0 # via moto docutils==0.14 # via botocore factory-boy==2.10.0 faker==0.8.13 -flask==0.12.2 # via pytest-flask +flask==1.0 # via pytest-flask freezegun==0.3.10 idna==2.6 # via cryptography, requests itsdangerous==0.24 # via flask diff --git a/requirements.in b/requirements.in index c88af93a..1bd54d20 100644 --- a/requirements.in +++ b/requirements.in @@ -3,6 +3,7 @@ acme alembic-autogenerate-enums arrow +asyncpool boto3 CloudFlare cryptography @@ -16,8 +17,10 @@ Flask-SQLAlchemy Flask==0.12 Flask-Cors future +gevent gunicorn inflection +janus jinja2 lockfile marshmallow-sqlalchemy diff --git a/requirements.txt b/requirements.txt index ca0be5b3..5351ee50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,17 +10,18 @@ alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography +asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.8 -botocore==1.10.8 # via boto3, s3transfer +boto3==1.7.10 +botocore==1.10.10 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask cloudflare==2.1.0 cryptography==2.2.2 docutils==0.14 # via botocore flask-bcrypt==0.7.1 -flask-cors==3.0.3 +flask-cors==3.0.4 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -29,10 +30,13 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 +gevent==1.2.2 +greenlet==0.4.13 # via gevent gunicorn==19.7.1 idna==2.6 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask +janus==0.3.1 jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme @@ -41,7 +45,7 @@ lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.0 # via jinja2, mako marshmallow-sqlalchemy==0.13.2 -marshmallow==2.15.0 +marshmallow==2.15.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1