Additional work

lemur/certificates/schemas.py

@@ -74,7 +74,14 @@ class CertificateInputSchema(CertificateCreationSchema):
     dns_provider = fields.Nested(DnsProviderSchema, missing={}, required=False, allow_none=True)
 
     csr = fields.String(validate=validators.csr)
-    key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')
+
+    key_type = fields.String(
+        validate=validate.OneOf(
+            ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1',
+             'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1','ECCSECT163K1', 'ECCSECT233K1',
+             'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1',
+             'ECCSECT409R1', 'ECCSECT571R2']),
+        missing='RSA2048')
 
     notify = fields.Boolean(default=True)
     rotation = fields.Boolean()

lemur/pending_certificates/cli.py

@@ -2,17 +2,24 @@
 .. module: lemur.pending_certificates.cli
 
 .. moduleauthor:: James Chuong <jchuong@instartlogic.com>
+.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
 """
-
 from flask_script import Manager
+from multiprocessing import Pool
 
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.users import service as user_service
 
 manager = Manager(usage="Handles pending certificate related tasks.")
+agents = 20
 
 
+# Need to call this multiple times and store status of the cert in DB. If it is being worked on by a worker, and which
+# worker.
+# Then open up an arbitrary number of copies of this? every minute??
+# Or instead how about you send in a list of all pending certificates, make all the dns changes at once, then loop
+# through and wait for each one to complete?
 @manager.option('-i', dest='ids', action='append', help='IDs of pending certificates to fetch')
 def fetch(ids):
     """
@@ -22,10 +29,10 @@ def fetch(ids):
         ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
              `python manager.py pending_certs fetch -i 123 321 all`
     """
-    new = 0
-    failed = 0
     pending_certs = pending_certificate_service.get_pending_certs(ids)
     user = user_service.get_by_username('lemur')
+    new = 0
+    failed = 0
 
     for cert in pending_certs:
         authority = plugins.get(cert.authority.plugin_name)
@@ -43,6 +50,40 @@ def fetch(ids):
     print(
         "[+] Certificates: New: {new} Failed: {failed}".format(
             new=new,
-            failed=failed
+            failed=failed,
+        )
+    )
+
+
+def fetch_all():
+    """
+    Attempt to get full certificates for each pending certificate listed.
+
+    Args:
+        ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
+             `python manager.py pending_certs fetch -i 123 321 all`
+    """
+    pending_certs = pending_certificate_service.get_pending_certs('all')
+    user = user_service.get_by_username('lemur')
+    new = 0
+    failed = 0
+    certs = authority.get_ordered_certificates(pending_certs)
+    for cert in certs:
+        authority = plugins.get(cert.authority.plugin_name)
+        real_cert = authority.get_ordered_certificate(cert)
+        if real_cert:
+            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # the pending certificate
+            pending_certificate_service.create_certificate(cert, real_cert, user)
+            pending_certificate_service.delete(cert)
+            # add metrics to metrics extension
+            new += 1
+        else:
+            pending_certificate_service.increment_attempt(cert)
+            failed += 1
+    print(
+        "[+] Certificates: New: {new} Failed: {failed}".format(
+            new=new,
+            failed=failed,
         )
     )

lemur/plugins/lemur_acme/plugin.py

@@ -236,6 +236,49 @@ class ACMEIssuerPlugin(IssuerPlugin):
         }
         return cert
 
+    def get_ordered_certificates(self, pending_certs):
+        pending = []
+        for pending_cert in pending_certs:
+            acme_client, registration = setup_acme_client(pending_cert.authority)
+            order_info = authorization_service.get(pending_cert.external_id)
+            dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
+            dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
+            authorizations = get_authorizations(
+                acme_client, order_info.account_number, order_info.domains, dns_provider_type)
+            pending.append({
+                "acme_client": acme_client,
+                "account_number": order_info.account_number,
+                "dns_provider_type": dns_provider_type,
+                "authorizations": authorizations,
+                "pending_cert": pending_cert,
+            })
+
+        certs = []
+
+        for entry in pending:
+            finalize_authorizations(
+                pending["acme_client"],
+                pending["account_number"],
+                pending["dns_provider_type"],
+                pending["authorizations"]
+            )
+            pem_certificate, pem_certificate_chain = request_certificate(
+                pending["acme_client"],
+                pending["authorizations"],
+                pending["pending_cert"].csr
+            )
+
+            cert = {
+                'body': "\n".join(str(pem_certificate).splitlines()),
+                'chain': "\n".join(str(pem_certificate_chain).splitlines()),
+                'external_id': str(pending_cert.external_id)
+            }
+            certs.append({
+                "cert": cert,
+                "pending_cert": pending_cert,
+            })
+        return certs
+
     def create_certificate(self, csr, issuer_options):
         """
         Creates an ACME certificate.

requirements-dev.txt

@@ -23,7 +23,7 @@ pyyaml==3.12              # via aspy.yaml, pre-commit
 requests-toolbelt==0.8.0  # via twine
 requests==2.18.4          # via requests-toolbelt, twine
 six==1.11.0               # via cfgv, pre-commit
-tqdm==4.23.0              # via twine
+tqdm==4.23.1              # via twine
 twine==1.11.0
 urllib3==1.22             # via requests
 virtualenv==15.2.0        # via pre-commit

requirements-docs.txt

@@ -11,17 +11,19 @@ alembic==0.9.9
 aniso8601==3.0.0
 arrow==0.12.1
 asn1crypto==0.24.0
+asyncpool==1.0
 babel==2.5.3              # via sphinx
 bcrypt==3.1.4
 blinker==1.4
-boto3==1.7.6
+boto3==1.7.10
-botocore==1.10.6
+botocore==1.10.10
 cffi==1.11.5
 click==6.7
+cloudflare==2.1.0
 cryptography==2.2.2
 docutils==0.14
 flask-bcrypt==0.7.1
-flask-cors==3.0.3
+flask-cors==3.0.4
 flask-mail==0.9.1
 flask-migrate==2.1.1
 flask-principal==0.4.0
@@ -30,6 +32,8 @@ flask-script==2.0.6
 flask-sqlalchemy==2.3.2
 flask==0.12
 future==0.16.0
+gevent==1.2.2
+greenlet==0.4.13
 gunicorn==19.7.1
 idna==2.6
 imagesize==1.0.0          # via sphinx
@@ -38,13 +42,14 @@ itsdangerous==0.24
 jinja2==2.10
 jmespath==0.9.3
 josepy==1.1.0
+jsonlines==1.2.0
 lockfile==0.12.2
 mako==1.0.7
 markupsafe==1.0
 marshmallow-sqlalchemy==0.13.2
-marshmallow==2.15.0
+marshmallow==2.15.1
 mock==2.0.0
-ndg-httpsclient==0.4.4
+ndg-httpsclient==0.5.0
 packaging==17.1           # via sphinx
 paramiko==2.4.1
 pbr==4.0.2
@@ -62,6 +67,7 @@ pyrfc3339==1.0
 python-dateutil==2.7.2
 python-editor==1.0.3
 pytz==2018.4
+pyyaml==3.12
 raven[flask]==6.7.0
 requests[security]==2.11.1
 retrying==1.3.3
@@ -69,7 +75,7 @@ s3transfer==0.1.13
 six==1.11.0
 snowballstemmer==1.2.1    # via sphinx
 sphinx-rtd-theme==0.3.0
-sphinx==1.7.3
+sphinx==1.7.4
 sphinxcontrib-httpdomain==1.6.1
 sphinxcontrib-websupport==1.0.1  # via sphinx
 sqlalchemy-utils==0.33.2

requirements-tests.txt

@@ -7,9 +7,9 @@
 asn1crypto==0.24.0        # via cryptography
 attrs==17.4.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.7.8              # via moto
+boto3==1.7.10             # via moto
 boto==2.48.0              # via moto
-botocore==1.10.8          # via boto3, moto, s3transfer
+botocore==1.10.10         # via boto3, moto, s3transfer
 certifi==2018.4.16        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -17,12 +17,12 @@ click==6.7                # via flask
 cookies==2.2.1            # via moto, responses
 coverage==4.5.1
 cryptography==2.2.2       # via moto
-docker-pycreds==0.2.2     # via docker
+docker-pycreds==0.2.3     # via docker
-docker==3.2.1             # via moto
+docker==3.3.0             # via moto
 docutils==0.14            # via botocore
 factory-boy==2.10.0
 faker==0.8.13
-flask==0.12.2             # via pytest-flask
+flask==1.0                # via pytest-flask
 freezegun==0.3.10
 idna==2.6                 # via cryptography, requests
 itsdangerous==0.24        # via flask

requirements.in

@@ -3,6 +3,7 @@
 acme
 alembic-autogenerate-enums
 arrow
+asyncpool
 boto3
 CloudFlare
 cryptography
@@ -16,8 +17,10 @@ Flask-SQLAlchemy
 Flask==0.12
 Flask-Cors
 future
+gevent
 gunicorn
 inflection
+janus
 jinja2
 lockfile
 marshmallow-sqlalchemy

requirements.txt

@@ -10,17 +10,18 @@ alembic==0.9.9            # via flask-migrate
 aniso8601==3.0.0          # via flask-restful
 arrow==0.12.1
 asn1crypto==0.24.0        # via cryptography
+asyncpool==1.0
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.7.8
+boto3==1.7.10
-botocore==1.10.8          # via boto3, s3transfer
+botocore==1.10.10         # via boto3, s3transfer
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 click==6.7                # via flask
 cloudflare==2.1.0
 cryptography==2.2.2
 docutils==0.14            # via botocore
 flask-bcrypt==0.7.1
-flask-cors==3.0.3
+flask-cors==3.0.4
 flask-mail==0.9.1
 flask-migrate==2.1.1
 flask-principal==0.4.0
@@ -29,10 +30,13 @@ flask-script==2.0.6
 flask-sqlalchemy==2.3.2
 flask==0.12
 future==0.16.0
+gevent==1.2.2
+greenlet==0.4.13          # via gevent
 gunicorn==19.7.1
 idna==2.6                 # via cryptography
 inflection==0.3.1
 itsdangerous==0.24        # via flask
+janus==0.3.1
 jinja2==2.10
 jmespath==0.9.3           # via boto3, botocore
 josepy==1.1.0             # via acme
@@ -41,7 +45,7 @@ lockfile==0.12.2
 mako==1.0.7               # via alembic
 markupsafe==1.0           # via jinja2, mako
 marshmallow-sqlalchemy==0.13.2
-marshmallow==2.15.0
+marshmallow==2.15.1
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.0
 paramiko==2.4.1