Celery integration

lemur/certificates/service.py

@@ -5,32 +5,27 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import arrow
+from cryptography import x509
-
-from flask import current_app
 from sqlalchemy import func, or_, not_, cast, Integer
 
-from cryptography import x509
+import arrow
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
+from flask import current_app
 
 from lemur import database
-from lemur.extensions import metrics, sentry, signals
-from lemur.plugins.base import plugins
-from lemur.common.utils import generate_private_key, truthiness
-
-from lemur.roles.models import Role
-from lemur.domains.models import Domain
 from lemur.authorities.models import Authority
-from lemur.destinations.models import Destination
 from lemur.certificates.models import Certificate
+from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
+from lemur.common.utils import generate_private_key, truthiness
+from lemur.destinations.models import Destination
+from lemur.domains.models import Domain
+from lemur.extensions import metrics, sentry, signals
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
-
+from lemur.plugins.base import plugins
-from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
-
 from lemur.roles import service as role_service
-
+from lemur.roles.models import Role
 
 csr_created = signals.signal('csr_created', "CSR generated")
 csr_imported = signals.signal('csr_imported', "CSR imported from external source")
@@ -95,7 +90,7 @@ def get_all_pending_cleaning(source):
     :param source:
     :return:
     """
-    return Certificate.query.filter(Certificate.sources.any(id=source.id))\
+    return Certificate.query.filter(Certificate.sources.any(id=source.id)) \
         .filter(not_(Certificate.endpoints.any())).all()
 
 
@@ -109,8 +104,8 @@ def get_all_pending_reissue():
 
     :return:
     """
-    return Certificate.query.filter(Certificate.rotation == True)\
+    return Certificate.query.filter(Certificate.rotation == True) \
-        .filter(not_(Certificate.replaced.any()))\
+        .filter(not_(Certificate.replaced.any())) \
         .filter(Certificate.in_rotation_window == True).all()  # noqa
 
 
@@ -280,6 +275,11 @@ def create(**kwargs):
     if isinstance(cert, Certificate):
         certificate_issued.send(certificate=cert, authority=cert.authority)
         metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
+
+    if isinstance(cert, PendingCertificate) and cert.authority.plugin_name == 'acme-issuer':
+        # Call Celery to create acme-issuer (LetsEncrypt) certificates
+        from lemur.common.celery import fetch_acme_cert
+        fetch_acme_cert.delay(cert.id)
     return cert
 
 
@@ -310,8 +310,8 @@ def render(args):
 
         if 'issuer' in terms:
             # we can't rely on issuer being correct in the cert directly so we combine queries
-            sub_query = database.session_query(Authority.id)\
+            sub_query = database.session_query(Authority.id) \
-                .filter(Authority.name.ilike(term))\
+                .filter(Authority.name.ilike(term)) \
                 .subquery()
 
             query = query.filter(
@@ -450,8 +450,8 @@ def stats(**kwargs):
     if kwargs.get('metric') == 'not_after':
         start = arrow.utcnow()
         end = start.replace(weeks=+32)
-        items = database.db.session.query(Certificate.issuer, func.count(Certificate.id))\
+        items = database.db.session.query(Certificate.issuer, func.count(Certificate.id)) \
-            .group_by(Certificate.issuer)\
+            .group_by(Certificate.issuer) \
             .filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \
             .filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all()
 

lemur/common/celery.py

@@ -0,0 +1,119 @@
+"""
+This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
+when invoked.
+
+When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
+beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
+command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
+
+"""
+import copy
+import sys
+
+from celery import Celery
+from flask import current_app
+
+from lemur.authorities.service import get as get_authority
+from lemur.factory import create_app
+from lemur.notifications.messaging import send_pending_failure_notification
+from lemur.pending_certificates import service as pending_certificate_service
+from lemur.plugins.base import plugins
+from lemur.users import service as user_service
+
+flask_app = create_app()
+
+
+def make_celery(app):
+    celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
+                    broker=app.config['CELERY_BROKER_URL'])
+    celery.conf.update(app.config)
+    TaskBase = celery.Task
+
+    class ContextTask(TaskBase):
+        abstract = True
+
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return TaskBase.__call__(self, *args, **kwargs)
+
+    celery.Task = ContextTask
+    return celery
+
+
+celery = make_celery(flask_app)
+
+
+@celery.task()
+def fetch_acme_cert(id):
+    """
+    Attempt to get the full certificate for the pending certificate listed.
+
+    Args:
+        id: an id of a PendingCertificate
+    """
+    log_data = {
+        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
+    }
+    pending_certs = pending_certificate_service.get_pending_certs([id])
+    user = user_service.get_by_username('lemur')
+    new = 0
+    failed = 0
+    wrong_issuer = 0
+    acme_certs = []
+
+    # We only care about certs using the acme-issuer plugin
+    for cert in pending_certs:
+        cert_authority = get_authority(cert.authority_id)
+        if cert_authority.plugin_name == 'acme-issuer':
+            acme_certs.append(cert)
+        else:
+            wrong_issuer += 1
+
+    authority = plugins.get("acme-issuer")
+    resolved_certs = authority.get_ordered_certificates(acme_certs)
+
+    for cert in resolved_certs:
+        real_cert = cert.get("cert")
+        # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
+        pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
+
+        if real_cert:
+            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # the pending certificate
+            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
+            pending_certificate_service.delete_by_id(pending_cert.id)
+            # add metrics to metrics extension
+            new += 1
+        else:
+            failed += 1
+            error_log = copy.deepcopy(log_data)
+            error_log["message"] = "Pending certificate creation failure"
+            error_log["pending_cert_id"] = pending_cert.id
+            error_log["last_error"] = cert.get("last_error")
+            error_log["cn"] = pending_cert.cn
+
+            if pending_cert.number_attempts > 4:
+                error_log["message"] = "Deleting pending certificate"
+                send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
+                pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
+            else:
+                pending_certificate_service.increment_attempt(pending_cert)
+                pending_certificate_service.update(
+                    cert.get("pending_cert").id,
+                    status=str(cert.get("last_error"))
+                )
+                # Add failed pending cert task back to queue
+                fetch_acme_cert.delay(id)
+            current_app.logger.error(error_log)
+    log_data["message"] = "Complete"
+    log_data["new"] = new
+    log_data["failed"] = failed
+    log_data["wrong_issuer"] = wrong_issuer
+    current_app.logger.debug(log_data)
+    print(
+        "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
+            new=new,
+            failed=failed,
+            wrong_issuer=wrong_issuer
+        )
+    )

requirements-dev.txt

@@ -5,26 +5,26 @@
 #    pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
 #
 aspy.yaml==1.1.1          # via pre-commit
-cached-property==1.4.3    # via pre-commit
+cached-property==1.5.1    # via pre-commit
 certifi==2018.8.24        # via requests
 cfgv==1.1.0               # via pre-commit
 chardet==3.0.4            # via requests
 flake8==3.5.0
-identify==1.1.4           # via pre-commit
+identify==1.1.5           # via pre-commit
 idna==2.7                 # via requests
 invoke==1.1.1
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.2
 pkginfo==1.4.2            # via twine
-pre-commit==1.10.5
+pre-commit==1.11.0
 pycodestyle==2.3.1        # via flake8
 pyflakes==1.6.0           # via flake8
 pyyaml==3.13              # via aspy.yaml, pre-commit
 requests-toolbelt==0.8.0  # via twine
 requests==2.19.1          # via requests-toolbelt, twine
 six==1.11.0               # via cfgv, pre-commit
-toml==0.9.4               # via pre-commit
+toml==0.9.6               # via pre-commit
-tqdm==4.25.0              # via twine
+tqdm==4.26.0              # via twine
 twine==1.11.0
 urllib3==1.23             # via requests
 virtualenv==16.0.0        # via pre-commit

requirements-docs.txt

@@ -4,19 +4,22 @@
 #
 #    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
-acme==0.26.1
+acme==0.27.1
 alabaster==0.7.11         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.0
+amqp==2.3.2
 aniso8601==3.0.2
 arrow==0.12.1
 asn1crypto==0.24.0
 asyncpool==1.0
 babel==2.6.0              # via sphinx
 bcrypt==3.1.4
+billiard==3.5.0.4
 blinker==1.4
-boto3==1.8.1
+boto3==1.7.79
-botocore==1.11.1
+botocore==1.10.84
+celery[redis]==4.2.1
 certifi==2018.8.24
 cffi==1.11.5
 chardet==3.0.4
@@ -39,13 +42,14 @@ flask==0.12
 future==0.16.0
 gunicorn==19.9.0
 idna==2.7
-imagesize==1.0.0          # via sphinx
+imagesize==1.1.0          # via sphinx
 inflection==0.3.1
 itsdangerous==0.24
 jinja2==2.10
 jmespath==0.9.3
 josepy==1.1.0
 jsonlines==1.2.0
+kombu==4.2.1
 lockfile==0.12.2
 mako==1.0.7
 markupsafe==1.0
@@ -72,6 +76,7 @@ python-editor==1.0.3
 pytz==2018.5
 pyyaml==3.13
 raven[flask]==6.9.0
+redis==2.10.6
 requests-toolbelt==0.8.0
 requests[security]==2.19.1
 retrying==1.3.3
@@ -79,12 +84,13 @@ s3transfer==0.1.13
 six==1.11.0
 snowballstemmer==1.2.1    # via sphinx
 sphinx-rtd-theme==0.4.1
-sphinx==1.7.7
+sphinx==1.8.0
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx
 sqlalchemy-utils==0.33.3
-sqlalchemy==1.2.10
+sqlalchemy==1.2.11
 tabulate==0.8.2
 urllib3==1.23
+vine==1.1.4
 werkzeug==0.14.1
 xmltodict==0.11.0

requirements-tests.in

@@ -4,7 +4,7 @@ coverage
 factory-boy
 Faker
 freezegun
-moto
+moto==1.3.4 # Issue with moto: https://github.com/spulec/moto/issues/1813
 nose
 pyflakes
 pytest

requirements-tests.txt

@@ -5,12 +5,12 @@
 #    pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in
 #
 asn1crypto==0.24.0        # via cryptography
-atomicwrites==1.2.0       # via pytest
+atomicwrites==1.2.1       # via pytest
-attrs==18.1.0             # via pytest
+attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.8.1              # via moto
+boto3==1.9.3              # via moto
 boto==2.49.0              # via moto
-botocore==1.11.1          # via boto3, moto, s3transfer
+botocore==1.12.3          # via boto3, moto, s3transfer
 certifi==2018.8.24        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -40,14 +40,14 @@ moto==1.3.4
 nose==1.3.7
 pbr==4.2.0                # via mock
 pluggy==0.7.1             # via pytest
-py==1.5.4                 # via pytest
+py==1.6.0                 # via pytest
 pyaml==17.12.1            # via moto
 pycparser==2.18           # via cffi
 pycryptodome==3.6.6       # via python-jose
 pyflakes==2.0.0
-pytest-flask==0.10.0
+pytest-flask==0.12.0
 pytest-mock==1.10.0
-pytest==3.7.3
+pytest==3.8.0
 python-dateutil==2.7.3    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
 pytz==2018.5              # via moto
@@ -59,7 +59,7 @@ s3transfer==0.1.13        # via boto3
 six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
 text-unidecode==1.2       # via faker
 urllib3==1.23             # via botocore, requests
-websocket-client==0.51.0  # via docker
+websocket-client==0.53.0  # via docker
 werkzeug==0.14.1          # via flask, moto, pytest-flask
 wrapt==1.10.11            # via aws-xray-sdk
 xmltodict==0.11.0         # via moto

requirements.in

@@ -4,7 +4,9 @@ acme
 alembic-autogenerate-enums
 arrow
 asyncpool
-boto3==1.7.79
+boto3==1.7.79 # Issue with moto: https://github.com/spulec/moto/issues/1813
+botocore== 1.10.84 # Issue with moto: https://github.com/spulec/moto/issues/1813
+celery[redis]
 certifi
 CloudFlare
 cryptography

requirements.txt

@@ -4,17 +4,20 @@
 #
 #    pip-compile --no-index --output-file requirements.txt requirements.in
 #
-acme==0.26.1
+acme==0.27.1
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.0            # via flask-migrate
+amqp==2.3.2               # via kombu
 aniso8601==3.0.2          # via flask-restful
 arrow==0.12.1
 asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
+billiard==3.5.0.4         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
 boto3==1.7.79
-botocore==1.10.84         # via boto3, s3transfer
+botocore==1.10.84
+celery[redis]==4.2.1
 certifi==2018.8.24
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
@@ -43,6 +46,7 @@ jinja2==2.10
 jmespath==0.9.3           # via boto3, botocore
 josepy==1.1.0             # via acme
 jsonlines==1.2.0          # via cloudflare
+kombu==4.2.1              # via celery
 lockfile==0.12.2
 mako==1.0.7               # via alembic
 markupsafe==1.0           # via jinja2, mako
@@ -64,17 +68,19 @@ pyrfc3339==1.1            # via acme
 python-dateutil==2.7.3    # via alembic, arrow, botocore
 python-editor==1.0.3      # via alembic
 python-ldap==3.1.0
-pytz==2018.5              # via acme, flask-restful, pyrfc3339
+pytz==2018.5              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==3.13              # via cloudflare
 raven[flask]==6.9.0
+redis==2.10.6             # via celery
 requests-toolbelt==0.8.0  # via acme
 requests[security]==2.19.1
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.11.0
-sqlalchemy-utils==0.33.3
+sqlalchemy-utils==0.33.4
 sqlalchemy==1.2.11        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
 urllib3==1.23             # via requests
+vine==1.1.4               # via amqp
 werkzeug==0.14.1          # via flask
 xmltodict==0.11.0