Merge pull request #2158 from castrapel/celery_pending

Add async call to create pending cert when needed

lemur/certificates/models.py

@@ -138,7 +138,6 @@ class Certificate(db.Model):
     logs = relationship('Log', backref='certificate')
     endpoints = relationship('Endpoint', backref='certificate')
     rotation_policy = relationship("RotationPolicy")
-
     sensitive_fields = ('private_key',)
 
     def __init__(self, **kwargs):

lemur/certificates/service.py

@@ -276,6 +276,14 @@ def create(**kwargs):
         certificate_issued.send(certificate=cert, authority=cert.authority)
         metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
 
+    if isinstance(cert, PendingCertificate):
+        # We need to refresh the pending certificate to avoid "Instance is not bound to a Session; "
+        # "attribute refresh operation cannot proceed"
+        pending_cert = database.session_query(PendingCertificate).get(cert.id)
+        from lemur.common.celery import fetch_acme_cert
+        if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False):
+            fetch_acme_cert.apply_async((pending_cert.id,), countdown=5)
+
     return cert
 
 

lemur/common/celery.py

@@ -25,8 +25,8 @@ flask_app = create_app()
 
 
 def make_celery(app):
-    celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
+    celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'),
-                    broker=app.config['CELERY_BROKER_URL'])
+                    broker=app.config.get('CELERY_BROKER_URL'))
     celery.conf.update(app.config)
     TaskBase = celery.Task
 

lemur/manage.py

@@ -47,7 +47,7 @@ from lemur.logs.models import Log  # noqa
 from lemur.endpoints.models import Endpoint  # noqa
 from lemur.policies.models import RotationPolicy  # noqa
 from lemur.pending_certificates.models import PendingCertificate  # noqa
-
+from lemur.dns_providers.models import DnsProvider  # noqa
 
 manager = Manager(create_app)
 manager.add_option('-c', '--config', dest='config')

lemur/tests/conf.py

@@ -1,7 +1,7 @@
-
 # This is just Python which means you can inherit and tweak settings
 
 import os
+
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
 THREADS_PER_PAGE = 8
@@ -78,14 +78,12 @@ DIGICERT_API_KEY = 'api-key'
 DIGICERT_ORG_ID = 111111
 DIGICERT_ROOT = "ROOT"
 
-
 VERISIGN_URL = 'http://example.com'
 VERISIGN_PEM_PATH = '~/'
 VERISIGN_FIRST_NAME = 'Jim'
 VERISIGN_LAST_NAME = 'Bob'
 VERSIGN_EMAIL = 'jim@example.com'
 
-
 ACME_AWS_ACCOUNT_NUMBER = '11111111111'
 
 ACME_PRIVATE_KEY = '''
@@ -180,6 +178,7 @@ ACME_URL = 'https://acme-v01.api.letsencrypt.org'
 ACME_EMAIL = 'jim@example.com'
 ACME_TEL = '4088675309'
 ACME_DIRECTORY_URL = 'https://acme-v01.api.letsencrypt.org'
+ACME_DISABLE_AUTORESOLVE = True
 
 LDAP_AUTH = True
 LDAP_BIND_URI = 'ldap://localhost'

lemur/tests/test_pending_certificates.py

@@ -2,11 +2,10 @@ import json
 
 import pytest
 
+from lemur.pending_certificates.views import *  # noqa
 from .vectors import CSR_STR, INTERMEDIATE_CERT_STR, VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, \
     VALID_USER_HEADER_TOKEN, WILDCARD_CERT_STR
 
-from lemur.pending_certificates.views import *  # noqa
-
 
 def test_increment_attempt(pending_certificate):
     from lemur.pending_certificates.service import increment_attempt
@@ -17,7 +16,8 @@ def test_increment_attempt(pending_certificate):
 
 def test_create_pending_certificate(async_issuer_plugin, async_authority, user):
     from lemur.certificates.service import create
-    pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'], common_name='ACommonName')
+    pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'],
+                          common_name='ACommonName')
     assert pending_cert.external_id == '12345'
 
 

requirements-dev.txt

@@ -14,7 +14,7 @@ docutils==0.14            # via readme-renderer
 flake8==3.5.0
 identify==1.1.7           # via pre-commit
 idna==2.7                 # via requests
-importlib-metadata==0.6   # via pre-commit
+importlib-metadata==0.7   # via pre-commit
 importlib-resources==1.0.2  # via pre-commit
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
@@ -23,7 +23,7 @@ pkginfo==1.4.2            # via twine
 pre-commit==1.12.0
 pycodestyle==2.3.1        # via flake8
 pyflakes==1.6.0           # via flake8
-pygments==2.2.0           # via readme-renderer
+pygments==2.3.0           # via readme-renderer
 pyyaml==3.13              # via aspy.yaml, pre-commit
 readme-renderer==24.0     # via twine
 requests-toolbelt==0.8.0  # via twine

requirements-docs.txt

@@ -4,10 +4,10 @@
 #
 #    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
-acme==0.27.1
+acme==0.28.0
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.2
+alembic==1.0.5
 amqp==2.3.2
 aniso8601==4.0.1
 arrow==0.12.1
@@ -17,23 +17,23 @@ babel==2.6.0              # via sphinx
 bcrypt==3.1.4
 billiard==3.5.0.4
 blinker==1.4
-boto3==1.9.37
+boto3==1.9.53
-botocore==1.12.37
+botocore==1.12.53
 celery[redis]==4.2.1
 certifi==2018.10.15
 cffi==1.11.5
 chardet==3.0.4
 click==7.0
 cloudflare==2.1.0
-cryptography==2.3.1
+cryptography==2.4.2
 dnspython3==1.15.0
 dnspython==1.15.0
 docutils==0.14
 dyn==1.8.1
 flask-bcrypt==0.7.1
-flask-cors==3.0.6
+flask-cors==3.0.7
 flask-mail==0.9.1
-flask-migrate==2.3.0
+flask-migrate==2.3.1
 flask-principal==0.4.0
 flask-restful==0.3.6
 flask-script==2.0.6
@@ -61,11 +61,11 @@ packaging==18.0           # via sphinx
 paramiko==2.4.2
 pbr==5.1.1
 pem==18.2.0
-psycopg2==2.7.5
+psycopg2==2.7.6.1
 pyasn1-modules==0.2.2
 pyasn1==0.4.4
 pycparser==2.19
-pygments==2.2.0           # via sphinx
+pygments==2.3.0           # via sphinx
 pyjwt==1.6.4
 pynacl==1.3.0
 pyopenssl==18.0.0
@@ -76,9 +76,9 @@ python-editor==1.0.3
 pytz==2018.7
 pyyaml==3.13
 raven[flask]==6.9.0
-redis==2.10.6
+redis==3.0.1
 requests-toolbelt==0.8.0
-requests[security]==2.20.0
+requests[security]==2.20.1
 retrying==1.3.3
 s3transfer==0.1.13
 six==1.11.0
@@ -87,8 +87,8 @@ sphinx-rtd-theme==0.4.2
 sphinx==1.8.2
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx
-sqlalchemy-utils==0.33.6
+sqlalchemy-utils==0.33.8
-sqlalchemy==1.2.13
+sqlalchemy==1.2.14
 tabulate==0.8.2
 urllib3==1.24.1
 vine==1.1.4

requirements-tests.txt

@@ -8,21 +8,21 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.9.42             # via moto
+boto3==1.9.53             # via moto
 boto==2.49.0              # via moto
-botocore==1.12.42         # via boto3, moto, s3transfer
+botocore==1.12.53         # via boto3, moto, s3transfer
 certifi==2018.10.15       # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
 click==7.0                # via flask
 coverage==4.5.2
-cryptography==2.4.1       # via moto
+cryptography==2.4.2       # via moto
 docker-pycreds==0.3.0     # via docker
 docker==3.5.1             # via moto
 docutils==0.14            # via botocore
 ecdsa==0.13               # via python-jose
 factory-boy==2.11.1
-faker==0.9.2
+faker==1.0.0
 flask==1.0.2              # via pytest-flask
 freezegun==0.3.11
 future==0.17.1            # via python-jose
@@ -40,20 +40,20 @@ nose==1.3.7
 pbr==5.1.1                # via mock
 pluggy==0.8.0             # via pytest
 py==1.7.0                 # via pytest
-pyaml==17.12.1            # via moto
+pyaml==18.11.0            # via moto
 pycparser==2.19           # via cffi
-pycryptodome==3.7.0       # via python-jose
+pycryptodome==3.7.2       # via python-jose
 pyflakes==2.0.0
 pytest-flask==0.14.0
 pytest-mock==1.10.0
-pytest==3.10.1
+pytest==4.0.1
 python-dateutil==2.7.5    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
 pytz==2018.7              # via moto
 pyyaml==3.13              # via pyaml
 requests-mock==1.5.2
 requests==2.20.1          # via aws-xray-sdk, docker, moto, requests-mock, responses
-responses==0.10.3         # via moto
+responses==0.10.4         # via moto
 s3transfer==0.1.13        # via boto3
 six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
 text-unidecode==1.2       # via faker

requirements.in

@@ -36,6 +36,7 @@ pyjwt
 pyOpenSSL
 python_ldap
 raven[flask]
+redis<3 # redis>=3 is not compatible with celery
 requests
 retrying
 six

requirements.txt

@@ -6,7 +6,7 @@
 #
 acme==0.28.0
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.2            # via flask-migrate
+alembic==1.0.5            # via flask-migrate
 amqp==2.3.2               # via kombu
 aniso8601==4.0.1          # via flask-restful
 arrow==0.12.1
@@ -15,15 +15,15 @@ asyncpool==1.0
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
 billiard==3.5.0.4         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.9.42
+boto3==1.9.53
-botocore==1.12.42
+botocore==1.12.53
 celery[redis]==4.2.1
 certifi==2018.10.15
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
 click==7.0                # via flask
 cloudflare==2.1.0
-cryptography==2.4.1
+cryptography==2.4.2
 dnspython3==1.15.0
 dnspython==1.15.0         # via dnspython3
 docutils==0.14            # via botocore
@@ -31,7 +31,7 @@ dyn==1.8.1
 flask-bcrypt==0.7.1
 flask-cors==3.0.7
 flask-mail==0.9.1
-flask-migrate==2.3.0
+flask-migrate==2.3.1
 flask-principal==0.4.0
 flask-restful==0.3.6
 flask-script==2.0.6
@@ -71,13 +71,13 @@ python-ldap==3.1.0
 pytz==2018.7              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==3.13              # via cloudflare
 raven[flask]==6.9.0
-redis==2.10.6             # via celery
+redis==2.10.6
 requests-toolbelt==0.8.0  # via acme
 requests[security]==2.20.1
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.11.0
-sqlalchemy-utils==0.33.6
+sqlalchemy-utils==0.33.8
 sqlalchemy==1.2.14        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
 urllib3==1.24.1           # via botocore, requests