diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 13c0f8bf..57592d56 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -5,32 +5,27 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import arrow - -from flask import current_app +from cryptography import x509 from sqlalchemy import func, or_, not_, cast, Integer -from cryptography import x509 +import arrow from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization +from flask import current_app from lemur import database -from lemur.extensions import metrics, sentry, signals -from lemur.plugins.base import plugins -from lemur.common.utils import generate_private_key, truthiness - -from lemur.roles.models import Role -from lemur.domains.models import Domain from lemur.authorities.models import Authority -from lemur.destinations.models import Destination from lemur.certificates.models import Certificate +from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema +from lemur.common.utils import generate_private_key, truthiness +from lemur.destinations.models import Destination +from lemur.domains.models import Domain +from lemur.extensions import metrics, sentry, signals from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate - -from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema - +from lemur.plugins.base import plugins from lemur.roles import service as role_service - +from lemur.roles.models import Role csr_created = signals.signal('csr_created', "CSR generated") csr_imported = signals.signal('csr_imported', "CSR imported from external source") @@ -95,8 +90,8 @@ def get_all_pending_cleaning(source): :param source: :return: """ - return Certificate.query.filter(Certificate.sources.any(id=source.id))\ - .filter(not_(Certificate.endpoints.any())).all() + return Certificate.query.filter(Certificate.sources.any(id=source.id)) \ + .filter(not_(Certificate.endpoints.any())).all() def get_all_pending_reissue(): @@ -109,8 +104,8 @@ def get_all_pending_reissue(): :return: """ - return Certificate.query.filter(Certificate.rotation == True)\ - .filter(not_(Certificate.replaced.any()))\ + return Certificate.query.filter(Certificate.rotation == True) \ + .filter(not_(Certificate.replaced.any())) \ .filter(Certificate.in_rotation_window == True).all() # noqa @@ -280,6 +275,11 @@ def create(**kwargs): if isinstance(cert, Certificate): certificate_issued.send(certificate=cert, authority=cert.authority) metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer)) + + if isinstance(cert, PendingCertificate) and cert.authority.plugin_name == 'acme-issuer': + # Call Celery to create acme-issuer (LetsEncrypt) certificates + from lemur.common.celery import fetch_acme_cert + fetch_acme_cert.delay(cert.id) return cert @@ -310,8 +310,8 @@ def render(args): if 'issuer' in terms: # we can't rely on issuer being correct in the cert directly so we combine queries - sub_query = database.session_query(Authority.id)\ - .filter(Authority.name.ilike(term))\ + sub_query = database.session_query(Authority.id) \ + .filter(Authority.name.ilike(term)) \ .subquery() query = query.filter( @@ -450,8 +450,8 @@ def stats(**kwargs): if kwargs.get('metric') == 'not_after': start = arrow.utcnow() end = start.replace(weeks=+32) - items = database.db.session.query(Certificate.issuer, func.count(Certificate.id))\ - .group_by(Certificate.issuer)\ + items = database.db.session.query(Certificate.issuer, func.count(Certificate.id)) \ + .group_by(Certificate.issuer) \ .filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \ .filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all() diff --git a/lemur/common/celery.py b/lemur/common/celery.py new file mode 100644 index 00000000..fc19df32 --- /dev/null +++ b/lemur/common/celery.py @@ -0,0 +1,119 @@ +""" +This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start +when invoked. + +When ran in development mode (LEMUR_CONFIG= 4: + error_log["message"] = "Deleting pending certificate" + send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) + pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) + else: + pending_certificate_service.increment_attempt(pending_cert) + pending_certificate_service.update( + cert.get("pending_cert").id, + status=str(cert.get("last_error")) + ) + # Add failed pending cert task back to queue + fetch_acme_cert.delay(id) + current_app.logger.error(error_log) + log_data["message"] = "Complete" + log_data["new"] = new + log_data["failed"] = failed + log_data["wrong_issuer"] = wrong_issuer + current_app.logger.debug(log_data) + print( + "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( + new=new, + failed=failed, + wrong_issuer=wrong_issuer + ) + ) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9b8728ae..8c6b8b2f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,26 +5,26 @@ # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # aspy.yaml==1.1.1 # via pre-commit -cached-property==1.4.3 # via pre-commit +cached-property==1.5.1 # via pre-commit certifi==2018.8.24 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.1.4 # via pre-commit +identify==1.1.5 # via pre-commit idna==2.7 # via requests invoke==1.1.1 mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine -pre-commit==1.10.5 +pre-commit==1.11.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.13 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.19.1 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -toml==0.9.4 # via pre-commit -tqdm==4.25.0 # via twine +toml==0.9.6 # via pre-commit +tqdm==4.26.0 # via twine twine==1.11.0 urllib3==1.23 # via requests virtualenv==16.0.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index f3278b66..7e8d2d5e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,19 +4,22 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.26.1 +acme==0.27.1 alabaster==0.7.11 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.0 +amqp==2.3.2 aniso8601==3.0.2 arrow==0.12.1 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 +billiard==3.5.0.4 blinker==1.4 -boto3==1.8.1 -botocore==1.11.1 +boto3==1.7.79 +botocore==1.10.84 +celery[redis]==4.2.1 certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 @@ -39,13 +42,14 @@ flask==0.12 future==0.16.0 gunicorn==19.9.0 idna==2.7 -imagesize==1.0.0 # via sphinx +imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==0.24 jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 +kombu==4.2.1 lockfile==0.12.2 mako==1.0.7 markupsafe==1.0 @@ -72,6 +76,7 @@ python-editor==1.0.3 pytz==2018.5 pyyaml==3.13 raven[flask]==6.9.0 +redis==2.10.6 requests-toolbelt==0.8.0 requests[security]==2.19.1 retrying==1.3.3 @@ -79,12 +84,13 @@ s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.1 -sphinx==1.7.7 +sphinx==1.8.0 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 -sqlalchemy==1.2.10 +sqlalchemy==1.2.11 tabulate==0.8.2 urllib3==1.23 +vine==1.1.4 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.in b/requirements-tests.in index 02a2b0ae..efb4570a 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -4,7 +4,7 @@ coverage factory-boy Faker freezegun -moto +moto==1.3.4 # Issue with moto: https://github.com/spulec/moto/issues/1813 nose pyflakes pytest diff --git a/requirements-tests.txt b/requirements-tests.txt index a34bf1fd..1e67cbf3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,12 +5,12 @@ # pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography -atomicwrites==1.2.0 # via pytest -attrs==18.1.0 # via pytest +atomicwrites==1.2.1 # via pytest +attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.8.1 # via moto +boto3==1.9.3 # via moto boto==2.49.0 # via moto -botocore==1.11.1 # via boto3, moto, s3transfer +botocore==1.12.3 # via boto3, moto, s3transfer certifi==2018.8.24 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -40,14 +40,14 @@ moto==1.3.4 nose==1.3.7 pbr==4.2.0 # via mock pluggy==0.7.1 # via pytest -py==1.5.4 # via pytest +py==1.6.0 # via pytest pyaml==17.12.1 # via moto pycparser==2.18 # via cffi pycryptodome==3.6.6 # via python-jose pyflakes==2.0.0 -pytest-flask==0.10.0 +pytest-flask==0.12.0 pytest-mock==1.10.0 -pytest==3.7.3 +pytest==3.8.0 python-dateutil==2.7.3 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.5 # via moto @@ -59,7 +59,7 @@ s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.23 # via botocore, requests -websocket-client==0.51.0 # via docker +websocket-client==0.53.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.10.11 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.in b/requirements.in index 9d77bee2..a2b920c3 100644 --- a/requirements.in +++ b/requirements.in @@ -4,7 +4,9 @@ acme alembic-autogenerate-enums arrow asyncpool -boto3==1.7.79 +boto3==1.7.79 # Issue with moto: https://github.com/spulec/moto/issues/1813 +botocore== 1.10.84 # Issue with moto: https://github.com/spulec/moto/issues/1813 +celery[redis] certifi CloudFlare cryptography diff --git a/requirements.txt b/requirements.txt index 6da0f82b..43ebdb63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,17 +4,20 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.26.1 +acme==0.27.1 alembic-autogenerate-enums==0.0.2 alembic==1.0.0 # via flask-migrate +amqp==2.3.2 # via kombu aniso8601==3.0.2 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko +billiard==3.5.0.4 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.7.79 -botocore==1.10.84 # via boto3, s3transfer +botocore==1.10.84 +celery[redis]==4.2.1 certifi==2018.8.24 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -43,6 +46,7 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare +kombu==4.2.1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.0 # via jinja2, mako @@ -64,17 +68,19 @@ pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.5 # via acme, flask-restful, pyrfc3339 +pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 +redis==2.10.6 # via celery requests-toolbelt==0.8.0 # via acme requests[security]==2.19.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.3 +sqlalchemy-utils==0.33.4 sqlalchemy==1.2.11 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.23 # via requests +vine==1.1.4 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0