From 565142f98557f02cd54e93e88c83edcb46ca11ab Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 14 May 2019 12:52:30 -0700 Subject: [PATCH 1/2] Add soft timeouts to celery jobs; Check for PEM in LE order --- lemur/common/celery.py | 38 ++++++++++++++++++++++++------ lemur/plugins/lemur_acme/plugin.py | 7 +++++- requirements-dev.txt | 14 +++++------ requirements-docs.txt | 14 +++++------ requirements-tests.txt | 21 +++++++++-------- requirements.txt | 12 +++++----- 6 files changed, 68 insertions(+), 38 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 10747d31..45e3fd78 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -12,9 +12,11 @@ import sys from datetime import datetime, timezone, timedelta from celery import Celery +from celery.exceptions import SoftTimeLimitExceeded from flask import current_app from lemur.authorities.service import get as get_authority +from lemur.extensions import metrics, sentry from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service @@ -62,7 +64,7 @@ def is_task_active(fun, task_id, args): return False -@celery.task() +@celery.task(soft_time_limit=600) def fetch_acme_cert(id): """ Attempt to get the full certificate for the pending certificate listed. @@ -70,11 +72,24 @@ def fetch_acme_cert(id): Args: id: an id of a PendingCertificate """ + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), - "message": "Resolving pending certificate {}".format(id) + "message": "Resolving pending certificate {}".format(id), + "task_id": task_id, + "id": id, } + current_app.logger.debug(log_data) + + if task_id and is_task_active(log_data["function"], task_id, (id,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + pending_certs = pending_certificate_service.get_pending_certs([id]) new = 0 failed = 0 @@ -192,7 +207,7 @@ def remove_old_acme_certs(): log_data['pending_cert_name'] = cert.name log_data['message'] = "Deleting pending certificate" current_app.logger.debug(log_data) - pending_certificate_service.delete(cert.id) + pending_certificate_service.delete(cert) @celery.task() @@ -231,7 +246,7 @@ def sync_all_sources(): sync_source.delay(source.label) -@celery.task() +@celery.task(soft_time_limit=3600) def sync_source(source): """ This celery task will sync the specified source. @@ -241,7 +256,9 @@ def sync_source(source): """ function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) - task_id = celery.current_task.request.id + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id log_data = { "function": function, "message": "Syncing source", @@ -250,11 +267,18 @@ def sync_source(source): } current_app.logger.debug(log_data) - if is_task_active(function, task_id, (source,)): + if task_id and is_task_active(function, task_id, (source,)): log_data["message"] = "Skipping task: Task is already active" current_app.logger.debug(log_data) return - sync([source]) + try: + sync([source]) + except SoftTimeLimitExceeded: + log_data["message"] = "Error syncing source: Time limit exceeded." + sentry.captureException() + metrics.send('sync_source_timeout', 'counter', 1, metric_tags={'source': source}) + return + log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 06dec882..d9c41968 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -17,7 +17,7 @@ import time import OpenSSL.crypto import josepy as jose -from acme import challenges, messages +from acme import challenges, errors, messages from acme.client import BackwardsCompatibleClientV2, ClientNetwork from acme.errors import PollError, TimeoutError, WildcardUnsupportedError from acme.messages import Error as AcmeError @@ -155,6 +155,11 @@ class AcmeHandler(object): metrics.send('request_certificate_error', 'counter', 1) current_app.logger.error(f"Unable to resolve Acme order: {order.uri}", exc_info=True) raise + except errors.ValidationError: + if order.fullchain_pem: + orderr = order + else: + raise pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, diff --git a/requirements-dev.txt b/requirements-dev.txt index 29509d99..1a5b5f9d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index +# pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.2.0 # via pre-commit bleach==3.1.0 # via readme-renderer @@ -11,26 +11,26 @@ cfgv==1.6.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.4.2 # via pre-commit +identify==1.4.3 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.9 # via pre-commit +importlib-metadata==0.12 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.16.0 +pre-commit==1.16.1 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 -pygments==2.3.1 # via readme-renderer +pygments==2.4.0 # via readme-renderer pyyaml==5.1 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.31.1 # via twine +tqdm==4.32.1 # via twine twine==1.13.0 urllib3==1.24.3 # via requests virtualenv==16.5.0 # via pre-commit webencodings==0.5.1 # via bleach -zipp==0.4.0 # via importlib-metadata +zipp==0.5.0 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index fef37c08..f23de8f4 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index +# pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.34.1 +acme==0.34.2 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.10 @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.6.0.0 blinker==1.4 -boto3==1.9.143 -botocore==1.12.143 +boto3==1.9.147 +botocore==1.12.147 celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 @@ -54,11 +54,11 @@ josepy==1.1.0 jsonlines==1.2.0 kombu==4.5.0 lockfile==0.12.2 -mako==1.0.9 +mako==1.0.10 markupsafe==1.1.1 marshmallow-sqlalchemy==0.16.3 marshmallow==2.19.2 -mock==3.0.4 +mock==3.0.5 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 @@ -68,7 +68,7 @@ pyasn1-modules==0.2.5 pyasn1==0.4.5 pycparser==2.19 pycryptodomex==3.8.1 -pygments==2.3.1 # via sphinx +pygments==2.4.0 # via sphinx pyjks==19.0.0 pyjwt==1.7.1 pynacl==1.3.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 5d28412c..27837359 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,19 +2,19 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index +# pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-sam-translator==1.11.0 # via cfn-lint aws-xray-sdk==2.4.2 # via moto -boto3==1.9.143 # via aws-sam-translator, moto +boto3==1.9.147 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.143 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.12.147 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2019.3.9 # via requests cffi==1.12.3 # via cryptography -cfn-lint==0.19.1 # via moto +cfn-lint==0.20.1 # via moto chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.3 @@ -23,8 +23,8 @@ docker-pycreds==0.4.0 # via docker docker==3.7.2 # via moto docutils==0.14 # via botocore ecdsa==0.13.2 # via python-jose -factory-boy==2.11.1 -faker==1.0.5 +factory-boy==2.12.0 +faker==1.0.7 flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.17.1 # via aws-xray-sdk, python-jose @@ -38,18 +38,18 @@ jsonpickle==1.1 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==2.6.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 -mock==3.0.4 # via moto +mock==3.0.5 # via moto more-itertools==7.0.0 # via pytest moto==1.3.8 nose==1.3.7 -pluggy==0.9.0 # via pytest +pluggy==0.11.0 # via pytest py==1.8.0 # via pytest pyasn1==0.4.5 # via rsa pycparser==2.19 # via cffi pyflakes==2.1.1 -pytest-flask==0.14.0 +pytest-flask==0.15.0 pytest-mock==1.10.4 -pytest==4.4.1 +pytest==4.5.0 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto pytz==2019.1 # via moto @@ -62,6 +62,7 @@ s3transfer==0.2.0 # via boto3 six==1.12.0 # via aws-sam-translator, cfn-lint, cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.3 # via botocore, requests +wcwidth==0.1.7 # via pytest websocket-client==0.56.0 # via docker werkzeug==0.15.2 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements.txt b/requirements.txt index fe27838b..935e85ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements.txt requirements.in -U --no-index +# pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.34.1 +acme==0.34.2 alembic-autogenerate-enums==0.0.2 alembic==1.0.10 # via flask-migrate amqp==2.4.2 # via kombu @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.6.0.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.143 -botocore==1.12.143 +boto3==1.9.147 +botocore==1.12.147 celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 @@ -51,11 +51,11 @@ josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.5.0 lockfile==0.12.2 -mako==1.0.9 # via alembic +mako==1.0.10 # via alembic markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.16.3 marshmallow==2.19.2 -mock==3.0.4 # via acme +mock==3.0.5 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pem==19.1.0 From 5d8f71c3e405d8bede63f844383f8de97b47f296 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 14 May 2019 13:02:24 -0700 Subject: [PATCH 2/2] nt --- lemur/common/celery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 45e3fd78..ce386ffd 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -275,6 +275,7 @@ def sync_source(source): sync([source]) except SoftTimeLimitExceeded: log_data["message"] = "Error syncing source: Time limit exceeded." + current_app.logger.error(log_data) sentry.captureException() metrics.send('sync_source_timeout', 'counter', 1, metric_tags={'source': source}) return