Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2789 from castrapel/celery-timeouts-LE-validation 

Add soft timeouts to celery jobs; Check for PEM in LE order
This commit is contained in:
Curtis 2019-05-14 14:09:02 -07:00 committed by GitHub
parent 0f2773c986 5d8f71c3e4
commit 302219325b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 69 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lemur/common/celery.py
View File

@ -12,9 +12,11 @@ import sys
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
from celery import Celery from celery import Celery
from celery.exceptions import SoftTimeLimitExceeded
from flask import current_app from flask import current_app
from lemur.authorities.service import get as get_authority from lemur.authorities.service import get as get_authority
from lemur.extensions import metrics, sentry
from lemur.factory import create_app from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service from lemur.pending_certificates import service as pending_certificate_service
@ -62,7 +64,7 @@ def is_task_active(fun, task_id, args):
return False return False
@celery.task() @celery.task(soft_time_limit=600)
def fetch_acme_cert(id): def fetch_acme_cert(id):
""" """
Attempt to get the full certificate for the pending certificate listed. Attempt to get the full certificate for the pending certificate listed.
@ -70,11 +72,24 @@ def fetch_acme_cert(id):
Args: Args:
id: an id of a PendingCertificate id: an id of a PendingCertificate
""" """
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = { log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
"message": "Resolving pending certificate {}".format(id) "message": "Resolving pending certificate {}".format(id),
"task_id": task_id,
"id": id,
} }
current_app.logger.debug(log_data) current_app.logger.debug(log_data)
if task_id and is_task_active(log_data["function"], task_id, (id,)):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
pending_certs = pending_certificate_service.get_pending_certs([id]) pending_certs = pending_certificate_service.get_pending_certs([id])
new = 0 new = 0
failed = 0 failed = 0
@ -192,7 +207,7 @@ def remove_old_acme_certs():
log_data['pending_cert_name'] = cert.name log_data['pending_cert_name'] = cert.name
log_data['message'] = "Deleting pending certificate" log_data['message'] = "Deleting pending certificate"
current_app.logger.debug(log_data) current_app.logger.debug(log_data)
pending_certificate_service.delete(cert.id) pending_certificate_service.delete(cert)
@celery.task() @celery.task()
@ -231,7 +246,7 @@ def sync_all_sources():
sync_source.delay(source.label) sync_source.delay(source.label)
@celery.task() @celery.task(soft_time_limit=3600)
def sync_source(source): def sync_source(source):
""" """
This celery task will sync the specified source. This celery task will sync the specified source.
@ -241,6 +256,8 @@ def sync_source(source):
""" """
function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) function = "{}.{}".format(__name__, sys._getframe().f_code.co_name)
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id task_id = celery.current_task.request.id
log_data = { log_data = {
"function": function, "function": function,
@ -250,11 +267,19 @@ def sync_source(source):
} }
current_app.logger.debug(log_data) current_app.logger.debug(log_data)
if is_task_active(function, task_id, (source,)): if task_id and is_task_active(function, task_id, (source,)):
log_data["message"] = "Skipping task: Task is already active" log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data) current_app.logger.debug(log_data)
return return
try:
sync([source]) sync([source])
except SoftTimeLimitExceeded:
log_data["message"] = "Error syncing source: Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send('sync_source_timeout', 'counter', 1, metric_tags={'source': source})
return
log_data["message"] = "Done syncing source" log_data["message"] = "Done syncing source"
current_app.logger.debug(log_data) current_app.logger.debug(log_data)

7
lemur/plugins/lemur_acme/plugin.py
View File

@ -17,7 +17,7 @@ import time
import OpenSSL.crypto import OpenSSL.crypto
import josepy as jose import josepy as jose
from acme import challenges, messages from acme import challenges, errors, messages
from acme.client import BackwardsCompatibleClientV2, ClientNetwork from acme.client import BackwardsCompatibleClientV2, ClientNetwork
from acme.errors import PollError, TimeoutError, WildcardUnsupportedError from acme.errors import PollError, TimeoutError, WildcardUnsupportedError
from acme.messages import Error as AcmeError from acme.messages import Error as AcmeError
@ -155,6 +155,11 @@ class AcmeHandler(object):
metrics.send('request_certificate_error', 'counter', 1) metrics.send('request_certificate_error', 'counter', 1)
current_app.logger.error(f"Unable to resolve Acme order: {order.uri}", exc_info=True) current_app.logger.error(f"Unable to resolve Acme order: {order.uri}", exc_info=True)
raise raise
except errors.ValidationError:
if order.fullchain_pem:
orderr = order
else:
raise
pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,

14
requirements-dev.txt
View File

@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile # This file is autogenerated by pip-compile
# To update, run: # To update, run:
# #
# pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in
# #
aspy.yaml==1.2.0 # via pre-commit aspy.yaml==1.2.0 # via pre-commit
bleach==3.1.0 # via readme-renderer bleach==3.1.0 # via readme-renderer
@ -11,26 +11,26 @@ cfgv==1.6.0 # via pre-commit
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
docutils==0.14 # via readme-renderer docutils==0.14 # via readme-renderer
flake8==3.5.0 flake8==3.5.0
identify==1.4.2 # via pre-commit identify==1.4.3 # via pre-commit
idna==2.8 # via requests idna==2.8 # via requests
importlib-metadata==0.9 # via pre-commit importlib-metadata==0.12 # via pre-commit
invoke==1.2.0 invoke==1.2.0
mccabe==0.6.1 # via flake8 mccabe==0.6.1 # via flake8
nodeenv==1.3.3 nodeenv==1.3.3
pkginfo==1.5.0.1 # via twine pkginfo==1.5.0.1 # via twine
pre-commit==1.16.0 pre-commit==1.16.1
pycodestyle==2.3.1 # via flake8 pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8 pyflakes==1.6.0 # via flake8
pygments==2.3.1 # via readme-renderer pygments==2.4.0 # via readme-renderer
pyyaml==5.1 pyyaml==5.1
readme-renderer==24.0 # via twine readme-renderer==24.0 # via twine
requests-toolbelt==0.9.1 # via twine requests-toolbelt==0.9.1 # via twine
requests==2.21.0 # via requests-toolbelt, twine requests==2.21.0 # via requests-toolbelt, twine
six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit toml==0.10.0 # via pre-commit
tqdm==4.31.1 # via twine tqdm==4.32.1 # via twine
twine==1.13.0 twine==1.13.0
urllib3==1.24.3 # via requests urllib3==1.24.3 # via requests
virtualenv==16.5.0 # via pre-commit virtualenv==16.5.0 # via pre-commit
webencodings==0.5.1 # via bleach webencodings==0.5.1 # via bleach
zipp==0.4.0 # via importlib-metadata zipp==0.5.0 # via importlib-metadata

14
requirements-docs.txt
View File

@ -2,9 +2,9 @@
# This file is autogenerated by pip-compile # This file is autogenerated by pip-compile
# To update, run: # To update, run:
# #
# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
# #
acme==0.34.1 acme==0.34.2
alabaster==0.7.12 # via sphinx alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2 alembic-autogenerate-enums==0.0.2
alembic==1.0.10 alembic==1.0.10
@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx
bcrypt==3.1.6 bcrypt==3.1.6
billiard==3.6.0.0 billiard==3.6.0.0
blinker==1.4 blinker==1.4
boto3==1.9.143 boto3==1.9.147
botocore==1.12.143 botocore==1.12.147
celery[redis]==4.3.0 celery[redis]==4.3.0
certifi==2019.3.9 certifi==2019.3.9
certsrv==2.1.1 certsrv==2.1.1
@ -54,11 +54,11 @@ josepy==1.1.0
jsonlines==1.2.0 jsonlines==1.2.0
kombu==4.5.0 kombu==4.5.0
lockfile==0.12.2 lockfile==0.12.2
mako==1.0.9 mako==1.0.10
markupsafe==1.1.1 markupsafe==1.1.1
marshmallow-sqlalchemy==0.16.3 marshmallow-sqlalchemy==0.16.3
marshmallow==2.19.2 marshmallow==2.19.2
mock==3.0.4 mock==3.0.5
ndg-httpsclient==0.5.1 ndg-httpsclient==0.5.1
packaging==19.0 # via sphinx packaging==19.0 # via sphinx
paramiko==2.4.2 paramiko==2.4.2
@ -68,7 +68,7 @@ pyasn1-modules==0.2.5
pyasn1==0.4.5 pyasn1==0.4.5
pycparser==2.19 pycparser==2.19
pycryptodomex==3.8.1 pycryptodomex==3.8.1
pygments==2.3.1 # via sphinx pygments==2.4.0 # via sphinx
pyjks==19.0.0 pyjks==19.0.0
pyjwt==1.7.1 pyjwt==1.7.1
pynacl==1.3.0 pynacl==1.3.0

21
requirements-tests.txt
View File

@ -2,19 +2,19 @@
# This file is autogenerated by pip-compile # This file is autogenerated by pip-compile
# To update, run: # To update, run:
# #
# pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index # pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in
# #
asn1crypto==0.24.0 # via cryptography asn1crypto==0.24.0 # via cryptography
atomicwrites==1.3.0 # via pytest atomicwrites==1.3.0 # via pytest
attrs==19.1.0 # via pytest attrs==19.1.0 # via pytest
aws-sam-translator==1.11.0 # via cfn-lint aws-sam-translator==1.11.0 # via cfn-lint
aws-xray-sdk==2.4.2 # via moto aws-xray-sdk==2.4.2 # via moto
boto3==1.9.143 # via aws-sam-translator, moto boto3==1.9.147 # via aws-sam-translator, moto
boto==2.49.0 # via moto boto==2.49.0 # via moto
botocore==1.12.143 # via aws-xray-sdk, boto3, moto, s3transfer botocore==1.12.147 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2019.3.9 # via requests certifi==2019.3.9 # via requests
cffi==1.12.3 # via cryptography cffi==1.12.3 # via cryptography
cfn-lint==0.19.1 # via moto cfn-lint==0.20.1 # via moto
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
click==7.0 # via flask click==7.0 # via flask
coverage==4.5.3 coverage==4.5.3
@ -23,8 +23,8 @@ docker-pycreds==0.4.0 # via docker
docker==3.7.2 # via moto docker==3.7.2 # via moto
docutils==0.14 # via botocore docutils==0.14 # via botocore
ecdsa==0.13.2 # via python-jose ecdsa==0.13.2 # via python-jose
factory-boy==2.11.1 factory-boy==2.12.0
faker==1.0.5 faker==1.0.7
flask==1.0.2 # via pytest-flask flask==1.0.2 # via pytest-flask
freezegun==0.3.11 freezegun==0.3.11
future==0.17.1 # via aws-xray-sdk, python-jose future==0.17.1 # via aws-xray-sdk, python-jose
@ -38,18 +38,18 @@ jsonpickle==1.1 # via aws-xray-sdk
jsonpointer==2.0 # via jsonpatch jsonpointer==2.0 # via jsonpatch
jsonschema==2.6.0 # via aws-sam-translator, cfn-lint jsonschema==2.6.0 # via aws-sam-translator, cfn-lint
markupsafe==1.1.1 # via jinja2 markupsafe==1.1.1 # via jinja2
mock==3.0.4 # via moto mock==3.0.5 # via moto
more-itertools==7.0.0 # via pytest more-itertools==7.0.0 # via pytest
moto==1.3.8 moto==1.3.8
nose==1.3.7 nose==1.3.7
pluggy==0.9.0 # via pytest pluggy==0.11.0 # via pytest
py==1.8.0 # via pytest py==1.8.0 # via pytest
pyasn1==0.4.5 # via rsa pyasn1==0.4.5 # via rsa
pycparser==2.19 # via cffi pycparser==2.19 # via cffi
pyflakes==2.1.1 pyflakes==2.1.1
pytest-flask==0.14.0 pytest-flask==0.15.0
pytest-mock==1.10.4 pytest-mock==1.10.4
pytest==4.4.1 pytest==4.5.0
python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
python-jose==3.0.1 # via moto python-jose==3.0.1 # via moto
pytz==2019.1 # via moto pytz==2019.1 # via moto
@ -62,6 +62,7 @@ s3transfer==0.2.0 # via boto3
six==1.12.0 # via aws-sam-translator, cfn-lint, cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client six==1.12.0 # via aws-sam-translator, cfn-lint, cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker text-unidecode==1.2 # via faker
urllib3==1.24.3 # via botocore, requests urllib3==1.24.3 # via botocore, requests
wcwidth==0.1.7 # via pytest
websocket-client==0.56.0 # via docker websocket-client==0.56.0 # via docker
werkzeug==0.15.2 # via flask, moto, pytest-flask werkzeug==0.15.2 # via flask, moto, pytest-flask
wrapt==1.11.1 # via aws-xray-sdk wrapt==1.11.1 # via aws-xray-sdk

12
requirements.txt
View File

@ -2,9 +2,9 @@
# This file is autogenerated by pip-compile # This file is autogenerated by pip-compile
# To update, run: # To update, run:
# #
# pip-compile --output-file requirements.txt requirements.in -U --no-index # pip-compile --no-index --output-file=requirements.txt requirements.in
# #
acme==0.34.1 acme==0.34.2
alembic-autogenerate-enums==0.0.2 alembic-autogenerate-enums==0.0.2
alembic==1.0.10 # via flask-migrate alembic==1.0.10 # via flask-migrate
amqp==2.4.2 # via kombu amqp==2.4.2 # via kombu
@ -15,8 +15,8 @@ asyncpool==1.0
bcrypt==3.1.6 # via flask-bcrypt, paramiko bcrypt==3.1.6 # via flask-bcrypt, paramiko
billiard==3.6.0.0 # via celery billiard==3.6.0.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.9.143 boto3==1.9.147
botocore==1.12.143 botocore==1.12.147
celery[redis]==4.3.0 celery[redis]==4.3.0
certifi==2019.3.9 certifi==2019.3.9
certsrv==2.1.1 certsrv==2.1.1
@ -51,11 +51,11 @@ josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare jsonlines==1.2.0 # via cloudflare
kombu==4.5.0 kombu==4.5.0
lockfile==0.12.2 lockfile==0.12.2
mako==1.0.9 # via alembic mako==1.0.10 # via alembic
markupsafe==1.1.1 # via jinja2, mako markupsafe==1.1.1 # via jinja2, mako
marshmallow-sqlalchemy==0.16.3 marshmallow-sqlalchemy==0.16.3
marshmallow==2.19.2 marshmallow==2.19.2
mock==3.0.4 # via acme mock==3.0.5 # via acme
ndg-httpsclient==0.5.1 ndg-httpsclient==0.5.1
paramiko==2.4.2 paramiko==2.4.2
pem==19.1.0 pem==19.1.0