Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add soft timeouts to celery jobs; Check for PEM in LE order

This commit is contained in:
Curtis Castrapel 2019-05-14 12:52:30 -07:00
parent 0f2773c986
commit 565142f985
6 changed files with 68 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
lemur/common/celery.py
View File

@ -12,9 +12,11 @@ import sys
from datetime import datetime, timezone, timedelta
from celery import Celery
from celery.exceptions import SoftTimeLimitExceeded
from flask import current_app
from lemur.authorities.service import get as get_authority
from lemur.extensions import metrics, sentry
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
@ -62,7 +64,7 @@ def is_task_active(fun, task_id, args):
return False
@celery.task()
@celery.task(soft_time_limit=600)
def fetch_acme_cert(id):
"""
Attempt to get the full certificate for the pending certificate listed.
@ -70,11 +72,24 @@ def fetch_acme_cert(id):
Args:
id: an id of a PendingCertificate
"""
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name),
"message": "Resolving pending certificate {}".format(id)
"message": "Resolving pending certificate {}".format(id),
"task_id": task_id,
"id": id,
}
current_app.logger.debug(log_data)
if task_id and is_task_active(log_data["function"], task_id, (id,)):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
pending_certs = pending_certificate_service.get_pending_certs([id])
new = 0
failed = 0
@ -192,7 +207,7 @@ def remove_old_acme_certs():
log_data['pending_cert_name'] = cert.name
log_data['message'] = "Deleting pending certificate"
current_app.logger.debug(log_data)
pending_certificate_service.delete(cert.id)
pending_certificate_service.delete(cert)
@celery.task()
@ -231,7 +246,7 @@ def sync_all_sources():
sync_source.delay(source.label)
@celery.task()
@celery.task(soft_time_limit=3600)
def sync_source(source):
"""
This celery task will sync the specified source.
@ -241,7 +256,9 @@ def sync_source(source):
"""
function = "{}.{}".format(__name__, sys._getframe().f_code.co_name)
task_id = celery.current_task.request.id
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "Syncing source",
@ -250,11 +267,18 @@ def sync_source(source):
}
current_app.logger.debug(log_data)
if is_task_active(function, task_id, (source,)):
if task_id and is_task_active(function, task_id, (source,)):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
sync([source])
try:
sync([source])
except SoftTimeLimitExceeded:
log_data["message"] = "Error syncing source: Time limit exceeded."
sentry.captureException()
metrics.send('sync_source_timeout', 'counter', 1, metric_tags={'source': source})
return
log_data["message"] = "Done syncing source"
current_app.logger.debug(log_data)

7
lemur/plugins/lemur_acme/plugin.py
View File

@ -17,7 +17,7 @@ import time
import OpenSSL.crypto
import josepy as jose
from acme import challenges, messages
from acme import challenges, errors, messages
from acme.client import BackwardsCompatibleClientV2, ClientNetwork
from acme.errors import PollError, TimeoutError, WildcardUnsupportedError
from acme.messages import Error as AcmeError
@ -155,6 +155,11 @@ class AcmeHandler(object):
metrics.send('request_certificate_error', 'counter', 1)
current_app.logger.error(f"Unable to resolve Acme order: {order.uri}", exc_info=True)
raise
except errors.ValidationError:
if order.fullchain_pem:
orderr = order
else:
raise
pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,

14
requirements-dev.txt
View File

@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
# pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in
#
aspy.yaml==1.2.0 # via pre-commit
bleach==3.1.0 # via readme-renderer
@ -11,26 +11,26 @@ cfgv==1.6.0 # via pre-commit
chardet==3.0.4 # via requests
docutils==0.14 # via readme-renderer
flake8==3.5.0
identify==1.4.2 # via pre-commit
identify==1.4.3 # via pre-commit
idna==2.8 # via requests
importlib-metadata==0.9 # via pre-commit
importlib-metadata==0.12 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.3
pkginfo==1.5.0.1 # via twine
pre-commit==1.16.0
pre-commit==1.16.1
pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8
pygments==2.3.1 # via readme-renderer
pygments==2.4.0 # via readme-renderer
pyyaml==5.1
readme-renderer==24.0 # via twine
requests-toolbelt==0.9.1 # via twine
requests==2.21.0 # via requests-toolbelt, twine
six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.31.1 # via twine
tqdm==4.32.1 # via twine
twine==1.13.0
urllib3==1.24.3 # via requests
virtualenv==16.5.0 # via pre-commit
webencodings==0.5.1 # via bleach
zipp==0.4.0 # via importlib-metadata
zipp==0.5.0 # via importlib-metadata

14
requirements-docs.txt
View File

@ -2,9 +2,9 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
# pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
#
acme==0.34.1
acme==0.34.2
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.10
@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx
bcrypt==3.1.6
billiard==3.6.0.0
blinker==1.4
boto3==1.9.143
botocore==1.12.143
boto3==1.9.147
botocore==1.12.147
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
@ -54,11 +54,11 @@ josepy==1.1.0
jsonlines==1.2.0
kombu==4.5.0
lockfile==0.12.2
mako==1.0.9
mako==1.0.10
markupsafe==1.1.1
marshmallow-sqlalchemy==0.16.3
marshmallow==2.19.2
mock==3.0.4
mock==3.0.5
ndg-httpsclient==0.5.1
packaging==19.0 # via sphinx
paramiko==2.4.2
@ -68,7 +68,7 @@ pyasn1-modules==0.2.5
pyasn1==0.4.5
pycparser==2.19
pycryptodomex==3.8.1
pygments==2.3.1 # via sphinx
pygments==2.4.0 # via sphinx
pyjks==19.0.0
pyjwt==1.7.1
pynacl==1.3.0

21
requirements-tests.txt
View File

@ -2,19 +2,19 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
# pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in
#
asn1crypto==0.24.0 # via cryptography
atomicwrites==1.3.0 # via pytest
attrs==19.1.0 # via pytest
aws-sam-translator==1.11.0 # via cfn-lint
aws-xray-sdk==2.4.2 # via moto
boto3==1.9.143 # via aws-sam-translator, moto
boto3==1.9.147 # via aws-sam-translator, moto
boto==2.49.0 # via moto
botocore==1.12.143 # via aws-xray-sdk, boto3, moto, s3transfer
botocore==1.12.147 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2019.3.9 # via requests
cffi==1.12.3 # via cryptography
cfn-lint==0.19.1 # via moto
cfn-lint==0.20.1 # via moto
chardet==3.0.4 # via requests
click==7.0 # via flask
coverage==4.5.3
@ -23,8 +23,8 @@ docker-pycreds==0.4.0 # via docker
docker==3.7.2 # via moto
docutils==0.14 # via botocore
ecdsa==0.13.2 # via python-jose
factory-boy==2.11.1
faker==1.0.5
factory-boy==2.12.0
faker==1.0.7
flask==1.0.2 # via pytest-flask
freezegun==0.3.11
future==0.17.1 # via aws-xray-sdk, python-jose
@ -38,18 +38,18 @@ jsonpickle==1.1 # via aws-xray-sdk
jsonpointer==2.0 # via jsonpatch
jsonschema==2.6.0 # via aws-sam-translator, cfn-lint
markupsafe==1.1.1 # via jinja2
mock==3.0.4 # via moto
mock==3.0.5 # via moto
more-itertools==7.0.0 # via pytest
moto==1.3.8
nose==1.3.7
pluggy==0.9.0 # via pytest
pluggy==0.11.0 # via pytest
py==1.8.0 # via pytest
pyasn1==0.4.5 # via rsa
pycparser==2.19 # via cffi
pyflakes==2.1.1
pytest-flask==0.14.0
pytest-flask==0.15.0
pytest-mock==1.10.4
pytest==4.4.1
pytest==4.5.0
python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
python-jose==3.0.1 # via moto
pytz==2019.1 # via moto
@ -62,6 +62,7 @@ s3transfer==0.2.0 # via boto3
six==1.12.0 # via aws-sam-translator, cfn-lint, cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker
urllib3==1.24.3 # via botocore, requests
wcwidth==0.1.7 # via pytest
websocket-client==0.56.0 # via docker
werkzeug==0.15.2 # via flask, moto, pytest-flask
wrapt==1.11.1 # via aws-xray-sdk

12
requirements.txt
View File

@ -2,9 +2,9 @@
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file requirements.txt requirements.in -U --no-index
# pip-compile --no-index --output-file=requirements.txt requirements.in
#
acme==0.34.1
acme==0.34.2
alembic-autogenerate-enums==0.0.2
alembic==1.0.10 # via flask-migrate
amqp==2.4.2 # via kombu
@ -15,8 +15,8 @@ asyncpool==1.0
bcrypt==3.1.6 # via flask-bcrypt, paramiko
billiard==3.6.0.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.9.143
botocore==1.12.143
boto3==1.9.147
botocore==1.12.147
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
@ -51,11 +51,11 @@ josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare
kombu==4.5.0
lockfile==0.12.2
mako==1.0.9 # via alembic
mako==1.0.10 # via alembic
markupsafe==1.1.1 # via jinja2, mako
marshmallow-sqlalchemy==0.16.3
marshmallow==2.19.2
mock==3.0.4 # via acme
mock==3.0.5 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pem==19.1.0