Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Celery integration

This commit is contained in:
Curtis Castrapel 2018-09-13 10:35:54 -07:00
parent 3c521f66a5
commit 23382b2777
8 changed files with 181 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
lemur/certificates/service.py
View File

@ -5,32 +5,27 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import arrow
from flask import current_app
from cryptography import x509
from sqlalchemy import func, or_, not_, cast, Integer
from cryptography import x509
import arrow
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from flask import current_app
from lemur import database
from lemur.extensions import metrics, sentry, signals
from lemur.plugins.base import plugins
from lemur.common.utils import generate_private_key, truthiness
from lemur.roles.models import Role
from lemur.domains.models import Domain
from lemur.authorities.models import Authority
from lemur.destinations.models import Destination
from lemur.certificates.models import Certificate
from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
from lemur.common.utils import generate_private_key, truthiness
from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.extensions import metrics, sentry, signals
from lemur.notifications.models import Notification
from lemur.pending_certificates.models import PendingCertificate
from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
from lemur.plugins.base import plugins
from lemur.roles import service as role_service
from lemur.roles.models import Role
csr_created = signals.signal('csr_created', "CSR generated")
csr_imported = signals.signal('csr_imported', "CSR imported from external source")
@ -95,8 +90,8 @@ def get_all_pending_cleaning(source):
:param source:
:return:
"""
return Certificate.query.filter(Certificate.sources.any(id=source.id))\
.filter(not_(Certificate.endpoints.any())).all()
return Certificate.query.filter(Certificate.sources.any(id=source.id)) \
.filter(not_(Certificate.endpoints.any())).all()
def get_all_pending_reissue():
@ -109,8 +104,8 @@ def get_all_pending_reissue():
:return:
"""
return Certificate.query.filter(Certificate.rotation == True)\
.filter(not_(Certificate.replaced.any()))\
return Certificate.query.filter(Certificate.rotation == True) \
.filter(not_(Certificate.replaced.any())) \
.filter(Certificate.in_rotation_window == True).all() # noqa
@ -280,6 +275,11 @@ def create(**kwargs):
if isinstance(cert, Certificate):
certificate_issued.send(certificate=cert, authority=cert.authority)
metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
if isinstance(cert, PendingCertificate) and cert.authority.plugin_name == 'acme-issuer':
# Call Celery to create acme-issuer (LetsEncrypt) certificates
from lemur.common.celery import fetch_acme_cert
fetch_acme_cert.delay(cert.id)
return cert
@ -310,8 +310,8 @@ def render(args):
if 'issuer' in terms:
# we can't rely on issuer being correct in the cert directly so we combine queries
sub_query = database.session_query(Authority.id)\
.filter(Authority.name.ilike(term))\
sub_query = database.session_query(Authority.id) \
.filter(Authority.name.ilike(term)) \
.subquery()
query = query.filter(
@ -450,8 +450,8 @@ def stats(**kwargs):
if kwargs.get('metric') == 'not_after':
start = arrow.utcnow()
end = start.replace(weeks=+32)
items = database.db.session.query(Certificate.issuer, func.count(Certificate.id))\
.group_by(Certificate.issuer)\
items = database.db.session.query(Certificate.issuer, func.count(Certificate.id)) \
.group_by(Certificate.issuer) \
.filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \
.filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all()

119
lemur/common/celery.py Normal file
View File

@ -0,0 +1,119 @@
"""
This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
when invoked.
When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
"""
import copy
import sys
from celery import Celery
from flask import current_app
from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service
flask_app = create_app()
def make_celery(app):
celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
celery = make_celery(flask_app)
@celery.task()
def fetch_acme_cert(id):
"""
Attempt to get the full certificate for the pending certificate listed.
Args:
id: an id of a PendingCertificate
"""
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
}
pending_certs = pending_certificate_service.get_pending_certs([id])
user = user_service.get_by_username('lemur')
new = 0
failed = 0
wrong_issuer = 0
acme_certs = []
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
acme_certs.append(cert)
else:
wrong_issuer += 1
authority = plugins.get("acme-issuer")
resolved_certs = authority.get_ordered_certificates(acme_certs)
for cert in resolved_certs:
real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
pending_certificate_service.create_certificate(pending_cert, real_cert, user)
pending_certificate_service.delete_by_id(pending_cert.id)
# add metrics to metrics extension
new += 1
else:
failed += 1
error_log = copy.deepcopy(log_data)
error_log["message"] = "Pending certificate creation failure"
error_log["pending_cert_id"] = pending_cert.id
error_log["last_error"] = cert.get("last_error")
error_log["cn"] = pending_cert.cn
if pending_cert.number_attempts > 4:
error_log["message"] = "Deleting pending certificate"
send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
else:
pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update(
cert.get("pending_cert").id,
status=str(cert.get("last_error"))
)
# Add failed pending cert task back to queue
fetch_acme_cert.delay(id)
current_app.logger.error(error_log)
log_data["message"] = "Complete"
log_data["new"] = new
log_data["failed"] = failed
log_data["wrong_issuer"] = wrong_issuer
current_app.logger.debug(log_data)
print(
"[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
new=new,
failed=failed,
wrong_issuer=wrong_issuer
)
)

10
requirements-dev.txt
View File

@ -5,26 +5,26 @@
# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
#
aspy.yaml==1.1.1 # via pre-commit
cached-property==1.4.3 # via pre-commit
cached-property==1.5.1 # via pre-commit
certifi==2018.8.24 # via requests
cfgv==1.1.0 # via pre-commit
chardet==3.0.4 # via requests
flake8==3.5.0
identify==1.1.4 # via pre-commit
identify==1.1.5 # via pre-commit
idna==2.7 # via requests
invoke==1.1.1
mccabe==0.6.1 # via flake8
nodeenv==1.3.2
pkginfo==1.4.2 # via twine
pre-commit==1.10.5
pre-commit==1.11.0
pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8
pyyaml==3.13 # via aspy.yaml, pre-commit
requests-toolbelt==0.8.0 # via twine
requests==2.19.1 # via requests-toolbelt, twine
six==1.11.0 # via cfgv, pre-commit
toml==0.9.4 # via pre-commit
tqdm==4.25.0 # via twine
toml==0.9.6 # via pre-commit
tqdm==4.26.0 # via twine
twine==1.11.0
urllib3==1.23 # via requests
virtualenv==16.0.0 # via pre-commit

18
requirements-docs.txt
View File

@ -4,19 +4,22 @@
#
# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
#
acme==0.26.1
acme==0.27.1
alabaster==0.7.11 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.0
amqp==2.3.2
aniso8601==3.0.2
arrow==0.12.1
asn1crypto==0.24.0
asyncpool==1.0
babel==2.6.0 # via sphinx
bcrypt==3.1.4
billiard==3.5.0.4
blinker==1.4
boto3==1.8.1
botocore==1.11.1
boto3==1.7.79
botocore==1.10.84
celery[redis]==4.2.1
certifi==2018.8.24
cffi==1.11.5
chardet==3.0.4
@ -39,13 +42,14 @@ flask==0.12
future==0.16.0
gunicorn==19.9.0
idna==2.7
imagesize==1.0.0 # via sphinx
imagesize==1.1.0 # via sphinx
inflection==0.3.1
itsdangerous==0.24
jinja2==2.10
jmespath==0.9.3
josepy==1.1.0
jsonlines==1.2.0
kombu==4.2.1
lockfile==0.12.2
mako==1.0.7
markupsafe==1.0
@ -72,6 +76,7 @@ python-editor==1.0.3
pytz==2018.5
pyyaml==3.13
raven[flask]==6.9.0
redis==2.10.6
requests-toolbelt==0.8.0
requests[security]==2.19.1
retrying==1.3.3
@ -79,12 +84,13 @@ s3transfer==0.1.13
six==1.11.0
snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.4.1
sphinx==1.7.7
sphinx==1.8.0
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx
sqlalchemy-utils==0.33.3
sqlalchemy==1.2.10
sqlalchemy==1.2.11
tabulate==0.8.2
urllib3==1.23
vine==1.1.4
werkzeug==0.14.1
xmltodict==0.11.0

2
requirements-tests.in
View File

@ -4,7 +4,7 @@ coverage
factory-boy
Faker
freezegun
moto
moto==1.3.4 # Issue with moto: https://github.com/spulec/moto/issues/1813
nose
pyflakes
pytest

16
requirements-tests.txt
View File

@ -5,12 +5,12 @@
# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in
#
asn1crypto==0.24.0 # via cryptography
atomicwrites==1.2.0 # via pytest
attrs==18.1.0 # via pytest
atomicwrites==1.2.1 # via pytest
attrs==18.2.0 # via pytest
aws-xray-sdk==0.95 # via moto
boto3==1.8.1 # via moto
boto3==1.9.3 # via moto
boto==2.49.0 # via moto
botocore==1.11.1 # via boto3, moto, s3transfer
botocore==1.12.3 # via boto3, moto, s3transfer
certifi==2018.8.24 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
@ -40,14 +40,14 @@ moto==1.3.4
nose==1.3.7
pbr==4.2.0 # via mock
pluggy==0.7.1 # via pytest
py==1.5.4 # via pytest
py==1.6.0 # via pytest
pyaml==17.12.1 # via moto
pycparser==2.18 # via cffi
pycryptodome==3.6.6 # via python-jose
pyflakes==2.0.0
pytest-flask==0.10.0
pytest-flask==0.12.0
pytest-mock==1.10.0
pytest==3.7.3
pytest==3.8.0
python-dateutil==2.7.3 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.5 # via moto
@ -59,7 +59,7 @@ s3transfer==0.1.13 # via boto3
six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker
urllib3==1.23 # via botocore, requests
websocket-client==0.51.0 # via docker
websocket-client==0.53.0 # via docker
werkzeug==0.14.1 # via flask, moto, pytest-flask
wrapt==1.10.11 # via aws-xray-sdk
xmltodict==0.11.0 # via moto

4
requirements.in
View File

@ -4,7 +4,9 @@ acme
alembic-autogenerate-enums
arrow
asyncpool
boto3==1.7.79
boto3==1.7.79 # Issue with moto: https://github.com/spulec/moto/issues/1813
botocore== 1.10.84 # Issue with moto: https://github.com/spulec/moto/issues/1813
celery[redis]
certifi
CloudFlare
cryptography

14
requirements.txt
View File

@ -4,17 +4,20 @@
#
# pip-compile --no-index --output-file requirements.txt requirements.in
#
acme==0.26.1
acme==0.27.1
alembic-autogenerate-enums==0.0.2
alembic==1.0.0 # via flask-migrate
amqp==2.3.2 # via kombu
aniso8601==3.0.2 # via flask-restful
arrow==0.12.1
asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
bcrypt==3.1.4 # via flask-bcrypt, paramiko
billiard==3.5.0.4 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.7.79
botocore==1.10.84 # via boto3, s3transfer
botocore==1.10.84
celery[redis]==4.2.1
certifi==2018.8.24
cffi==1.11.5 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
@ -43,6 +46,7 @@ jinja2==2.10
jmespath==0.9.3 # via boto3, botocore
josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare
kombu==4.2.1 # via celery
lockfile==0.12.2
mako==1.0.7 # via alembic
markupsafe==1.0 # via jinja2, mako
@ -64,17 +68,19 @@ pyrfc3339==1.1 # via acme
python-dateutil==2.7.3 # via alembic, arrow, botocore
python-editor==1.0.3 # via alembic
python-ldap==3.1.0
pytz==2018.5 # via acme, flask-restful, pyrfc3339
pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339
pyyaml==3.13 # via cloudflare
raven[flask]==6.9.0
redis==2.10.6 # via celery
requests-toolbelt==0.8.0 # via acme
requests[security]==2.19.1
retrying==1.3.3
s3transfer==0.1.13 # via boto3
six==1.11.0
sqlalchemy-utils==0.33.3
sqlalchemy-utils==0.33.4
sqlalchemy==1.2.11 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2
urllib3==1.23 # via requests
vine==1.1.4 # via amqp
werkzeug==0.14.1 # via flask
xmltodict==0.11.0