Celery refactoring, celery beat job in configuration

This commit is contained in:
Curtis Castrapel 2018-09-17 10:52:12 -07:00
parent 23382b2777
commit 563f0fb9b2
8 changed files with 70 additions and 32 deletions

View File

@ -5,13 +5,12 @@
:license: Apache, see LICENSE for more details. :license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com> .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
""" """
from cryptography import x509
from sqlalchemy import func, or_, not_, cast, Integer
import arrow import arrow
from cryptography import x509
from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives import hashes, serialization
from flask import current_app from flask import current_app
from sqlalchemy import func, or_, not_, cast, Integer
from lemur import database from lemur import database
from lemur.authorities.models import Authority from lemur.authorities.models import Authority
@ -276,10 +275,6 @@ def create(**kwargs):
certificate_issued.send(certificate=cert, authority=cert.authority) certificate_issued.send(certificate=cert, authority=cert.authority)
metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer)) metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
if isinstance(cert, PendingCertificate) and cert.authority.plugin_name == 'acme-issuer':
# Call Celery to create acme-issuer (LetsEncrypt) certificates
from lemur.common.celery import fetch_acme_cert
fetch_acme_cert.delay(cert.id)
return cert return cert

View File

@ -8,7 +8,9 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
""" """
import copy import copy
import datetime
import sys import sys
from datetime import timezone
from celery import Celery from celery import Celery
from flask import current_app from flask import current_app
@ -117,3 +119,17 @@ def fetch_acme_cert(id):
wrong_issuer=wrong_issuer wrong_issuer=wrong_issuer
) )
) )
@celery.task()
def fetch_all_pending_acme_certs():
"""Instantiate celery workers to resolve all pending Acme certificates"""
pending_certs = pending_certificate_service.get_pending_certs('all')
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
if cert.last_updated == cert.date_created or datetime.datetime.now(
timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3):
fetch_acme_cert.delay(cert.id)

View File

@ -0,0 +1,24 @@
"""Add last_updated field to Pending Certs
Revision ID: 9392b9f9a805
Revises: 5ae0ecefb01f
Create Date: 2018-09-17 08:33:37.087488
"""
# revision identifiers, used by Alembic.
revision = '9392b9f9a805'
down_revision = '5ae0ecefb01f'
from alembic import op
from sqlalchemy_utils import ArrowType
import sqlalchemy as sa
def upgrade():
op.add_column('pending_certs', sa.Column('last_updated', ArrowType, server_default=sa.text('now()'), onupdate=sa.text('now()'),
nullable=False))
def downgrade():
op.drop_column('pending_certs', 'last_updated')

View File

@ -5,19 +5,18 @@
""" """
from datetime import datetime as dt from datetime import datetime as dt
from sqlalchemy.orm import relationship
from sqlalchemy import Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean from sqlalchemy import Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
from sqlalchemy_utils.types.arrow import ArrowType from sqlalchemy.orm import relationship
from sqlalchemy_utils import JSONType from sqlalchemy_utils import JSONType
from sqlalchemy_utils.types.arrow import ArrowType
from lemur.certificates.models import get_or_increase_name from lemur.certificates.models import get_or_increase_name
from lemur.common import defaults, utils from lemur.common import defaults, utils
from lemur.database import db from lemur.database import db
from lemur.utils import Vault
from lemur.models import pending_cert_source_associations, \ from lemur.models import pending_cert_source_associations, \
pending_cert_destination_associations, pending_cert_notification_associations, \ pending_cert_destination_associations, pending_cert_notification_associations, \
pending_cert_replacement_associations, pending_cert_role_associations pending_cert_replacement_associations, pending_cert_role_associations
from lemur.utils import Vault
class PendingCertificate(db.Model): class PendingCertificate(db.Model):
@ -40,6 +39,7 @@ class PendingCertificate(db.Model):
dns_provider_id = Column(Integer, ForeignKey('dns_providers.id', ondelete="CASCADE")) dns_provider_id = Column(Integer, ForeignKey('dns_providers.id', ondelete="CASCADE"))
status = Column(Text(), nullable=True) status = Column(Text(), nullable=True)
last_updated = Column(ArrowType, PassiveDefault(func.now()), onupdate=func.now(), nullable=False)
rotation = Column(Boolean, default=False) rotation = Column(Boolean, default=False)
user_id = Column(Integer, ForeignKey('users.id')) user_id = Column(Integer, ForeignKey('users.id'))
@ -47,9 +47,12 @@ class PendingCertificate(db.Model):
root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE")) root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id')) rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
notifications = relationship('Notification', secondary=pending_cert_notification_associations, backref='pending_cert', passive_deletes=True) notifications = relationship('Notification', secondary=pending_cert_notification_associations,
destinations = relationship('Destination', secondary=pending_cert_destination_associations, backref='pending_cert', passive_deletes=True) backref='pending_cert', passive_deletes=True)
sources = relationship('Source', secondary=pending_cert_source_associations, backref='pending_cert', passive_deletes=True) destinations = relationship('Destination', secondary=pending_cert_destination_associations, backref='pending_cert',
passive_deletes=True)
sources = relationship('Source', secondary=pending_cert_source_associations, backref='pending_cert',
passive_deletes=True)
roles = relationship('Role', secondary=pending_cert_role_associations, backref='pending_cert', passive_deletes=True) roles = relationship('Role', secondary=pending_cert_role_associations, backref='pending_cert', passive_deletes=True)
replaces = relationship('Certificate', replaces = relationship('Certificate',
secondary=pending_cert_replacement_associations, secondary=pending_cert_replacement_associations,
@ -77,7 +80,7 @@ class PendingCertificate(db.Model):
# TODO: Fix auto-generated name, it should be renamed on creation # TODO: Fix auto-generated name, it should be renamed on creation
self.name = get_or_increase_name( self.name = get_or_increase_name(
defaults.certificate_name(kwargs['common_name'], kwargs['authority'].name, defaults.certificate_name(kwargs['common_name'], kwargs['authority'].name,
dt.now(), dt.now(), False), self.external_id) dt.now(), dt.now(), False), self.external_id)
self.rename = True self.rename = True
self.cn = defaults.common_name(utils.parse_csr(self.csr)) self.cn = defaults.common_name(utils.parse_csr(self.csr))

View File

@ -1,5 +1,14 @@
from marshmallow import fields, post_load from marshmallow import fields, post_load
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.certificates.schemas import CertificateNestedOutputSchema
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.destinations.schemas import DestinationNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
from lemur.notifications import service as notification_service
from lemur.notifications.schemas import NotificationNestedOutputSchema
from lemur.policies.schemas import RotationPolicyNestedOutputSchema
from lemur.roles.schemas import RoleNestedOutputSchema
from lemur.schemas import ( from lemur.schemas import (
AssociatedCertificateSchema, AssociatedCertificateSchema,
AssociatedDestinationSchema, AssociatedDestinationSchema,
@ -8,18 +17,7 @@ from lemur.schemas import (
EndpointNestedOutputSchema, EndpointNestedOutputSchema,
ExtensionSchema ExtensionSchema
) )
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.users.schemas import UserNestedOutputSchema from lemur.users.schemas import UserNestedOutputSchema
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.certificates.schemas import CertificateNestedOutputSchema
from lemur.destinations.schemas import DestinationNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
from lemur.notifications.schemas import NotificationNestedOutputSchema
from lemur.roles.schemas import RoleNestedOutputSchema
from lemur.policies.schemas import RotationPolicyNestedOutputSchema
from lemur.notifications import service as notification_service
class PendingCertificateSchema(LemurInputSchema): class PendingCertificateSchema(LemurInputSchema):
@ -38,6 +36,7 @@ class PendingCertificateOutputSchema(LemurOutputSchema):
name = fields.String() name = fields.String()
number_attempts = fields.Integer() number_attempts = fields.Integer()
date_created = fields.Date() date_created = fields.Date()
last_updated = fields.Date()
rotation = fields.Boolean() rotation = fields.Boolean()
@ -88,7 +87,8 @@ class PendingCertificateEditInputSchema(PendingCertificateSchema):
""" """
if data['owner']: if data['owner']:
notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper()) notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']]) data['notifications'] += notification_service.create_default_expiration_notifications(notification_name,
[data['owner']])
return data return data

View File

@ -12,7 +12,7 @@ chardet==3.0.4 # via requests
flake8==3.5.0 flake8==3.5.0
identify==1.1.5 # via pre-commit identify==1.1.5 # via pre-commit
idna==2.7 # via requests idna==2.7 # via requests
invoke==1.1.1 invoke==1.2.0
mccabe==0.6.1 # via flake8 mccabe==0.6.1 # via flake8
nodeenv==1.3.2 nodeenv==1.3.2
pkginfo==1.4.2 # via twine pkginfo==1.4.2 # via twine

View File

@ -87,7 +87,7 @@ sphinx-rtd-theme==0.4.1
sphinx==1.8.0 sphinx==1.8.0
sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx sphinxcontrib-websupport==1.1.0 # via sphinx
sqlalchemy-utils==0.33.3 sqlalchemy-utils==0.33.4
sqlalchemy==1.2.11 sqlalchemy==1.2.11
tabulate==0.8.2 tabulate==0.8.2
urllib3==1.23 urllib3==1.23

View File

@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.2.1 # via pytest atomicwrites==1.2.1 # via pytest
attrs==18.2.0 # via pytest attrs==18.2.0 # via pytest
aws-xray-sdk==0.95 # via moto aws-xray-sdk==0.95 # via moto
boto3==1.9.3 # via moto boto3==1.9.4 # via moto
boto==2.49.0 # via moto boto==2.49.0 # via moto
botocore==1.12.3 # via boto3, moto, s3transfer botocore==1.12.4 # via boto3, moto, s3transfer
certifi==2018.8.24 # via requests certifi==2018.8.24 # via requests
cffi==1.11.5 # via cryptography cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
@ -23,7 +23,7 @@ docker==3.5.0 # via moto
docutils==0.14 # via botocore docutils==0.14 # via botocore
ecdsa==0.13 # via python-jose ecdsa==0.13 # via python-jose
factory-boy==2.11.1 factory-boy==2.11.1
faker==0.9.0 faker==0.9.1
flask==1.0.2 # via pytest-flask flask==1.0.2 # via pytest-flask
freezegun==0.3.10 freezegun==0.3.10
future==0.16.0 # via python-jose future==0.16.0 # via python-jose