Lemur LetsEncrypt Polling Support

This commit is contained in:
Curtis Castrapel 2018-10-11 22:01:05 -07:00
parent e91d8ec81b
commit cc18a68c00
7 changed files with 101 additions and 35 deletions

View File

@ -193,6 +193,8 @@ class CertificateOutputSchema(LemurOutputSchema):
name = fields.String() name = fields.String()
dns_provider_id = fields.Integer(required=False, allow_none=True) dns_provider_id = fields.Integer(required=False, allow_none=True)
date_created = ArrowDateTime() date_created = ArrowDateTime()
resolved = fields.Boolean(required=False, allow_none=True)
resolved_cert_id = fields.Integer(required=False, allow_none=True)
rotation = fields.Boolean() rotation = fields.Boolean()

View File

@ -20,7 +20,6 @@ from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins from lemur.plugins.base import plugins
from lemur.users import service as user_service
flask_app = create_app() flask_app = create_app()
@ -57,7 +56,6 @@ def fetch_acme_cert(id):
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
} }
pending_certs = pending_certificate_service.get_pending_certs([id]) pending_certs = pending_certificate_service.get_pending_certs([id])
user = user_service.get_by_username('lemur')
new = 0 new = 0
failed = 0 failed = 0
wrong_issuer = 0 wrong_issuer = 0
@ -78,12 +76,22 @@ def fetch_acme_cert(id):
real_cert = cert.get("cert") real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if not pending_cert:
log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
current_app.logger.error(log_data)
continue
if real_cert: if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete # If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate # the pending certificate as resolved
pending_certificate_service.create_certificate(pending_cert, real_cert, user) final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
pending_certificate_service.delete_by_id(pending_cert.id) pending_certificate_service.update(
cert.get("pending_cert").id,
resolved=True
)
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved_cert_id=final_cert.id
)
# add metrics to metrics extension # add metrics to metrics extension
new += 1 new += 1
else: else:
@ -97,7 +105,11 @@ def fetch_acme_cert(id):
if pending_cert.number_attempts > 4: if pending_cert.number_attempts > 4:
error_log["message"] = "Deleting pending certificate" error_log["message"] = "Deleting pending certificate"
send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) # Mark the pending cert as True
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved=True
)
else: else:
pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update( pending_certificate_service.update(
@ -124,7 +136,7 @@ def fetch_acme_cert(id):
@celery.task() @celery.task()
def fetch_all_pending_acme_certs(): def fetch_all_pending_acme_certs():
"""Instantiate celery workers to resolve all pending Acme certificates""" """Instantiate celery workers to resolve all pending Acme certificates"""
pending_certs = pending_certificate_service.get_pending_certs('all') pending_certs = pending_certificate_service.get_unresolved_pending_certs()
# We only care about certs using the acme-issuer plugin # We only care about certs using the acme-issuer plugin
for cert in pending_certs: for cert in pending_certs:

View File

@ -0,0 +1,24 @@
"""Add status to pending certificate, and store resolved cert id
Revision ID: 984178255c83
Revises: f2383bf08fbc
Create Date: 2018-10-11 20:49:12.704563
"""
# revision identifiers, used by Alembic.
revision = '984178255c83'
down_revision = 'f2383bf08fbc'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('pending_certs', sa.Column('resolved', sa.Boolean(), nullable=True))
op.add_column('pending_certs', sa.Column('resolved_cert_id', sa.Integer(), nullable=True))
def downgrade():
op.drop_column('pending_certs', 'resolved_cert_id')
op.drop_column('pending_certs', 'resolved')

View File

@ -15,7 +15,6 @@ from lemur.authorities.service import get as get_authority
from lemur.notifications.messaging import send_pending_failure_notification from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins from lemur.plugins.base import plugins
from lemur.users import service as user_service
manager = Manager(usage="Handles pending certificate related tasks.") manager = Manager(usage="Handles pending certificate related tasks.")
@ -30,7 +29,7 @@ def fetch(ids):
`python manager.py pending_certs fetch -i 123 321 all` `python manager.py pending_certs fetch -i 123 321 all`
""" """
pending_certs = pending_certificate_service.get_pending_certs(ids) pending_certs = pending_certificate_service.get_pending_certs(ids)
user = user_service.get_by_username('lemur')
new = 0 new = 0
failed = 0 failed = 0
@ -38,10 +37,17 @@ def fetch(ids):
authority = plugins.get(cert.authority.plugin_name) authority = plugins.get(cert.authority.plugin_name)
real_cert = authority.get_ordered_certificate(cert) real_cert = authority.get_ordered_certificate(cert)
if real_cert: if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete # If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate # the pending certificate as resolved
pending_certificate_service.create_certificate(cert, real_cert, user) final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user)
pending_certificate_service.delete(cert) pending_certificate_service.update(
cert.id,
resolved=True
)
pending_certificate_service.update(
cert.id,
resolved_cert_id=final_cert.id
)
# add metrics to metrics extension # add metrics to metrics extension
new += 1 new += 1
else: else:
@ -66,8 +72,7 @@ def fetch_all_acme():
log_data = { log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
} }
pending_certs = pending_certificate_service.get_pending_certs('all') pending_certs = pending_certificate_service.get_unresolved_pending_certs()
user = user_service.get_by_username('lemur')
new = 0 new = 0
failed = 0 failed = 0
wrong_issuer = 0 wrong_issuer = 0
@ -90,10 +95,17 @@ def fetch_all_acme():
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if real_cert: if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete # If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate # the pending certificate as resolved
pending_certificate_service.create_certificate(pending_cert, real_cert, user) final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
pending_certificate_service.delete_by_id(pending_cert.id) pending_certificate_service.update(
pending_cert.id,
resolved=True
)
pending_certificate_service.update(
pending_cert.id,
resolved_cert_id=final_cert.id
)
# add metrics to metrics extension # add metrics to metrics extension
new += 1 new += 1
else: else:
@ -105,9 +117,13 @@ def fetch_all_acme():
error_log["cn"] = pending_cert.cn error_log["cn"] = pending_cert.cn
if pending_cert.number_attempts > 4: if pending_cert.number_attempts > 4:
error_log["message"] = "Deleting pending certificate" error_log["message"] = "Marking pending certificate"
send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) # Mark "resolved" as True
pending_certificate_service.update(
cert.id,
resolved=True
)
else: else:
pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update( pending_certificate_service.update(

View File

@ -29,6 +29,8 @@ class PendingCertificate(db.Model):
notify = Column(Boolean, default=True) notify = Column(Boolean, default=True)
number_attempts = Column(Integer) number_attempts = Column(Integer)
rename = Column(Boolean, default=True) rename = Column(Boolean, default=True)
resolved = Column(Boolean, default=False)
resolved_cert_id = Column(Integer, nullable=True)
cn = Column(String(128)) cn = Column(String(128))
csr = Column(Text(), nullable=False) csr = Column(Text(), nullable=False)

View File

@ -37,6 +37,8 @@ class PendingCertificateOutputSchema(LemurOutputSchema):
number_attempts = fields.Integer() number_attempts = fields.Integer()
date_created = fields.Date() date_created = fields.Date()
last_updated = fields.Date() last_updated = fields.Date()
resolved = fields.Boolean(required=False)
resolved_cert_id = fields.Integer(required=False)
rotation = fields.Boolean() rotation = fields.Boolean()

View File

@ -4,25 +4,21 @@
.. moduleauthor:: James Chuong <jchuong@instartlogic.com> .. moduleauthor:: James Chuong <jchuong@instartlogic.com>
""" """
import arrow import arrow
from sqlalchemy import or_, cast, Integer from sqlalchemy import or_, cast, Integer
from lemur import database from lemur import database
from lemur.common.utils import truthiness
from lemur.plugins.base import plugins
from lemur.roles.models import Role
from lemur.domains.models import Domain
from lemur.authorities.models import Authority from lemur.authorities.models import Authority
from lemur.certificates import service as certificate_service
from lemur.certificates.schemas import CertificateUploadInputSchema
from lemur.common.utils import truthiness
from lemur.destinations.models import Destination from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.notifications.models import Notification from lemur.notifications.models import Notification
from lemur.pending_certificates.models import PendingCertificate from lemur.pending_certificates.models import PendingCertificate
from lemur.plugins.base import plugins
from lemur.certificates import service as certificate_service from lemur.roles.models import Role
from lemur.users import service as user_service from lemur.users import service as user_service
from lemur.certificates.schemas import CertificateUploadInputSchema
def get(pending_cert_id): def get(pending_cert_id):
""" """
@ -63,6 +59,15 @@ def delete_by_id(id):
database.delete(get(id)) database.delete(get(id))
def get_unresolved_pending_certs():
"""
Retrieve a list of unresolved pending certs given a list of ids
Filters out non-existing pending certs
"""
query = database.session_query(PendingCertificate).filter(PendingCertificate.resolved.is_(False))
return database.find_all(query, PendingCertificate, {}).all()
def get_pending_certs(pending_ids): def get_pending_certs(pending_ids):
""" """
Retrieve a list of pending certs given a list of ids Retrieve a list of pending certs given a list of ids
@ -116,6 +121,7 @@ def create_certificate(pending_certificate, certificate, user):
# If generating name from certificate, remove the one from pending certificate # If generating name from certificate, remove the one from pending certificate
del data['name'] del data['name']
data['creator'] = creator data['creator'] = creator
cert = certificate_service.import_certificate(**data) cert = certificate_service.import_certificate(**data)
database.update(cert) database.update(cert)
return cert return cert
@ -221,4 +227,6 @@ def render(args):
now = arrow.now().format('YYYY-MM-DD') now = arrow.now().format('YYYY-MM-DD')
query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now) query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now)
# Only show unresolved certificates in the UI
query = query.filter(PendingCertificate.resolved.is_(False))
return database.sort_and_page(query, PendingCertificate, args) return database.sort_and_page(query, PendingCertificate, args)