From cc18a68c005e3c6c33c6634dccea5fb7800a8856 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 11 Oct 2018 22:01:05 -0700 Subject: [PATCH 1/6] Lemur LetsEncrypt Polling Support --- lemur/certificates/schemas.py | 2 + lemur/common/celery.py | 30 ++++++++++----- lemur/migrations/versions/984178255c83_.py | 24 ++++++++++++ lemur/pending_certificates/cli.py | 44 +++++++++++++++------- lemur/pending_certificates/models.py | 2 + lemur/pending_certificates/schemas.py | 2 + lemur/pending_certificates/service.py | 32 ++++++++++------ 7 files changed, 101 insertions(+), 35 deletions(-) create mode 100644 lemur/migrations/versions/984178255c83_.py diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 032a9175..a92f1f66 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -193,6 +193,8 @@ class CertificateOutputSchema(LemurOutputSchema): name = fields.String() dns_provider_id = fields.Integer(required=False, allow_none=True) date_created = ArrowDateTime() + resolved = fields.Boolean(required=False, allow_none=True) + resolved_cert_id = fields.Integer(required=False, allow_none=True) rotation = fields.Boolean() diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 1858b4b5..3f425950 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -20,7 +20,6 @@ from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.users import service as user_service flask_app = create_app() @@ -57,7 +56,6 @@ def fetch_acme_cert(id): "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } pending_certs = pending_certificate_service.get_pending_certs([id]) - user = user_service.get_by_username('lemur') new = 0 failed = 0 wrong_issuer = 0 @@ -78,12 +76,22 @@ def fetch_acme_cert(id): real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) - + if not pending_cert: + log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?" + current_app.logger.error(log_data) + continue if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(pending_cert, real_cert, user) - pending_certificate_service.delete_by_id(pending_cert.id) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user) + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved=True + ) + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -97,7 +105,11 @@ def fetch_acme_cert(id): if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) + # Mark the pending cert as True + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved=True + ) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( @@ -124,7 +136,7 @@ def fetch_acme_cert(id): @celery.task() def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" - pending_certs = pending_certificate_service.get_pending_certs('all') + pending_certs = pending_certificate_service.get_unresolved_pending_certs() # We only care about certs using the acme-issuer plugin for cert in pending_certs: diff --git a/lemur/migrations/versions/984178255c83_.py b/lemur/migrations/versions/984178255c83_.py new file mode 100644 index 00000000..40d2ce31 --- /dev/null +++ b/lemur/migrations/versions/984178255c83_.py @@ -0,0 +1,24 @@ +"""Add status to pending certificate, and store resolved cert id + +Revision ID: 984178255c83 +Revises: f2383bf08fbc +Create Date: 2018-10-11 20:49:12.704563 + +""" + +# revision identifiers, used by Alembic. +revision = '984178255c83' +down_revision = 'f2383bf08fbc' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('pending_certs', sa.Column('resolved', sa.Boolean(), nullable=True)) + op.add_column('pending_certs', sa.Column('resolved_cert_id', sa.Integer(), nullable=True)) + + +def downgrade(): + op.drop_column('pending_certs', 'resolved_cert_id') + op.drop_column('pending_certs', 'resolved') diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index cbce700d..c4e62549 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -15,7 +15,6 @@ from lemur.authorities.service import get as get_authority from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.users import service as user_service manager = Manager(usage="Handles pending certificate related tasks.") @@ -30,7 +29,7 @@ def fetch(ids): `python manager.py pending_certs fetch -i 123 321 all` """ pending_certs = pending_certificate_service.get_pending_certs(ids) - user = user_service.get_by_username('lemur') + new = 0 failed = 0 @@ -38,10 +37,17 @@ def fetch(ids): authority = plugins.get(cert.authority.plugin_name) real_cert = authority.get_ordered_certificate(cert) if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(cert, real_cert, user) - pending_certificate_service.delete(cert) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user) + pending_certificate_service.update( + cert.id, + resolved=True + ) + pending_certificate_service.update( + cert.id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -66,8 +72,7 @@ def fetch_all_acme(): log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } - pending_certs = pending_certificate_service.get_pending_certs('all') - user = user_service.get_by_username('lemur') + pending_certs = pending_certificate_service.get_unresolved_pending_certs() new = 0 failed = 0 wrong_issuer = 0 @@ -90,10 +95,17 @@ def fetch_all_acme(): pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(pending_cert, real_cert, user) - pending_certificate_service.delete_by_id(pending_cert.id) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user) + pending_certificate_service.update( + pending_cert.id, + resolved=True + ) + pending_certificate_service.update( + pending_cert.id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -105,9 +117,13 @@ def fetch_all_acme(): error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > 4: - error_log["message"] = "Deleting pending certificate" + error_log["message"] = "Marking pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) + # Mark "resolved" as True + pending_certificate_service.update( + cert.id, + resolved=True + ) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index a1f61fa1..1261177d 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -29,6 +29,8 @@ class PendingCertificate(db.Model): notify = Column(Boolean, default=True) number_attempts = Column(Integer) rename = Column(Boolean, default=True) + resolved = Column(Boolean, default=False) + resolved_cert_id = Column(Integer, nullable=True) cn = Column(String(128)) csr = Column(Text(), nullable=False) diff --git a/lemur/pending_certificates/schemas.py b/lemur/pending_certificates/schemas.py index 252cd892..fbc94f4e 100644 --- a/lemur/pending_certificates/schemas.py +++ b/lemur/pending_certificates/schemas.py @@ -37,6 +37,8 @@ class PendingCertificateOutputSchema(LemurOutputSchema): number_attempts = fields.Integer() date_created = fields.Date() last_updated = fields.Date() + resolved = fields.Boolean(required=False) + resolved_cert_id = fields.Integer(required=False) rotation = fields.Boolean() diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 64c03c46..405b2c4b 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -4,25 +4,21 @@ .. moduleauthor:: James Chuong """ import arrow - from sqlalchemy import or_, cast, Integer from lemur import database -from lemur.common.utils import truthiness -from lemur.plugins.base import plugins - -from lemur.roles.models import Role -from lemur.domains.models import Domain from lemur.authorities.models import Authority +from lemur.certificates import service as certificate_service +from lemur.certificates.schemas import CertificateUploadInputSchema +from lemur.common.utils import truthiness from lemur.destinations.models import Destination +from lemur.domains.models import Domain from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate - -from lemur.certificates import service as certificate_service +from lemur.plugins.base import plugins +from lemur.roles.models import Role from lemur.users import service as user_service -from lemur.certificates.schemas import CertificateUploadInputSchema - def get(pending_cert_id): """ @@ -63,6 +59,15 @@ def delete_by_id(id): database.delete(get(id)) +def get_unresolved_pending_certs(): + """ + Retrieve a list of unresolved pending certs given a list of ids + Filters out non-existing pending certs + """ + query = database.session_query(PendingCertificate).filter(PendingCertificate.resolved.is_(False)) + return database.find_all(query, PendingCertificate, {}).all() + + def get_pending_certs(pending_ids): """ Retrieve a list of pending certs given a list of ids @@ -116,6 +121,7 @@ def create_certificate(pending_certificate, certificate, user): # If generating name from certificate, remove the one from pending certificate del data['name'] data['creator'] = creator + cert = certificate_service.import_certificate(**data) database.update(cert) return cert @@ -172,8 +178,8 @@ def render(args): if 'issuer' in terms: # we can't rely on issuer being correct in the cert directly so we combine queries - sub_query = database.session_query(Authority.id)\ - .filter(Authority.name.ilike('%{0}%'.format(terms[1])))\ + sub_query = database.session_query(Authority.id) \ + .filter(Authority.name.ilike('%{0}%'.format(terms[1]))) \ .subquery() query = query.filter( @@ -221,4 +227,6 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now) + # Only show unresolved certificates in the UI + query = query.filter(PendingCertificate.resolved.is_(False)) return database.sort_and_page(query, PendingCertificate, args) From 4b3d458dba49869a841d2b6695f8eff55afa7f95 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 12 Oct 2018 05:46:58 -0700 Subject: [PATCH 2/6] Celery task to delete old pending certs --- lemur/common/celery.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 3f425950..8cc02daf 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -8,10 +8,10 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B """ import copy -import datetime import sys -from datetime import timezone +from datetime import datetime, timezone, timedelta +import arrow from celery import Celery from flask import current_app @@ -145,3 +145,21 @@ def fetch_all_pending_acme_certs(): if cert.last_updated == cert.date_created or datetime.datetime.now( timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3): fetch_acme_cert.delay(cert.id) + + +@celery.task() +def remove_old_acme_certs(): + """Prune old pending acme certificates from the database""" + log_data = { + "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) + } + pending_certs = pending_certificate_service.get_pending_certs('all') + + # Delete pending certs more than a week old + for cert in pending_certs: + if arrow.utcnow() - cert.last_updated > timedelta(days=7): + log_data['pending_cert_id'] = cert.id + log_data['pending_cert_name'] = cert.name + log_data['message'] = "Deleting pending certificate" + current_app.logger.debug(log_data) + pending_certificate_service.delete(cert.id) From 6073f9e7b632ae2d3859bbf13c211634fff9fdfa Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 12 Oct 2018 05:51:30 -0700 Subject: [PATCH 3/6] datetime ref fix --- lemur/common/celery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 8cc02daf..b0280afc 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -142,8 +142,8 @@ def fetch_all_pending_acme_certs(): for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': - if cert.last_updated == cert.date_created or datetime.datetime.now( - timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3): + if cert.last_updated == cert.date_created or datetime.now( + timezone.utc) - cert.last_updated > timedelta(minutes=3): fetch_acme_cert.delay(cert.id) From 13ef9656667737ed2e8f7a106eacecde6ff5814a Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 12 Oct 2018 05:56:14 -0700 Subject: [PATCH 4/6] nit: comments --- lemur/common/celery.py | 2 +- lemur/pending_certificates/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b0280afc..b7a0da5a 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -105,7 +105,7 @@ def fetch_acme_cert(id): if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - # Mark the pending cert as True + # Mark the pending cert as resolved pending_certificate_service.update( cert.get("pending_cert").id, resolved=True diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index c4e62549..6424c5f5 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -117,7 +117,7 @@ def fetch_all_acme(): error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > 4: - error_log["message"] = "Marking pending certificate" + error_log["message"] = "Marking pending certificate as resolved" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) # Mark "resolved" as True pending_certificate_service.update( From 89a077e54c17378f079572ab71b413a73c4e0002 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 12 Oct 2018 07:14:31 -0700 Subject: [PATCH 5/6] minor change to pass stuck github check --- lemur/pending_certificates/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index 6424c5f5..ccad8de5 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -22,7 +22,7 @@ manager = Manager(usage="Handles pending certificate related tasks.") @manager.option('-i', dest='ids', action='append', help='IDs of pending certificates to fetch') def fetch(ids): """ - Attempt to get full certificates for each pending certificate listed. + Attempt to get full certificate for each pending certificate listed. Args: ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI) From a912c3488deac5750c4c63b80e869389a271c594 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 12 Oct 2018 07:25:58 -0700 Subject: [PATCH 6/6] python fix to retrigger tests --- lemur/common/celery.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b7a0da5a..d3986351 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -11,7 +11,6 @@ import copy import sys from datetime import datetime, timezone, timedelta -import arrow from celery import Celery from flask import current_app @@ -157,7 +156,7 @@ def remove_old_acme_certs(): # Delete pending certs more than a week old for cert in pending_certs: - if arrow.utcnow() - cert.last_updated > timedelta(days=7): + if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): log_data['pending_cert_id'] = cert.id log_data['pending_cert_name'] = cert.name log_data['message'] = "Deleting pending certificate"