diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 032a9175..a92f1f66 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -193,6 +193,8 @@ class CertificateOutputSchema(LemurOutputSchema): name = fields.String() dns_provider_id = fields.Integer(required=False, allow_none=True) date_created = ArrowDateTime() + resolved = fields.Boolean(required=False, allow_none=True) + resolved_cert_id = fields.Integer(required=False, allow_none=True) rotation = fields.Boolean() diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 1858b4b5..3f425950 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -20,7 +20,6 @@ from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.users import service as user_service flask_app = create_app() @@ -57,7 +56,6 @@ def fetch_acme_cert(id): "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } pending_certs = pending_certificate_service.get_pending_certs([id]) - user = user_service.get_by_username('lemur') new = 0 failed = 0 wrong_issuer = 0 @@ -78,12 +76,22 @@ def fetch_acme_cert(id): real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) - + if not pending_cert: + log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?" + current_app.logger.error(log_data) + continue if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(pending_cert, real_cert, user) - pending_certificate_service.delete_by_id(pending_cert.id) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user) + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved=True + ) + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -97,7 +105,11 @@ def fetch_acme_cert(id): if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) + # Mark the pending cert as True + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved=True + ) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( @@ -124,7 +136,7 @@ def fetch_acme_cert(id): @celery.task() def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" - pending_certs = pending_certificate_service.get_pending_certs('all') + pending_certs = pending_certificate_service.get_unresolved_pending_certs() # We only care about certs using the acme-issuer plugin for cert in pending_certs: diff --git a/lemur/migrations/versions/984178255c83_.py b/lemur/migrations/versions/984178255c83_.py new file mode 100644 index 00000000..40d2ce31 --- /dev/null +++ b/lemur/migrations/versions/984178255c83_.py @@ -0,0 +1,24 @@ +"""Add status to pending certificate, and store resolved cert id + +Revision ID: 984178255c83 +Revises: f2383bf08fbc +Create Date: 2018-10-11 20:49:12.704563 + +""" + +# revision identifiers, used by Alembic. +revision = '984178255c83' +down_revision = 'f2383bf08fbc' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('pending_certs', sa.Column('resolved', sa.Boolean(), nullable=True)) + op.add_column('pending_certs', sa.Column('resolved_cert_id', sa.Integer(), nullable=True)) + + +def downgrade(): + op.drop_column('pending_certs', 'resolved_cert_id') + op.drop_column('pending_certs', 'resolved') diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index cbce700d..c4e62549 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -15,7 +15,6 @@ from lemur.authorities.service import get as get_authority from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.users import service as user_service manager = Manager(usage="Handles pending certificate related tasks.") @@ -30,7 +29,7 @@ def fetch(ids): `python manager.py pending_certs fetch -i 123 321 all` """ pending_certs = pending_certificate_service.get_pending_certs(ids) - user = user_service.get_by_username('lemur') + new = 0 failed = 0 @@ -38,10 +37,17 @@ def fetch(ids): authority = plugins.get(cert.authority.plugin_name) real_cert = authority.get_ordered_certificate(cert) if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(cert, real_cert, user) - pending_certificate_service.delete(cert) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user) + pending_certificate_service.update( + cert.id, + resolved=True + ) + pending_certificate_service.update( + cert.id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -66,8 +72,7 @@ def fetch_all_acme(): log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } - pending_certs = pending_certificate_service.get_pending_certs('all') - user = user_service.get_by_username('lemur') + pending_certs = pending_certificate_service.get_unresolved_pending_certs() new = 0 failed = 0 wrong_issuer = 0 @@ -90,10 +95,17 @@ def fetch_all_acme(): pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(pending_cert, real_cert, user) - pending_certificate_service.delete_by_id(pending_cert.id) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user) + pending_certificate_service.update( + pending_cert.id, + resolved=True + ) + pending_certificate_service.update( + pending_cert.id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -105,9 +117,13 @@ def fetch_all_acme(): error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > 4: - error_log["message"] = "Deleting pending certificate" + error_log["message"] = "Marking pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) + # Mark "resolved" as True + pending_certificate_service.update( + cert.id, + resolved=True + ) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index a1f61fa1..1261177d 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -29,6 +29,8 @@ class PendingCertificate(db.Model): notify = Column(Boolean, default=True) number_attempts = Column(Integer) rename = Column(Boolean, default=True) + resolved = Column(Boolean, default=False) + resolved_cert_id = Column(Integer, nullable=True) cn = Column(String(128)) csr = Column(Text(), nullable=False) diff --git a/lemur/pending_certificates/schemas.py b/lemur/pending_certificates/schemas.py index 252cd892..fbc94f4e 100644 --- a/lemur/pending_certificates/schemas.py +++ b/lemur/pending_certificates/schemas.py @@ -37,6 +37,8 @@ class PendingCertificateOutputSchema(LemurOutputSchema): number_attempts = fields.Integer() date_created = fields.Date() last_updated = fields.Date() + resolved = fields.Boolean(required=False) + resolved_cert_id = fields.Integer(required=False) rotation = fields.Boolean() diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 64c03c46..405b2c4b 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -4,25 +4,21 @@ .. moduleauthor:: James Chuong """ import arrow - from sqlalchemy import or_, cast, Integer from lemur import database -from lemur.common.utils import truthiness -from lemur.plugins.base import plugins - -from lemur.roles.models import Role -from lemur.domains.models import Domain from lemur.authorities.models import Authority +from lemur.certificates import service as certificate_service +from lemur.certificates.schemas import CertificateUploadInputSchema +from lemur.common.utils import truthiness from lemur.destinations.models import Destination +from lemur.domains.models import Domain from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate - -from lemur.certificates import service as certificate_service +from lemur.plugins.base import plugins +from lemur.roles.models import Role from lemur.users import service as user_service -from lemur.certificates.schemas import CertificateUploadInputSchema - def get(pending_cert_id): """ @@ -63,6 +59,15 @@ def delete_by_id(id): database.delete(get(id)) +def get_unresolved_pending_certs(): + """ + Retrieve a list of unresolved pending certs given a list of ids + Filters out non-existing pending certs + """ + query = database.session_query(PendingCertificate).filter(PendingCertificate.resolved.is_(False)) + return database.find_all(query, PendingCertificate, {}).all() + + def get_pending_certs(pending_ids): """ Retrieve a list of pending certs given a list of ids @@ -116,6 +121,7 @@ def create_certificate(pending_certificate, certificate, user): # If generating name from certificate, remove the one from pending certificate del data['name'] data['creator'] = creator + cert = certificate_service.import_certificate(**data) database.update(cert) return cert @@ -172,8 +178,8 @@ def render(args): if 'issuer' in terms: # we can't rely on issuer being correct in the cert directly so we combine queries - sub_query = database.session_query(Authority.id)\ - .filter(Authority.name.ilike('%{0}%'.format(terms[1])))\ + sub_query = database.session_query(Authority.id) \ + .filter(Authority.name.ilike('%{0}%'.format(terms[1]))) \ .subquery() query = query.filter( @@ -221,4 +227,6 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now) + # Only show unresolved certificates in the UI + query = query.filter(PendingCertificate.resolved.is_(False)) return database.sort_and_page(query, PendingCertificate, args)