Merge pull request #1831 from castrapel/lemur_letsencrypt_resolved_info

Lemur LetsEncrypt Polling Support

lemur/certificates/schemas.py

@@ -193,6 +193,8 @@ class CertificateOutputSchema(LemurOutputSchema):
     name = fields.String()
     dns_provider_id = fields.Integer(required=False, allow_none=True)
     date_created = ArrowDateTime()
+    resolved = fields.Boolean(required=False, allow_none=True)
+    resolved_cert_id = fields.Integer(required=False, allow_none=True)
 
     rotation = fields.Boolean()
 

lemur/common/celery.py

@@ -8,9 +8,8 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
 
 """
 import copy
-import datetime
 import sys
-from datetime import timezone
+from datetime import datetime, timezone, timedelta
 
 from celery import Celery
 from flask import current_app
@@ -20,7 +19,6 @@ from lemur.factory import create_app
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
-from lemur.users import service as user_service
 
 flask_app = create_app()
 
@@ -57,7 +55,6 @@ def fetch_acme_cert(id):
         "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
     }
     pending_certs = pending_certificate_service.get_pending_certs([id])
-    user = user_service.get_by_username('lemur')
     new = 0
     failed = 0
     wrong_issuer = 0
@@ -78,12 +75,22 @@ def fetch_acme_cert(id):
         real_cert = cert.get("cert")
         # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
         pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
-
+        if not pending_cert:
+            log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
+            current_app.logger.error(log_data)
+            continue
         if real_cert:
-            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # If a real certificate was returned from issuer, then create it in Lemur and mark
-            # the pending certificate
+            # the pending certificate as resolved
-            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
+            final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
-            pending_certificate_service.delete_by_id(pending_cert.id)
+            pending_certificate_service.update(
+                cert.get("pending_cert").id,
+                resolved=True
+            )
+            pending_certificate_service.update(
+                cert.get("pending_cert").id,
+                resolved_cert_id=final_cert.id
+            )
             # add metrics to metrics extension
             new += 1
         else:
@@ -97,7 +104,11 @@ def fetch_acme_cert(id):
             if pending_cert.number_attempts > 4:
                 error_log["message"] = "Deleting pending certificate"
                 send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
-                pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
+                # Mark the pending cert as resolved
+                pending_certificate_service.update(
+                    cert.get("pending_cert").id,
+                    resolved=True
+                )
             else:
                 pending_certificate_service.increment_attempt(pending_cert)
                 pending_certificate_service.update(
@@ -124,12 +135,30 @@ def fetch_acme_cert(id):
 @celery.task()
 def fetch_all_pending_acme_certs():
     """Instantiate celery workers to resolve all pending Acme certificates"""
-    pending_certs = pending_certificate_service.get_pending_certs('all')
+    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
 
     # We only care about certs using the acme-issuer plugin
     for cert in pending_certs:
         cert_authority = get_authority(cert.authority_id)
         if cert_authority.plugin_name == 'acme-issuer':
-            if cert.last_updated == cert.date_created or datetime.datetime.now(
+            if cert.last_updated == cert.date_created or datetime.now(
-                    timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3):
+                    timezone.utc) - cert.last_updated > timedelta(minutes=3):
                 fetch_acme_cert.delay(cert.id)
+
+
+@celery.task()
+def remove_old_acme_certs():
+    """Prune old pending acme certificates from the database"""
+    log_data = {
+        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
+    }
+    pending_certs = pending_certificate_service.get_pending_certs('all')
+
+    # Delete pending certs more than a week old
+    for cert in pending_certs:
+        if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
+            log_data['pending_cert_id'] = cert.id
+            log_data['pending_cert_name'] = cert.name
+            log_data['message'] = "Deleting pending certificate"
+            current_app.logger.debug(log_data)
+            pending_certificate_service.delete(cert.id)

lemur/migrations/versions/984178255c83_.py

@@ -0,0 +1,24 @@
+"""Add status to pending certificate, and store resolved cert id
+
+Revision ID: 984178255c83
+Revises: f2383bf08fbc
+Create Date: 2018-10-11 20:49:12.704563
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '984178255c83'
+down_revision = 'f2383bf08fbc'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('pending_certs', sa.Column('resolved', sa.Boolean(), nullable=True))
+    op.add_column('pending_certs', sa.Column('resolved_cert_id', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('pending_certs', 'resolved_cert_id')
+    op.drop_column('pending_certs', 'resolved')

lemur/pending_certificates/cli.py

@@ -15,7 +15,6 @@ from lemur.authorities.service import get as get_authority
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
-from lemur.users import service as user_service
 
 manager = Manager(usage="Handles pending certificate related tasks.")
 
@@ -23,14 +22,14 @@ manager = Manager(usage="Handles pending certificate related tasks.")
 @manager.option('-i', dest='ids', action='append', help='IDs of pending certificates to fetch')
 def fetch(ids):
     """
-    Attempt to get full certificates for each pending certificate listed.
+    Attempt to get full certificate for each pending certificate listed.
 
     Args:
         ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
              `python manager.py pending_certs fetch -i 123 321 all`
     """
     pending_certs = pending_certificate_service.get_pending_certs(ids)
-    user = user_service.get_by_username('lemur')
+
     new = 0
     failed = 0
 
@@ -38,10 +37,17 @@ def fetch(ids):
         authority = plugins.get(cert.authority.plugin_name)
         real_cert = authority.get_ordered_certificate(cert)
         if real_cert:
-            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # If a real certificate was returned from issuer, then create it in Lemur and mark
-            # the pending certificate
+            # the pending certificate as resolved
-            pending_certificate_service.create_certificate(cert, real_cert, user)
+            final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user)
-            pending_certificate_service.delete(cert)
+            pending_certificate_service.update(
+                cert.id,
+                resolved=True
+            )
+            pending_certificate_service.update(
+                cert.id,
+                resolved_cert_id=final_cert.id
+            )
             # add metrics to metrics extension
             new += 1
         else:
@@ -66,8 +72,7 @@ def fetch_all_acme():
     log_data = {
         "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
     }
-    pending_certs = pending_certificate_service.get_pending_certs('all')
+    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
-    user = user_service.get_by_username('lemur')
     new = 0
     failed = 0
     wrong_issuer = 0
@@ -90,10 +95,17 @@ def fetch_all_acme():
         pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
 
         if real_cert:
-            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # If a real certificate was returned from issuer, then create it in Lemur and mark
-            # the pending certificate
+            # the pending certificate as resolved
-            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
+            final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
-            pending_certificate_service.delete_by_id(pending_cert.id)
+            pending_certificate_service.update(
+                pending_cert.id,
+                resolved=True
+            )
+            pending_certificate_service.update(
+                pending_cert.id,
+                resolved_cert_id=final_cert.id
+            )
             # add metrics to metrics extension
             new += 1
         else:
@@ -105,9 +117,13 @@ def fetch_all_acme():
             error_log["cn"] = pending_cert.cn
 
             if pending_cert.number_attempts > 4:
-                error_log["message"] = "Deleting pending certificate"
+                error_log["message"] = "Marking pending certificate as resolved"
                 send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
-                pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
+                # Mark "resolved" as True
+                pending_certificate_service.update(
+                    cert.id,
+                    resolved=True
+                )
             else:
                 pending_certificate_service.increment_attempt(pending_cert)
                 pending_certificate_service.update(

lemur/pending_certificates/models.py

@@ -29,6 +29,8 @@ class PendingCertificate(db.Model):
     notify = Column(Boolean, default=True)
     number_attempts = Column(Integer)
     rename = Column(Boolean, default=True)
+    resolved = Column(Boolean, default=False)
+    resolved_cert_id = Column(Integer, nullable=True)
 
     cn = Column(String(128))
     csr = Column(Text(), nullable=False)

lemur/pending_certificates/schemas.py

@@ -37,6 +37,8 @@ class PendingCertificateOutputSchema(LemurOutputSchema):
     number_attempts = fields.Integer()
     date_created = fields.Date()
     last_updated = fields.Date()
+    resolved = fields.Boolean(required=False)
+    resolved_cert_id = fields.Integer(required=False)
 
     rotation = fields.Boolean()
 

lemur/pending_certificates/service.py

@@ -4,25 +4,21 @@
 .. moduleauthor:: James Chuong <jchuong@instartlogic.com>
 """
 import arrow
-
 from sqlalchemy import or_, cast, Integer
 
 from lemur import database
-from lemur.common.utils import truthiness
-from lemur.plugins.base import plugins
-
-from lemur.roles.models import Role
-from lemur.domains.models import Domain
 from lemur.authorities.models import Authority
+from lemur.certificates import service as certificate_service
+from lemur.certificates.schemas import CertificateUploadInputSchema
+from lemur.common.utils import truthiness
 from lemur.destinations.models import Destination
+from lemur.domains.models import Domain
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
-
+from lemur.plugins.base import plugins
-from lemur.certificates import service as certificate_service
+from lemur.roles.models import Role
 from lemur.users import service as user_service
 
-from lemur.certificates.schemas import CertificateUploadInputSchema
-
 
 def get(pending_cert_id):
     """
@@ -63,6 +59,15 @@ def delete_by_id(id):
     database.delete(get(id))
 
 
+def get_unresolved_pending_certs():
+    """
+    Retrieve a list of unresolved pending certs given a list of ids
+    Filters out non-existing pending certs
+    """
+    query = database.session_query(PendingCertificate).filter(PendingCertificate.resolved.is_(False))
+    return database.find_all(query, PendingCertificate, {}).all()
+
+
 def get_pending_certs(pending_ids):
     """
     Retrieve a list of pending certs given a list of ids
@@ -116,6 +121,7 @@ def create_certificate(pending_certificate, certificate, user):
         # If generating name from certificate, remove the one from pending certificate
         del data['name']
     data['creator'] = creator
+
     cert = certificate_service.import_certificate(**data)
     database.update(cert)
     return cert
@@ -172,8 +178,8 @@ def render(args):
 
         if 'issuer' in terms:
             # we can't rely on issuer being correct in the cert directly so we combine queries
-            sub_query = database.session_query(Authority.id)\
+            sub_query = database.session_query(Authority.id) \
-                .filter(Authority.name.ilike('%{0}%'.format(terms[1])))\
+                .filter(Authority.name.ilike('%{0}%'.format(terms[1]))) \
                 .subquery()
 
             query = query.filter(
@@ -221,4 +227,6 @@ def render(args):
         now = arrow.now().format('YYYY-MM-DD')
         query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now)
 
+    # Only show unresolved certificates in the UI
+    query = query.filter(PendingCertificate.resolved.is_(False))
     return database.sort_and_page(query, PendingCertificate, args)