Merge branch 'github' into get_by_attributes

lemur/certificates/models.py

@@ -5,41 +5,32 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import arrow
 from datetime import timedelta
 
-from flask import current_app
+import arrow
-
 from cryptography import x509
 from cryptography.hazmat.primitives.asymmetric import rsa
-
+from flask import current_app
 from idna.core import InvalidCodepoint
-
+from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean, Index
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import case, extract
-from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
-
 from sqlalchemy_utils.types.arrow import ArrowType
 from werkzeug.utils import cached_property
 
-from lemur.database import db
-from lemur.extensions import sentry
-
-from lemur.utils import Vault
 from lemur.common import defaults, utils
-
-from lemur.plugins.base import plugins
-
-from lemur.extensions import metrics
 from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
-
+from lemur.database import db
+from lemur.domains.models import Domain
+from lemur.extensions import metrics
+from lemur.extensions import sentry
 from lemur.models import certificate_associations, certificate_source_associations, \
     certificate_destination_associations, certificate_notification_associations, \
     certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations
-
+from lemur.plugins.base import plugins
-from lemur.domains.models import Domain
 from lemur.policies.models import RotationPolicy
+from lemur.utils import Vault
 
 
 def get_sequence(name):
@@ -87,6 +78,7 @@ def get_or_increase_name(name, serial):
 class Certificate(db.Model):
     __tablename__ = 'certificates'
     id = Column(Integer, primary_key=True)
+    ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
     external_id = Column(String(128))
     owner = Column(String(128), nullable=False)
     name = Column(String(256), unique=True)

lemur/certificates/schemas.py

@@ -193,6 +193,8 @@ class CertificateOutputSchema(LemurOutputSchema):
     name = fields.String()
     dns_provider_id = fields.Integer(required=False, allow_none=True)
     date_created = ArrowDateTime()
+    resolved = fields.Boolean(required=False, allow_none=True)
+    resolved_cert_id = fields.Integer(required=False, allow_none=True)
 
     rotation = fields.Boolean()
 

lemur/common/celery.py

@@ -8,9 +8,8 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
 
 """
 import copy
-import datetime
 import sys
-from datetime import timezone
+from datetime import datetime, timezone, timedelta
 
 from celery import Celery
 from flask import current_app
@@ -20,7 +19,6 @@ from lemur.factory import create_app
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
-from lemur.users import service as user_service
 
 flask_app = create_app()
 
@@ -57,7 +55,6 @@ def fetch_acme_cert(id):
         "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
     }
     pending_certs = pending_certificate_service.get_pending_certs([id])
-    user = user_service.get_by_username('lemur')
     new = 0
     failed = 0
     wrong_issuer = 0
@@ -78,12 +75,22 @@ def fetch_acme_cert(id):
         real_cert = cert.get("cert")
         # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
         pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
-
+        if not pending_cert:
+            log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
+            current_app.logger.error(log_data)
+            continue
         if real_cert:
-            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # If a real certificate was returned from issuer, then create it in Lemur and mark
-            # the pending certificate
+            # the pending certificate as resolved
-            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
+            final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
-            pending_certificate_service.delete_by_id(pending_cert.id)
+            pending_certificate_service.update(
+                cert.get("pending_cert").id,
+                resolved=True
+            )
+            pending_certificate_service.update(
+                cert.get("pending_cert").id,
+                resolved_cert_id=final_cert.id
+            )
             # add metrics to metrics extension
             new += 1
         else:
@@ -97,7 +104,11 @@ def fetch_acme_cert(id):
             if pending_cert.number_attempts > 4:
                 error_log["message"] = "Deleting pending certificate"
                 send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
-                pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
+                # Mark the pending cert as resolved
+                pending_certificate_service.update(
+                    cert.get("pending_cert").id,
+                    resolved=True
+                )
             else:
                 pending_certificate_service.increment_attempt(pending_cert)
                 pending_certificate_service.update(
@@ -124,12 +135,30 @@ def fetch_acme_cert(id):
 @celery.task()
 def fetch_all_pending_acme_certs():
     """Instantiate celery workers to resolve all pending Acme certificates"""
-    pending_certs = pending_certificate_service.get_pending_certs('all')
+    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
 
     # We only care about certs using the acme-issuer plugin
     for cert in pending_certs:
         cert_authority = get_authority(cert.authority_id)
         if cert_authority.plugin_name == 'acme-issuer':
-            if cert.last_updated == cert.date_created or datetime.datetime.now(
+            if cert.last_updated == cert.date_created or datetime.now(
-                    timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3):
+                    timezone.utc) - cert.last_updated > timedelta(minutes=3):
                 fetch_acme_cert.delay(cert.id)
+
+
+@celery.task()
+def remove_old_acme_certs():
+    """Prune old pending acme certificates from the database"""
+    log_data = {
+        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
+    }
+    pending_certs = pending_certificate_service.get_pending_certs('all')
+
+    # Delete pending certs more than a week old
+    for cert in pending_certs:
+        if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
+            log_data['pending_cert_id'] = cert.id
+            log_data['pending_cert_name'] = cert.name
+            log_data['message'] = "Deleting pending certificate"
+            current_app.logger.debug(log_data)
+            pending_certificate_service.delete(cert.id)

lemur/domains/models.py

@@ -15,7 +15,7 @@ from lemur.database import db
 class Domain(db.Model):
     __tablename__ = 'domains'
     id = Column(Integer, primary_key=True)
-    name = Column(String(256))
+    name = Column(String(256), index=True)
     sensitive = Column(Boolean, default=False)
 
     def __repr__(self):

lemur/migrations/versions/984178255c83_.py

@@ -0,0 +1,24 @@
+"""Add status to pending certificate, and store resolved cert id
+
+Revision ID: 984178255c83
+Revises: f2383bf08fbc
+Create Date: 2018-10-11 20:49:12.704563
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '984178255c83'
+down_revision = 'f2383bf08fbc'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('pending_certs', sa.Column('resolved', sa.Boolean(), nullable=True))
+    op.add_column('pending_certs', sa.Column('resolved_cert_id', sa.Integer(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('pending_certs', 'resolved_cert_id')
+    op.drop_column('pending_certs', 'resolved')

lemur/migrations/versions/c87cb989af04_.py

@@ -0,0 +1,19 @@
+"""Create an index on the domains table for the domain name
+Revision ID: c87cb989af04
+Revises: 9392b9f9a805
+Create Date: 2018-10-11 09:44:57.099854
+
+"""
+
+revision = 'c87cb989af04'
+down_revision = '9392b9f9a805'
+
+from alembic import op
+
+
+def upgrade():
+    op.create_index(op.f('ix_domains_name'), 'domains', ['name'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_domains_name'), table_name='domains')

lemur/migrations/versions/f2383bf08fbc_.py

@@ -0,0 +1,23 @@
+"""Create index on certificates table for id desc
+
+
+Revision ID: f2383bf08fbc
+Revises: c87cb989af04
+Create Date: 2018-10-11 11:23:31.195471
+
+"""
+
+revision = 'f2383bf08fbc'
+down_revision = 'c87cb989af04'
+
+import sqlalchemy as sa
+from alembic import op
+
+
+def upgrade():
+    op.create_index('ix_certificates_id_desc', 'certificates', [sa.text('id DESC')], unique=True,
+                    postgresql_using='btree')
+
+
+def downgrade():
+    op.drop_index('ix_certificates_id_desc', table_name='certificates')

lemur/pending_certificates/cli.py

@@ -15,7 +15,6 @@ from lemur.authorities.service import get as get_authority
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
-from lemur.users import service as user_service
 
 manager = Manager(usage="Handles pending certificate related tasks.")
 
@@ -23,14 +22,14 @@ manager = Manager(usage="Handles pending certificate related tasks.")
 @manager.option('-i', dest='ids', action='append', help='IDs of pending certificates to fetch')
 def fetch(ids):
     """
-    Attempt to get full certificates for each pending certificate listed.
+    Attempt to get full certificate for each pending certificate listed.
 
     Args:
         ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
              `python manager.py pending_certs fetch -i 123 321 all`
     """
     pending_certs = pending_certificate_service.get_pending_certs(ids)
-    user = user_service.get_by_username('lemur')
+
     new = 0
     failed = 0
 
@@ -38,10 +37,17 @@ def fetch(ids):
         authority = plugins.get(cert.authority.plugin_name)
         real_cert = authority.get_ordered_certificate(cert)
         if real_cert:
-            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # If a real certificate was returned from issuer, then create it in Lemur and mark
-            # the pending certificate
+            # the pending certificate as resolved
-            pending_certificate_service.create_certificate(cert, real_cert, user)
+            final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user)
-            pending_certificate_service.delete(cert)
+            pending_certificate_service.update(
+                cert.id,
+                resolved=True
+            )
+            pending_certificate_service.update(
+                cert.id,
+                resolved_cert_id=final_cert.id
+            )
             # add metrics to metrics extension
             new += 1
         else:
@@ -66,8 +72,7 @@ def fetch_all_acme():
     log_data = {
         "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
     }
-    pending_certs = pending_certificate_service.get_pending_certs('all')
+    pending_certs = pending_certificate_service.get_unresolved_pending_certs()
-    user = user_service.get_by_username('lemur')
     new = 0
     failed = 0
     wrong_issuer = 0
@@ -90,10 +95,17 @@ def fetch_all_acme():
         pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
 
         if real_cert:
-            # If a real certificate was returned from issuer, then create it in Lemur and delete
+            # If a real certificate was returned from issuer, then create it in Lemur and mark
-            # the pending certificate
+            # the pending certificate as resolved
-            pending_certificate_service.create_certificate(pending_cert, real_cert, user)
+            final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
-            pending_certificate_service.delete_by_id(pending_cert.id)
+            pending_certificate_service.update(
+                pending_cert.id,
+                resolved=True
+            )
+            pending_certificate_service.update(
+                pending_cert.id,
+                resolved_cert_id=final_cert.id
+            )
             # add metrics to metrics extension
             new += 1
         else:
@@ -105,9 +117,13 @@ def fetch_all_acme():
             error_log["cn"] = pending_cert.cn
 
             if pending_cert.number_attempts > 4:
-                error_log["message"] = "Deleting pending certificate"
+                error_log["message"] = "Marking pending certificate as resolved"
                 send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
-                pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
+                # Mark "resolved" as True
+                pending_certificate_service.update(
+                    cert.id,
+                    resolved=True
+                )
             else:
                 pending_certificate_service.increment_attempt(pending_cert)
                 pending_certificate_service.update(

lemur/pending_certificates/models.py

@@ -29,6 +29,8 @@ class PendingCertificate(db.Model):
     notify = Column(Boolean, default=True)
     number_attempts = Column(Integer)
     rename = Column(Boolean, default=True)
+    resolved = Column(Boolean, default=False)
+    resolved_cert_id = Column(Integer, nullable=True)
 
     cn = Column(String(128))
     csr = Column(Text(), nullable=False)

lemur/pending_certificates/schemas.py

@@ -37,6 +37,8 @@ class PendingCertificateOutputSchema(LemurOutputSchema):
     number_attempts = fields.Integer()
     date_created = fields.Date()
     last_updated = fields.Date()
+    resolved = fields.Boolean(required=False)
+    resolved_cert_id = fields.Integer(required=False)
 
     rotation = fields.Boolean()
 

lemur/pending_certificates/service.py

@@ -4,25 +4,21 @@
 .. moduleauthor:: James Chuong <jchuong@instartlogic.com>
 """
 import arrow
-
 from sqlalchemy import or_, cast, Integer
 
 from lemur import database
-from lemur.common.utils import truthiness
-from lemur.plugins.base import plugins
-
-from lemur.roles.models import Role
-from lemur.domains.models import Domain
 from lemur.authorities.models import Authority
+from lemur.certificates import service as certificate_service
+from lemur.certificates.schemas import CertificateUploadInputSchema
+from lemur.common.utils import truthiness
 from lemur.destinations.models import Destination
+from lemur.domains.models import Domain
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
-
+from lemur.plugins.base import plugins
-from lemur.certificates import service as certificate_service
+from lemur.roles.models import Role
 from lemur.users import service as user_service
 
-from lemur.certificates.schemas import CertificateUploadInputSchema
-
 
 def get(pending_cert_id):
     """
@@ -63,6 +59,15 @@ def delete_by_id(id):
     database.delete(get(id))
 
 
+def get_unresolved_pending_certs():
+    """
+    Retrieve a list of unresolved pending certs given a list of ids
+    Filters out non-existing pending certs
+    """
+    query = database.session_query(PendingCertificate).filter(PendingCertificate.resolved.is_(False))
+    return database.find_all(query, PendingCertificate, {}).all()
+
+
 def get_pending_certs(pending_ids):
     """
     Retrieve a list of pending certs given a list of ids
@@ -116,6 +121,7 @@ def create_certificate(pending_certificate, certificate, user):
         # If generating name from certificate, remove the one from pending certificate
         del data['name']
     data['creator'] = creator
+
     cert = certificate_service.import_certificate(**data)
     database.update(cert)
     return cert
@@ -172,8 +178,8 @@ def render(args):
 
         if 'issuer' in terms:
             # we can't rely on issuer being correct in the cert directly so we combine queries
-            sub_query = database.session_query(Authority.id)\
+            sub_query = database.session_query(Authority.id) \
-                .filter(Authority.name.ilike('%{0}%'.format(terms[1])))\
+                .filter(Authority.name.ilike('%{0}%'.format(terms[1]))) \
                 .subquery()
 
             query = query.filter(
@@ -221,4 +227,6 @@ def render(args):
         now = arrow.now().format('YYYY-MM-DD')
         query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now)
 
+    # Only show unresolved certificates in the UI
+    query = query.filter(PendingCertificate.resolved.is_(False))
     return database.sort_and_page(query, PendingCertificate, args)

requirements-dev.txt

@@ -5,26 +5,35 @@
 #    pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
 #
 aspy.yaml==1.1.1          # via pre-commit
+bleach==3.0.2             # via readme-renderer
 cached-property==1.5.1    # via pre-commit
 certifi==2018.8.24        # via requests
+cffi==1.11.5              # via cmarkgfm
 cfgv==1.1.0               # via pre-commit
 chardet==3.0.4            # via requests
+cmarkgfm==0.4.2           # via readme-renderer
+docutils==0.14            # via readme-renderer
 flake8==3.5.0
-identify==1.1.6           # via pre-commit
+future==0.16.0            # via readme-renderer
+identify==1.1.7           # via pre-commit
 idna==2.7                 # via requests
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.2
 pkginfo==1.4.2            # via twine
-pre-commit==1.11.0
+pre-commit==1.11.2
 pycodestyle==2.3.1        # via flake8
+pycparser==2.19           # via cffi
 pyflakes==1.6.0           # via flake8
+pygments==2.2.0           # via readme-renderer
 pyyaml==3.13              # via aspy.yaml, pre-commit
+readme-renderer==22.0     # via twine
 requests-toolbelt==0.8.0  # via twine
 requests==2.19.1          # via requests-toolbelt, twine
-six==1.11.0               # via cfgv, pre-commit
+six==1.11.0               # via bleach, cfgv, pre-commit, readme-renderer
-toml==0.9.6               # via pre-commit
+toml==0.10.0              # via pre-commit
 tqdm==4.26.0              # via twine
-twine==1.11.0
+twine==1.12.1
 urllib3==1.23             # via requests
 virtualenv==16.0.0        # via pre-commit
+webencodings==0.5.1       # via bleach

requirements-docs.txt

@@ -5,7 +5,7 @@
 #    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
 acme==0.27.1
-alabaster==0.7.11         # via sphinx
+alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.0
 amqp==2.3.2
@@ -54,10 +54,10 @@ lockfile==0.12.2
 mako==1.0.7
 markupsafe==1.0
 marshmallow-sqlalchemy==0.14.1
-marshmallow==2.15.4
+marshmallow==2.15.5
 mock==2.0.0
 ndg-httpsclient==0.5.1
-packaging==17.1           # via sphinx
+packaging==18.0           # via sphinx
 paramiko==2.4.1
 pbr==4.2.0
 pem==18.1.0
@@ -69,7 +69,7 @@ pygments==2.2.0           # via sphinx
 pyjwt==1.6.4
 pynacl==1.2.1
 pyopenssl==18.0.0
-pyparsing==2.2.0          # via packaging
+pyparsing==2.2.2          # via packaging
 pyrfc3339==1.1
 python-dateutil==2.7.3
 python-editor==1.0.3
@@ -83,8 +83,8 @@ retrying==1.3.3
 s3transfer==0.1.13
 six==1.11.0
 snowballstemmer==1.2.1    # via sphinx
-sphinx-rtd-theme==0.4.1
+sphinx-rtd-theme==0.4.2
-sphinx==1.8.0
+sphinx==1.8.1
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx
 sqlalchemy-utils==0.33.4

requirements-tests.txt

@@ -8,13 +8,13 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.9.4              # via moto
+boto3==1.9.21             # via moto
 boto==2.49.0              # via moto
-botocore==1.12.4          # via boto3, moto, s3transfer
+botocore==1.12.21         # via boto3, moto, s3transfer
 certifi==2018.8.24        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
-click==6.7                # via flask
+click==7.0                # via flask
 cookies==2.2.1            # via moto, responses
 coverage==4.5.1
 cryptography==2.3.1       # via moto
@@ -32,22 +32,22 @@ itsdangerous==0.24        # via flask
 jinja2==2.10              # via flask, moto
 jmespath==0.9.3           # via boto3, botocore
 jsondiff==1.1.1           # via moto
-jsonpickle==0.9.6         # via aws-xray-sdk
+jsonpickle==1.0           # via aws-xray-sdk
 markupsafe==1.0           # via jinja2
 mock==2.0.0               # via moto
 more-itertools==4.3.0     # via pytest
 moto==1.3.4
 nose==1.3.7
-pbr==4.2.0                # via mock
+pbr==4.3.0                # via mock
 pluggy==0.7.1             # via pytest
 py==1.6.0                 # via pytest
 pyaml==17.12.1            # via moto
-pycparser==2.18           # via cffi
+pycparser==2.19           # via cffi
 pycryptodome==3.6.6       # via python-jose
 pyflakes==2.0.0
-pytest-flask==0.12.0
+pytest-flask==0.13.0
 pytest-mock==1.10.0
-pytest==3.8.0
+pytest==3.8.2
 python-dateutil==2.7.3    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
 pytz==2018.5              # via moto

requirements.txt

@@ -21,7 +21,7 @@ celery[redis]==4.2.1
 certifi==2018.8.24
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
-click==6.7                # via flask
+click==7.0                # via flask
 cloudflare==2.1.0
 cryptography==2.3.1
 dnspython3==1.15.0
@@ -51,18 +51,18 @@ lockfile==0.12.2
 mako==1.0.7               # via alembic
 markupsafe==1.0           # via jinja2, mako
 marshmallow-sqlalchemy==0.14.1
-marshmallow==2.15.5
+marshmallow==2.16.0
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
-paramiko==2.4.1
+paramiko==2.4.2
-pbr==4.2.0                # via mock
+pbr==4.3.0                # via mock
-pem==18.1.0
+pem==18.2.0
 psycopg2==2.7.5
 pyasn1-modules==0.2.2     # via python-ldap
 pyasn1==0.4.4             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
-pycparser==2.18           # via cffi
+pycparser==2.19           # via cffi
 pyjwt==1.6.4
-pynacl==1.2.1             # via paramiko
+pynacl==1.3.0             # via paramiko
 pyopenssl==18.0.0
 pyrfc3339==1.1            # via acme
 python-dateutil==2.7.3    # via alembic, arrow, botocore
@@ -77,8 +77,8 @@ requests[security]==2.19.1
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.11.0
-sqlalchemy-utils==0.33.4
+sqlalchemy-utils==0.33.5
-sqlalchemy==1.2.11        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.2.12        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
 urllib3==1.23             # via requests
 vine==1.1.4               # via amqp