diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index b337c755..5e0c42cc 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -5,41 +5,32 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import arrow from datetime import timedelta -from flask import current_app - +import arrow from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa - +from flask import current_app from idna.core import InvalidCodepoint - +from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean, Index +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import case, extract -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean - from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property -from lemur.database import db -from lemur.extensions import sentry - -from lemur.utils import Vault from lemur.common import defaults, utils - -from lemur.plugins.base import plugins - -from lemur.extensions import metrics from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS - +from lemur.database import db +from lemur.domains.models import Domain +from lemur.extensions import metrics +from lemur.extensions import sentry from lemur.models import certificate_associations, certificate_source_associations, \ certificate_destination_associations, certificate_notification_associations, \ certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations - -from lemur.domains.models import Domain +from lemur.plugins.base import plugins from lemur.policies.models import RotationPolicy +from lemur.utils import Vault def get_sequence(name): @@ -87,6 +78,7 @@ def get_or_increase_name(name, serial): class Certificate(db.Model): __tablename__ = 'certificates' id = Column(Integer, primary_key=True) + ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True) external_id = Column(String(128)) owner = Column(String(128), nullable=False) name = Column(String(256), unique=True) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 032a9175..a92f1f66 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -193,6 +193,8 @@ class CertificateOutputSchema(LemurOutputSchema): name = fields.String() dns_provider_id = fields.Integer(required=False, allow_none=True) date_created = ArrowDateTime() + resolved = fields.Boolean(required=False, allow_none=True) + resolved_cert_id = fields.Integer(required=False, allow_none=True) rotation = fields.Boolean() diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 1858b4b5..d3986351 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -8,9 +8,8 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B """ import copy -import datetime import sys -from datetime import timezone +from datetime import datetime, timezone, timedelta from celery import Celery from flask import current_app @@ -20,7 +19,6 @@ from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.users import service as user_service flask_app = create_app() @@ -57,7 +55,6 @@ def fetch_acme_cert(id): "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } pending_certs = pending_certificate_service.get_pending_certs([id]) - user = user_service.get_by_username('lemur') new = 0 failed = 0 wrong_issuer = 0 @@ -78,12 +75,22 @@ def fetch_acme_cert(id): real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) - + if not pending_cert: + log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?" + current_app.logger.error(log_data) + continue if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(pending_cert, real_cert, user) - pending_certificate_service.delete_by_id(pending_cert.id) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user) + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved=True + ) + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -97,7 +104,11 @@ def fetch_acme_cert(id): if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) + # Mark the pending cert as resolved + pending_certificate_service.update( + cert.get("pending_cert").id, + resolved=True + ) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( @@ -124,12 +135,30 @@ def fetch_acme_cert(id): @celery.task() def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" - pending_certs = pending_certificate_service.get_pending_certs('all') + pending_certs = pending_certificate_service.get_unresolved_pending_certs() # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': - if cert.last_updated == cert.date_created or datetime.datetime.now( - timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3): + if cert.last_updated == cert.date_created or datetime.now( + timezone.utc) - cert.last_updated > timedelta(minutes=3): fetch_acme_cert.delay(cert.id) + + +@celery.task() +def remove_old_acme_certs(): + """Prune old pending acme certificates from the database""" + log_data = { + "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) + } + pending_certs = pending_certificate_service.get_pending_certs('all') + + # Delete pending certs more than a week old + for cert in pending_certs: + if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7): + log_data['pending_cert_id'] = cert.id + log_data['pending_cert_name'] = cert.name + log_data['message'] = "Deleting pending certificate" + current_app.logger.debug(log_data) + pending_certificate_service.delete(cert.id) diff --git a/lemur/domains/models.py b/lemur/domains/models.py index afd348d6..d0c7fef4 100644 --- a/lemur/domains/models.py +++ b/lemur/domains/models.py @@ -15,7 +15,7 @@ from lemur.database import db class Domain(db.Model): __tablename__ = 'domains' id = Column(Integer, primary_key=True) - name = Column(String(256)) + name = Column(String(256), index=True) sensitive = Column(Boolean, default=False) def __repr__(self): diff --git a/lemur/migrations/versions/984178255c83_.py b/lemur/migrations/versions/984178255c83_.py new file mode 100644 index 00000000..40d2ce31 --- /dev/null +++ b/lemur/migrations/versions/984178255c83_.py @@ -0,0 +1,24 @@ +"""Add status to pending certificate, and store resolved cert id + +Revision ID: 984178255c83 +Revises: f2383bf08fbc +Create Date: 2018-10-11 20:49:12.704563 + +""" + +# revision identifiers, used by Alembic. +revision = '984178255c83' +down_revision = 'f2383bf08fbc' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('pending_certs', sa.Column('resolved', sa.Boolean(), nullable=True)) + op.add_column('pending_certs', sa.Column('resolved_cert_id', sa.Integer(), nullable=True)) + + +def downgrade(): + op.drop_column('pending_certs', 'resolved_cert_id') + op.drop_column('pending_certs', 'resolved') diff --git a/lemur/migrations/versions/c87cb989af04_.py b/lemur/migrations/versions/c87cb989af04_.py new file mode 100644 index 00000000..4959e727 --- /dev/null +++ b/lemur/migrations/versions/c87cb989af04_.py @@ -0,0 +1,19 @@ +"""Create an index on the domains table for the domain name +Revision ID: c87cb989af04 +Revises: 9392b9f9a805 +Create Date: 2018-10-11 09:44:57.099854 + +""" + +revision = 'c87cb989af04' +down_revision = '9392b9f9a805' + +from alembic import op + + +def upgrade(): + op.create_index(op.f('ix_domains_name'), 'domains', ['name'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_domains_name'), table_name='domains') diff --git a/lemur/migrations/versions/f2383bf08fbc_.py b/lemur/migrations/versions/f2383bf08fbc_.py new file mode 100644 index 00000000..1fa36960 --- /dev/null +++ b/lemur/migrations/versions/f2383bf08fbc_.py @@ -0,0 +1,23 @@ +"""Create index on certificates table for id desc + + +Revision ID: f2383bf08fbc +Revises: c87cb989af04 +Create Date: 2018-10-11 11:23:31.195471 + +""" + +revision = 'f2383bf08fbc' +down_revision = 'c87cb989af04' + +import sqlalchemy as sa +from alembic import op + + +def upgrade(): + op.create_index('ix_certificates_id_desc', 'certificates', [sa.text('id DESC')], unique=True, + postgresql_using='btree') + + +def downgrade(): + op.drop_index('ix_certificates_id_desc', table_name='certificates') diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index cbce700d..ccad8de5 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -15,7 +15,6 @@ from lemur.authorities.service import get as get_authority from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.users import service as user_service manager = Manager(usage="Handles pending certificate related tasks.") @@ -23,14 +22,14 @@ manager = Manager(usage="Handles pending certificate related tasks.") @manager.option('-i', dest='ids', action='append', help='IDs of pending certificates to fetch') def fetch(ids): """ - Attempt to get full certificates for each pending certificate listed. + Attempt to get full certificate for each pending certificate listed. Args: ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI) `python manager.py pending_certs fetch -i 123 321 all` """ pending_certs = pending_certificate_service.get_pending_certs(ids) - user = user_service.get_by_username('lemur') + new = 0 failed = 0 @@ -38,10 +37,17 @@ def fetch(ids): authority = plugins.get(cert.authority.plugin_name) real_cert = authority.get_ordered_certificate(cert) if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(cert, real_cert, user) - pending_certificate_service.delete(cert) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user) + pending_certificate_service.update( + cert.id, + resolved=True + ) + pending_certificate_service.update( + cert.id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -66,8 +72,7 @@ def fetch_all_acme(): log_data = { "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) } - pending_certs = pending_certificate_service.get_pending_certs('all') - user = user_service.get_by_username('lemur') + pending_certs = pending_certificate_service.get_unresolved_pending_certs() new = 0 failed = 0 wrong_issuer = 0 @@ -90,10 +95,17 @@ def fetch_all_acme(): pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) if real_cert: - # If a real certificate was returned from issuer, then create it in Lemur and delete - # the pending certificate - pending_certificate_service.create_certificate(pending_cert, real_cert, user) - pending_certificate_service.delete_by_id(pending_cert.id) + # If a real certificate was returned from issuer, then create it in Lemur and mark + # the pending certificate as resolved + final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user) + pending_certificate_service.update( + pending_cert.id, + resolved=True + ) + pending_certificate_service.update( + pending_cert.id, + resolved_cert_id=final_cert.id + ) # add metrics to metrics extension new += 1 else: @@ -105,9 +117,13 @@ def fetch_all_acme(): error_log["cn"] = pending_cert.cn if pending_cert.number_attempts > 4: - error_log["message"] = "Deleting pending certificate" + error_log["message"] = "Marking pending certificate as resolved" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert)) + # Mark "resolved" as True + pending_certificate_service.update( + cert.id, + resolved=True + ) else: pending_certificate_service.increment_attempt(pending_cert) pending_certificate_service.update( diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index a1f61fa1..1261177d 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -29,6 +29,8 @@ class PendingCertificate(db.Model): notify = Column(Boolean, default=True) number_attempts = Column(Integer) rename = Column(Boolean, default=True) + resolved = Column(Boolean, default=False) + resolved_cert_id = Column(Integer, nullable=True) cn = Column(String(128)) csr = Column(Text(), nullable=False) diff --git a/lemur/pending_certificates/schemas.py b/lemur/pending_certificates/schemas.py index 252cd892..fbc94f4e 100644 --- a/lemur/pending_certificates/schemas.py +++ b/lemur/pending_certificates/schemas.py @@ -37,6 +37,8 @@ class PendingCertificateOutputSchema(LemurOutputSchema): number_attempts = fields.Integer() date_created = fields.Date() last_updated = fields.Date() + resolved = fields.Boolean(required=False) + resolved_cert_id = fields.Integer(required=False) rotation = fields.Boolean() diff --git a/lemur/pending_certificates/service.py b/lemur/pending_certificates/service.py index 64c03c46..405b2c4b 100644 --- a/lemur/pending_certificates/service.py +++ b/lemur/pending_certificates/service.py @@ -4,25 +4,21 @@ .. moduleauthor:: James Chuong """ import arrow - from sqlalchemy import or_, cast, Integer from lemur import database -from lemur.common.utils import truthiness -from lemur.plugins.base import plugins - -from lemur.roles.models import Role -from lemur.domains.models import Domain from lemur.authorities.models import Authority +from lemur.certificates import service as certificate_service +from lemur.certificates.schemas import CertificateUploadInputSchema +from lemur.common.utils import truthiness from lemur.destinations.models import Destination +from lemur.domains.models import Domain from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate - -from lemur.certificates import service as certificate_service +from lemur.plugins.base import plugins +from lemur.roles.models import Role from lemur.users import service as user_service -from lemur.certificates.schemas import CertificateUploadInputSchema - def get(pending_cert_id): """ @@ -63,6 +59,15 @@ def delete_by_id(id): database.delete(get(id)) +def get_unresolved_pending_certs(): + """ + Retrieve a list of unresolved pending certs given a list of ids + Filters out non-existing pending certs + """ + query = database.session_query(PendingCertificate).filter(PendingCertificate.resolved.is_(False)) + return database.find_all(query, PendingCertificate, {}).all() + + def get_pending_certs(pending_ids): """ Retrieve a list of pending certs given a list of ids @@ -116,6 +121,7 @@ def create_certificate(pending_certificate, certificate, user): # If generating name from certificate, remove the one from pending certificate del data['name'] data['creator'] = creator + cert = certificate_service.import_certificate(**data) database.update(cert) return cert @@ -172,8 +178,8 @@ def render(args): if 'issuer' in terms: # we can't rely on issuer being correct in the cert directly so we combine queries - sub_query = database.session_query(Authority.id)\ - .filter(Authority.name.ilike('%{0}%'.format(terms[1])))\ + sub_query = database.session_query(Authority.id) \ + .filter(Authority.name.ilike('%{0}%'.format(terms[1]))) \ .subquery() query = query.filter( @@ -221,4 +227,6 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now) + # Only show unresolved certificates in the UI + query = query.filter(PendingCertificate.resolved.is_(False)) return database.sort_and_page(query, PendingCertificate, args) diff --git a/requirements-dev.txt b/requirements-dev.txt index eb027066..279724af 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,26 +5,35 @@ # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # aspy.yaml==1.1.1 # via pre-commit +bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.8.24 # via requests +cffi==1.11.5 # via cmarkgfm cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests +cmarkgfm==0.4.2 # via readme-renderer +docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.1.6 # via pre-commit +future==0.16.0 # via readme-renderer +identify==1.1.7 # via pre-commit idna==2.7 # via requests invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine -pre-commit==1.11.0 +pre-commit==1.11.2 pycodestyle==2.3.1 # via flake8 +pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 +pygments==2.2.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit +readme-renderer==22.0 # via twine requests-toolbelt==0.8.0 # via twine requests==2.19.1 # via requests-toolbelt, twine -six==1.11.0 # via cfgv, pre-commit -toml==0.9.6 # via pre-commit +six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer +toml==0.10.0 # via pre-commit tqdm==4.26.0 # via twine -twine==1.11.0 +twine==1.12.1 urllib3==1.23 # via requests virtualenv==16.0.0 # via pre-commit +webencodings==0.5.1 # via bleach diff --git a/requirements-docs.txt b/requirements-docs.txt index e00030c6..b82b2f23 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -5,7 +5,7 @@ # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # acme==0.27.1 -alabaster==0.7.11 # via sphinx +alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.0 amqp==2.3.2 @@ -54,10 +54,10 @@ lockfile==0.12.2 mako==1.0.7 markupsafe==1.0 marshmallow-sqlalchemy==0.14.1 -marshmallow==2.15.4 +marshmallow==2.15.5 mock==2.0.0 ndg-httpsclient==0.5.1 -packaging==17.1 # via sphinx +packaging==18.0 # via sphinx paramiko==2.4.1 pbr==4.2.0 pem==18.1.0 @@ -69,7 +69,7 @@ pygments==2.2.0 # via sphinx pyjwt==1.6.4 pynacl==1.2.1 pyopenssl==18.0.0 -pyparsing==2.2.0 # via packaging +pyparsing==2.2.2 # via packaging pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 @@ -83,8 +83,8 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.4.1 -sphinx==1.8.0 +sphinx-rtd-theme==0.4.2 +sphinx==1.8.1 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.4 diff --git a/requirements-tests.txt b/requirements-tests.txt index 2ed9ae2f..d9807539 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,13 +8,13 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.4 # via moto +boto3==1.9.21 # via moto boto==2.49.0 # via moto -botocore==1.12.4 # via boto3, moto, s3transfer +botocore==1.12.21 # via boto3, moto, s3transfer certifi==2018.8.24 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests -click==6.7 # via flask +click==7.0 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.3.1 # via moto @@ -32,22 +32,22 @@ itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto -jsonpickle==0.9.6 # via aws-xray-sdk +jsonpickle==1.0 # via aws-xray-sdk markupsafe==1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.3.0 # via pytest moto==1.3.4 nose==1.3.7 -pbr==4.2.0 # via mock +pbr==4.3.0 # via mock pluggy==0.7.1 # via pytest py==1.6.0 # via pytest pyaml==17.12.1 # via moto -pycparser==2.18 # via cffi +pycparser==2.19 # via cffi pycryptodome==3.6.6 # via python-jose pyflakes==2.0.0 -pytest-flask==0.12.0 +pytest-flask==0.13.0 pytest-mock==1.10.0 -pytest==3.8.0 +pytest==3.8.2 python-dateutil==2.7.3 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.5 # via moto diff --git a/requirements.txt b/requirements.txt index 9b44bbbe..0d093ced 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ celery[redis]==4.2.1 certifi==2018.8.24 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==6.7 # via flask +click==7.0 # via flask cloudflare==2.1.0 cryptography==2.3.1 dnspython3==1.15.0 @@ -51,18 +51,18 @@ lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.0 # via jinja2, mako marshmallow-sqlalchemy==0.14.1 -marshmallow==2.15.5 +marshmallow==2.16.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 -paramiko==2.4.1 -pbr==4.2.0 # via mock -pem==18.1.0 +paramiko==2.4.2 +pbr==4.3.0 # via mock +pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap -pycparser==2.18 # via cffi +pycparser==2.19 # via cffi pyjwt==1.6.4 -pynacl==1.2.1 # via paramiko +pynacl==1.3.0 # via paramiko pyopenssl==18.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore @@ -77,8 +77,8 @@ requests[security]==2.19.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.4 -sqlalchemy==1.2.11 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.33.5 +sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.23 # via requests vine==1.1.4 # via amqp