diff --git a/docker-compose.yml b/docker-compose.yml index 20e3e649..66f2f0b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,3 +17,8 @@ services: environment: POSTGRES_USER: lemur POSTGRES_PASSWORD: lemur + + redis: + image: "redis:alpine" + ports: + - "6379:6379" diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 5e0c42cc..bc6dbad1 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -360,6 +360,7 @@ def update_destinations(target, value, initiator): status = SUCCESS_METRIC_STATUS except Exception as e: sentry.captureException() + raise metrics.send('destination_upload', 'counter', 1, metric_tags={'status': status, 'certificate': target.name, 'destination': value.label}) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index c8a5365b..b8016dd4 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -106,7 +106,7 @@ def get_all_pending_cleaning(source): :return: """ return Certificate.query.filter(Certificate.sources.any(id=source.id)) \ - .filter(not_(Certificate.endpoints.any())).all() + .filter(not_(Certificate.endpoints.any())).filter(Certificate.expired).all() def get_all_pending_reissue(): diff --git a/lemur/common/celery.py b/lemur/common/celery.py index d3986351..8dbb6c29 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -19,6 +19,7 @@ from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins +from lemur.sources.cli import clean, validate_sources flask_app = create_app() @@ -142,7 +143,7 @@ def fetch_all_pending_acme_certs(): cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': if cert.last_updated == cert.date_created or datetime.now( - timezone.utc) - cert.last_updated > timedelta(minutes=3): + timezone.utc) - cert.last_updated > timedelta(minutes=5): fetch_acme_cert.delay(cert.id) @@ -162,3 +163,28 @@ def remove_old_acme_certs(): log_data['message'] = "Deleting pending certificate" current_app.logger.debug(log_data) pending_certificate_service.delete(cert.id) + + +@celery.task() +def clean_all_sources(): + """ + This function will clean unused certificates from sources. This is a destructive operation and should only + be ran periodically. This function triggers one celery task per source. + """ + sources = validate_sources("all") + for source in sources: + current_app.logger.debug("Creating celery task to clean source {}".format(source.label)) + clean_source.delay(source.label) + + +@celery.task() +def clean_source(source): + """ + This celery task will clean the specified source. This is a destructive operation that will delete unused + certificates from each source. + + :param source: + :return: + """ + current_app.logger.debug("Cleaning source {}".format(source)) + clean([source], True) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 677728bb..b2a07798 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -66,7 +66,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): """ client = kwargs.pop('client') - if not path: + if not path or path == '/': path = '/' else: name = name + '-' + path.strip('/') diff --git a/requirements-dev.txt b/requirements-dev.txt index 6322e122..c473aa56 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,7 +32,7 @@ requests-toolbelt==0.8.0 # via twine requests==2.20.0 # via requests-toolbelt, twine six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.27.0 # via twine +tqdm==4.28.1 # via twine twine==1.12.1 urllib3==1.24 # via requests virtualenv==16.0.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index a7cfcdd9..6b49b64d 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -90,7 +90,7 @@ sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.6 sqlalchemy==1.2.12 tabulate==0.8.2 -urllib3==1.23 +urllib3==1.24 vine==1.1.4 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 504ef083..a851e620 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -46,7 +46,7 @@ pyaml==17.12.1 # via moto pycparser==2.19 # via cffi pycryptodome==3.6.6 # via python-jose pyflakes==2.0.0 -pytest-flask==0.13.0 +pytest-flask==0.14.0 pytest-mock==1.10.0 pytest==3.9.1 python-dateutil==2.7.3 # via botocore, faker, freezegun, moto