From 73ed5164cde151dfa6add34457dcac645e122e5f Mon Sep 17 00:00:00 2001
From: Curtis Castrapel <ccastrapel@gmail.com>
Date: Mon, 22 Oct 2018 14:51:13 -0700
Subject: [PATCH] deps

---
 docker-compose.yml            |  5 +++++
 lemur/certificates/models.py  |  1 +
 lemur/certificates/service.py |  2 +-
 lemur/common/celery.py        | 26 ++++++++++++++++++++++++++
 requirements-dev.txt          |  2 +-
 requirements-docs.txt         |  2 +-
 requirements-tests.txt        |  2 +-
 7 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 20e3e649..66f2f0b1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,3 +17,8 @@ services:
     environment:
       POSTGRES_USER: lemur
       POSTGRES_PASSWORD: lemur
+
+  redis:
+    image: "redis:alpine"
+    ports:
+      - "6379:6379"
diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py
index 5e0c42cc..bc6dbad1 100644
--- a/lemur/certificates/models.py
+++ b/lemur/certificates/models.py
@@ -360,6 +360,7 @@ def update_destinations(target, value, initiator):
             status = SUCCESS_METRIC_STATUS
     except Exception as e:
         sentry.captureException()
+        raise
 
     metrics.send('destination_upload', 'counter', 1,
                  metric_tags={'status': status, 'certificate': target.name, 'destination': value.label})
diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py
index 0bd50694..fd6142ee 100644
--- a/lemur/certificates/service.py
+++ b/lemur/certificates/service.py
@@ -90,7 +90,7 @@ def get_all_pending_cleaning(source):
     :return:
     """
     return Certificate.query.filter(Certificate.sources.any(id=source.id)) \
-        .filter(not_(Certificate.endpoints.any())).all()
+        .filter(not_(Certificate.endpoints.any())).filter(Certificate.expired).all()
 
 
 def get_all_pending_reissue():
diff --git a/lemur/common/celery.py b/lemur/common/celery.py
index d3986351..74ca7d9f 100644
--- a/lemur/common/celery.py
+++ b/lemur/common/celery.py
@@ -19,6 +19,7 @@ from lemur.factory import create_app
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
+from lemur.sources.cli import clean, validate_sources
 
 flask_app = create_app()
 
@@ -162,3 +163,28 @@ def remove_old_acme_certs():
             log_data['message'] = "Deleting pending certificate"
             current_app.logger.debug(log_data)
             pending_certificate_service.delete(cert.id)
+
+
+@celery.task()
+def clean_all_sources():
+    """
+    This function will clean unused certificates from sources. This is a destructive operation and should only
+    be ran periodically. This function triggers one celery task per source.
+    """
+    sources = validate_sources("all")
+    for source in sources:
+        current_app.logger.debug("Creating celery task to clean source {}".format(source.label))
+        clean_source.delay(source.label)
+
+
+@celery.task()
+def clean_source(source):
+    """
+    This celery task will clean the specified source. This is a destructive operation that will delete unused
+    certificates from each source.
+
+    :param source:
+    :return:
+    """
+    current_app.logger.debug("Cleaning source {}".format(source))
+    clean([source], True)
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 6322e122..c473aa56 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -32,7 +32,7 @@ requests-toolbelt==0.8.0  # via twine
 requests==2.20.0          # via requests-toolbelt, twine
 six==1.11.0               # via bleach, cfgv, pre-commit, readme-renderer
 toml==0.10.0              # via pre-commit
-tqdm==4.27.0              # via twine
+tqdm==4.28.1              # via twine
 twine==1.12.1
 urllib3==1.24             # via requests
 virtualenv==16.0.0        # via pre-commit
diff --git a/requirements-docs.txt b/requirements-docs.txt
index a7cfcdd9..6b49b64d 100644
--- a/requirements-docs.txt
+++ b/requirements-docs.txt
@@ -90,7 +90,7 @@ sphinxcontrib-websupport==1.1.0  # via sphinx
 sqlalchemy-utils==0.33.6
 sqlalchemy==1.2.12
 tabulate==0.8.2
-urllib3==1.23
+urllib3==1.24
 vine==1.1.4
 werkzeug==0.14.1
 xmltodict==0.11.0
diff --git a/requirements-tests.txt b/requirements-tests.txt
index 504ef083..a851e620 100644
--- a/requirements-tests.txt
+++ b/requirements-tests.txt
@@ -46,7 +46,7 @@ pyaml==17.12.1            # via moto
 pycparser==2.19           # via cffi
 pycryptodome==3.6.6       # via python-jose
 pyflakes==2.0.0
-pytest-flask==0.13.0
+pytest-flask==0.14.0
 pytest-mock==1.10.0
 pytest==3.9.1
 python-dateutil==2.7.3    # via botocore, faker, freezegun, moto