Merge branch 'master' into ilabun/optimize-certificates-sql

lemur/certificates/cli.py

@@ -5,29 +5,18 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import sys
 import multiprocessing
-from tabulate import tabulate
-from sqlalchemy import or_
-
+import sys
 from flask import current_app
-
-from flask_script import Manager
 from flask_principal import Identity, identity_changed
-
+from flask_script import Manager
+from sqlalchemy import or_
+from tabulate import tabulate
 
 from lemur import database
-from lemur.extensions import sentry
-from lemur.extensions import metrics
-from lemur.plugins.base import plugins
-from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
-from lemur.deployment import service as deployment_service
-from lemur.endpoints import service as endpoint_service
-from lemur.notifications.messaging import send_rotation_notification
-from lemur.domains.models import Domain
 from lemur.authorities.models import Authority
-from lemur.certificates.schemas import CertificateOutputSchema
 from lemur.certificates.models import Certificate
+from lemur.certificates.schemas import CertificateOutputSchema
 from lemur.certificates.service import (
     reissue_certificate,
     get_certificate_primitives,
@@ -35,9 +24,16 @@ from lemur.certificates.service import (
     get_by_name,
     get_all_certs,
     get,
+    get_all_certs_attached_to_endpoint_without_autorotate,
 )
-
 from lemur.certificates.verify import verify_string
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
+from lemur.deployment import service as deployment_service
+from lemur.domains.models import Domain
+from lemur.endpoints import service as endpoint_service
+from lemur.extensions import sentry, metrics
+from lemur.notifications.messaging import send_rotation_notification
+from lemur.plugins.base import plugins
 
 manager = Manager(usage="Handles all certificate related tasks.")
 
@@ -482,3 +478,38 @@ def check_revoked():
             cert.status = "unknown"
 
         database.update(cert)
+
+
+@manager.command
+def automatically_enable_autorotate():
+    """
+    This function automatically enables auto-rotation for unexpired certificates that are
+    attached to an endpoint but do not have autorotate enabled.
+
+    WARNING: This will overwrite the Auto-rotate toggle!
+    """
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+    }
+
+    permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", [])
+
+    eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate()
+    for cert in eligible_certs:
+
+        if cert.authority_id not in permitted_authorities:
+            continue
+
+        log_data["certificate"] = cert.name
+        log_data["certificate_id"] = cert.id
+        log_data["message"] = "Enabling auto-rotate for certificate"
+        current_app.logger.info(log_data)
+        # TODO:  add the cert destination to the logging
+        metrics.send("automatically_enable_autorotate",
+                     "counter", 1,
+                     metric_tags={"certificate": cert.name,
+                                  "certificate_id": cert.id,
+                                  "authority_id": cert.authority_id,
+                                  "authority_name": Authority.get(cert.authority_id).name})
+        cert.rotation = True
+        database.update(cert)

lemur/certificates/models.py

@@ -321,7 +321,8 @@ class Certificate(db.Model):
 
     @hybrid_property
     def expired(self):
-        if self.not_after <= arrow.utcnow():
+        # can't compare offset-naive and offset-aware datetimes
+        if arrow.Arrow.fromdatetime(self.not_after) <= arrow.utcnow():
             return True
 
     @expired.expression
@@ -445,6 +446,9 @@ def update_destinations(target, value, initiator):
     """
     destination_plugin = plugins.get(value.plugin_name)
     status = FAILURE_METRIC_STATUS
+
+    if target.expired:
+        return
     try:
         if target.private_key or not destination_plugin.requires_key:
             destination_plugin.upload(

lemur/certificates/service.py

@@ -103,12 +103,13 @@ def get_all_certs():
     return Certificate.query.all()
 
 
-def get_all_pending_cleaning(source):
+def get_all_pending_cleaning_expired(source):
     """
-    Retrieves all certificates that are available for cleaning.
+    Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not
+    attached to any endpoints.
 
-    :param source:
-    :return:
+    :param source: the source to search for certificates
+    :return: list of pending certificates
     """
     return (
         Certificate.query.filter(Certificate.sources.any(id=source.id))
@@ -118,6 +119,58 @@ def get_all_pending_cleaning(source):
     )
 
 
+def get_all_certs_attached_to_endpoint_without_autorotate():
+    """
+        Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled.
+
+        :return: list of certificates attached to an endpoint without autorotate
+        """
+    return (
+        Certificate.query.filter(Certificate.endpoints.any())
+        .filter(Certificate.rotation == False)
+        .filter(Certificate.not_after >= arrow.now())
+        .filter(not_(Certificate.replaced.any()))
+        .all()  # noqa
+    )
+
+
+def get_all_pending_cleaning_expiring_in_days(source, days_to_expire):
+    """
+    Retrieves all certificates that are available for cleaning, not attached to endpoint,
+    and within X days from expiration.
+
+    :param days_to_expire: defines how many days till the certificate is expired
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
+    expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD")
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.not_after < expiration_window)
+        .all()
+    )
+
+
+def get_all_pending_cleaning_issued_since_days(source, days_since_issuance):
+    """
+    Retrieves all certificates that are available for cleaning: not attached to endpoint, and X days since issuance.
+
+    :param days_since_issuance: defines how many days since the certificate is issued
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
+    not_in_use_window = (
+        arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD")
+    )
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.date_created > not_in_use_window)
+        .all()
+    )
+
+
 def get_all_pending_reissue():
     """
     Retrieves all certificates that need to be rotated.
@@ -332,9 +385,11 @@ def render(args):
 
     show_expired = args.pop("showExpired")
     if show_expired != 1:
-        one_month_old = arrow.now()\
-            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\
+        one_month_old = (
+            arrow.now()
+            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))
             .format("YYYY-MM-DD")
+        )
         query = query.filter(Certificate.not_after > one_month_old)
 
     time_range = args.pop("time_range")