Add email notifications for CA cert expiry

docs/administration.rst

@@ -348,6 +348,15 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
 
           LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
 
+.. data:: LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS
+    :noindex:
+
+       Notification interval set for CA certificate expiration notifications. If unspecified, the value [365, 180] is used (roughly one year and 6 months).
+
+       ::
+
+          LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS = [365, 180]
+
 
 Celery Options
 ---------------

docs/developer/plugins/index.rst

@@ -215,12 +215,13 @@ Notification
 ------------
 
 Lemur includes the ability to create Email notifications by **default**. These notifications
-currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and
+currently come in the form of expiration and rotation notices for all certificates, expiration notices for CA certificates,
+and ACME certificate creation failure notices. Lemur periodically checks certificate expiration dates and
 determines if a given certificate is eligible for notification. There are currently only two parameters used to
 determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
 of days the current date (UTC) is from that expiration date.
 
-Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable.
+Certificate expiration notifications can also be configured for Slack or AWS SNS. Other notifications are not configurable.
 Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email.
 
 There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for

docs/production/index.rst

@@ -325,7 +325,7 @@ celery tasks or cron jobs that run these commands.
 
 There are currently three commands that could/should be run on a periodic basis:
 
-- `notify`
+- `notify expirations` and `notify authority_expirations`
 - `check_revoked`
 - `sync`
 
@@ -334,13 +334,15 @@ If you are using LetsEncrypt, you must also run the following:
 - `fetch_all_pending_acme_certs`
 - `remove_old_acme_certs`
 
-How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
+How often you run these commands is largely up to the user. `notify` should be run once a day (more often will result in
+duplicate notifications). `check_revoked` is typically run at least once a day.
 `sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
 `remove_old_acme_certs` can be ran more rarely, such as once every week.
 
 Example cron entries::
 
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
+    0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
     */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
     0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
 
@@ -382,6 +384,20 @@ Example Celery configuration (To be placed in your configuration file)::
                 'expires': 180
             },
             'schedule': crontab(hour="*"),
+        },
+        'notify_expirations': {
+            'task': 'lemur.common.celery.notify_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
+        },
+        'notify_authority_expirations': {
+            'task': 'lemur.common.celery.notify_authority_expirations',
+            'options': {
+                'expires': 180
+            },
+            'schedule': crontab(hour=22, minute=0),
         }
     }
 

lemur/common/celery.py

@@ -820,6 +820,42 @@ def notify_expirations():
     return log_data
 
 
+@celery.task(soft_time_limit=3600)
+def notify_authority_expirations():
+    """
+    This celery task notifies about expiring certificate authority certs
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "notify for certificate authority cert expiration",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_notification.authority_expirations()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Notify expiring CA Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data
+
+
 @celery.task(soft_time_limit=3600)
 def enable_autorotate_for_certs_attached_to_endpoint():
     """

lemur/notifications/cli.py

@@ -10,6 +10,7 @@ from flask_script import Manager
 from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
 from lemur.extensions import sentry, metrics
 from lemur.notifications.messaging import send_expiration_notifications
+from lemur.notifications.messaging import send_authority_expiration_notifications
 
 manager = Manager(usage="Handles notification related tasks.")
 
@@ -24,7 +25,7 @@ manager = Manager(usage="Handles notification related tasks.")
 )
 def expirations(exclude):
     """
-    Runs Lemur's notification engine, that looks for expired certificates and sends
+    Runs Lemur's notification engine, that looks for expiring certificates and sends
     notifications out to those that have subscribed to them.
 
     Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur
@@ -50,3 +51,29 @@ def expirations(exclude):
     metrics.send(
         "expiration_notification_job", "counter", 1, metric_tags={"status": status}
     )
+
+
+def authority_expirations():
+    """
+    Runs Lemur's notification engine, that looks for expiring certificate authority certificates and sends
+    notifications out to the security team and owner.
+
+    :return:
+    """
+    status = FAILURE_METRIC_STATUS
+    try:
+        print("Starting to notify subscribers about expiring certificate authority certificates!")
+        success, failed = send_authority_expiration_notifications()
+        print(
+            "Finished notifying subscribers about expiring certificate authority certificates! "
+            "Sent: {success} Failed: {failed}".format(
+                success=success, failed=failed
+            )
+        )
+        status = SUCCESS_METRIC_STATUS
+    except Exception as e:
+        sentry.captureException()
+
+    metrics.send(
+        "authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
+    )

lemur/notifications/messaging.py

@@ -62,6 +62,34 @@ def get_certificates(exclude=None):
     return certs
 
 
+def get_expiring_authority_certificates():
+    """
+    Finds all certificate authority certificates that are eligible for expiration notifications.
+    :return:
+    """
+    now = arrow.utcnow()
+    authority_expiration_intervals = current_app.config.get("LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS",
+                                                            [365, 180])
+    max_not_after = now + timedelta(days=max(authority_expiration_intervals) + 1)
+
+    q = (
+        database.db.session.query(Certificate)
+        .filter(Certificate.not_after < max_not_after)
+        .filter(Certificate.notify == true())
+        .filter(Certificate.expired == false())
+        .filter(Certificate.revoked == false())
+        .filter(Certificate.root_authority_id.isnot(None))
+        .filter(Certificate.authority_id.is_(None))
+    )
+
+    certs = []
+    for c in windowed_query(q, Certificate.id, 10000):
+        days_remaining = (c.not_after - now).days
+        if days_remaining in authority_expiration_intervals:
+            certs.append(c)
+    return certs
+
+
 def get_eligible_certificates(exclude=None):
     """
     Finds all certificates that are eligible for certificate expiration notification.
@@ -90,6 +118,25 @@ def get_eligible_certificates(exclude=None):
     return certificates
 
 
+def get_eligible_authority_certificates():
+    """
+    Finds all certificate authority certificates that are eligible for certificate expiration notification.
+    Returns the set of all eligible CA certificates, grouped by owner and interval, with a list of applicable certs.
+    :return:
+    """
+    certificates = defaultdict(dict)
+    all_certs = get_expiring_authority_certificates()
+    now = arrow.utcnow()
+
+    # group by owner
+    for owner, owner_certs in groupby(all_certs, lambda x: x.owner):
+        # group by expiration interval
+        for interval, interval_certs in groupby(owner_certs, lambda x: (x.not_after - now).days):
+            certificates[owner][interval] = list(interval_certs)  # list(c for c in interval_certs)
+
+    return certificates
+
+
 def send_plugin_notification(event_type, data, recipients, notification):
     """
     Executes the plugin and handles failure.
@@ -176,6 +223,38 @@ def send_expiration_notifications(exclude):
     return success, failure
 
 
+def send_authority_expiration_notifications():
+    """
+    This function will check for upcoming certificate authority certificate expiration,
+    and send out notification emails at configured intervals.
+    """
+    success = failure = 0
+
+    # security team gets all
+    security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
+
+    for owner, owner_cert_groups in get_eligible_authority_certificates().items():
+        for interval, certificates in owner_cert_groups.items():
+            notification_data = []
+
+            for certificate in certificates:
+                cert_data = certificate_notification_output_schema.dump(
+                    certificate
+                ).data
+                notification_data.append(cert_data)
+
+            email_recipients = security_email + [owner]
+            if send_default_notification(
+                    "authority_expiration", notification_data, email_recipients,
+                    notification_options=[{'name': 'interval', 'value': interval}]
+            ):
+                success = len(email_recipients)
+            else:
+                failure = len(email_recipients)
+
+    return success, failure
+
+
 def send_default_notification(notification_type, data, targets, notification_options=None):
     """
     Sends a report to the specified target via the default notification plugin. Applicable for any notification_type.

lemur/plugins/lemur_email/templates/authority_expiration.html

@@ -0,0 +1,172 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+        "http://www.w3.org/TR/html4/loose.dtd">
+<html lang="en">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta name="viewport" content="initial-scale=1.0">    <!-- So that mobile webkit will display zoomed in -->
+    <meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
+
+    <title>Lemur</title>
+</head>
+
+<div style="margin:0;padding:0" bgcolor="#FFFFFF">
+    <table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
+        <tbody>
+        <tr height="32px"></tr>
+        <tr align="center">
+            <td width="32px"></td>
+            <td>
+                <table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
+                    <tbody>
+                    <tr>
+                        <td>
+                            <table width="100%" border="0" cellspacing="0" cellpadding="0">
+                                <tbody>
+                                <tr>
+                                    <td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
+                                        Lemur
+                                    </td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr height="16"></tr>
+                    <tr>
+                        <td>
+                            <table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
+                                   style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
+                                <tbody>
+                                <tr>
+                                    <td height="72px" colspan="3"></td>
+                                </tr>
+                                <tr>
+                                    <td width="32px"></td>
+                                    <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
+                                       Your CA certificate(s) are expiring in {{ message.options | interval }} days!
+                                    </td>
+                                    <td width="32px"></td>
+                                </tr>
+                                <tr>
+                                    <td height="18px" colspan="3"></td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr>
+                        <td>
+                            <table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
+                                   style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
+                                <tbody>
+                                <tr height="16px">
+                                    <td width="32px" rowspan="3"></td>
+                                    <td></td>
+                                    <td width="32px" rowspan="3"></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
+                                            <tbody>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    Hi,
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    <br>This is a Lemur CA certificate expiration notice. The follow CA certificates are expiring soon;
+                                                    please take manual action to renew them if necessary. You may also disable notifications via the
+                                                    Notify toggle in Lemur if they are no longer in use.
+                                                    <table border="0" cellspacing="0" cellpadding="0"
+                                                           style="margin-top:48px;margin-bottom:48px">
+                                                        <tbody>
+                                                        {%  for certificate in message.certificates %}
+                                                            <tr valign="middle">
+                                                                <td width="32px"></td>
+                                                                <td width="16px"></td>
+                                                                <td style="line-height:1.2">
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.name }}</span>
+                                                                    <br>
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
+                                                                        {{ certificate.endpoints | length }} Endpoints
+                                                                        <br>{{ certificate.owner }}
+                                                                        <br>{{ certificate.validityEnd | time }}
+                                                                        <a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
+                                                                    </span>
+                                                                </td>
+                                                            </tr>
+                                                                {%  if not loop.last %}
+                                                                <tr valign="middle">
+                                                                    <td width="32px" height="24px"></td>
+                                                                </tr>
+                                                                {% endif %}
+                                                        {%  endfor %}
+                                                        </tbody>
+                                                    </table>
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    Your action is required if the above CA certificates are still needed.
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    <br>Best,<br><span class="il">Lemur</span>
+                                                </td>
+                                            </tr>
+                                            <tr height="16px"></tr>
+                                            <tr>
+                                                <td>
+                                                    <table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
+                                                        <tbody>
+                                                        <tr>
+                                                            <td>*All expiration times are in UTC<br></td>
+                                                        </tr>
+                                                        </tbody>
+                                                    </table>
+                                                </td>
+                                            </tr>
+                                            </tbody>
+                                        </table>
+                                    </td>
+                                </tr>
+                                <tr height="32px"></tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr height="16"></tr>
+                    <tr>
+                        <td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
+                    </tr>
+                    <tr>
+                        <td>
+                            <table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
+                                <tbody>
+                                <tr>
+                                    <td>You received this mandatory email announcement to update you about
+                                        important changes to your <span class="il">TLS certificate</span>.
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span></div>
+                                    </td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    </tbody>
+                </table>
+            </td>
+            <td width="32px"></td>
+        </tr>
+        <tr height="32px"></tr>
+        </tbody>
+    </table>
+</div>

lemur/tests/test_messaging.py

@@ -5,6 +5,7 @@ import boto3
 import pytest
 from freezegun import freeze_time
 from moto import mock_ses
+from lemur.tests.factories import AuthorityFactory, CertificateFactory
 
 
 @mock_ses
@@ -125,3 +126,47 @@ def test_send_pending_failure_notification(notification_plugin, async_issuer_plu
     verify_sender_email()
 
     assert send_pending_failure_notification(pending_certificate)
+
+
+def test_get_authority_certificates():
+    from lemur.notifications.messaging import get_expiring_authority_certificates
+
+    certificate_1 = create_cert_that_expires_in_days(180)
+    certificate_2 = create_cert_that_expires_in_days(365)
+    create_cert_that_expires_in_days(364)
+    create_cert_that_expires_in_days(366)
+    create_cert_that_expires_in_days(179)
+    create_cert_that_expires_in_days(181)
+    create_cert_that_expires_in_days(1)
+
+    assert set(get_expiring_authority_certificates()) == {certificate_1, certificate_2}
+
+
+@mock_ses
+def test_send_authority_expiration_notifications():
+    from lemur.notifications.messaging import send_authority_expiration_notifications
+    verify_sender_email()
+
+    create_cert_that_expires_in_days(180)
+    create_cert_that_expires_in_days(180)  # two on the same day results in a single email
+    create_cert_that_expires_in_days(365)
+    create_cert_that_expires_in_days(364)
+    create_cert_that_expires_in_days(366)
+    create_cert_that_expires_in_days(179)
+    create_cert_that_expires_in_days(181)
+    create_cert_that_expires_in_days(1)
+
+    assert send_authority_expiration_notifications() == (2, 0)
+
+
+def create_cert_that_expires_in_days(days):
+    now = arrow.utcnow()
+    not_after = now + timedelta(days=days, hours=1)  # a bit more than specified since we'll check in the future
+
+    authority = AuthorityFactory()
+    certificate = CertificateFactory()
+    certificate.not_after = not_after
+    certificate.notify = True
+    certificate.root_authority_id = authority.id
+    certificate.authority_id = None
+    return certificate