Add email notifications for CA cert expiry

This commit is contained in:
Jasmine Schladen 2020-12-02 09:20:09 -08:00
parent cbdaa4e3e4
commit 85d99ded73
8 changed files with 390 additions and 5 deletions

View File

@ -348,6 +348,15 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2] LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
.. data:: LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS
:noindex:
Notification interval set for CA certificate expiration notifications. If unspecified, the value [365, 180] is used (roughly one year and 6 months).
::
LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS = [365, 180]
Celery Options Celery Options
--------------- ---------------

View File

@ -215,12 +215,13 @@ Notification
------------ ------------
Lemur includes the ability to create Email notifications by **default**. These notifications Lemur includes the ability to create Email notifications by **default**. These notifications
currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and currently come in the form of expiration and rotation notices for all certificates, expiration notices for CA certificates,
and ACME certificate creation failure notices. Lemur periodically checks certificate expiration dates and
determines if a given certificate is eligible for notification. There are currently only two parameters used to determines if a given certificate is eligible for notification. There are currently only two parameters used to
determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
of days the current date (UTC) is from that expiration date. of days the current date (UTC) is from that expiration date.
Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable. Certificate expiration notifications can also be configured for Slack or AWS SNS. Other notifications are not configurable.
Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email. Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email.
There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for

View File

@ -325,7 +325,7 @@ celery tasks or cron jobs that run these commands.
There are currently three commands that could/should be run on a periodic basis: There are currently three commands that could/should be run on a periodic basis:
- `notify` - `notify expirations` and `notify authority_expirations`
- `check_revoked` - `check_revoked`
- `sync` - `sync`
@ -334,13 +334,15 @@ If you are using LetsEncrypt, you must also run the following:
- `fetch_all_pending_acme_certs` - `fetch_all_pending_acme_certs`
- `remove_old_acme_certs` - `remove_old_acme_certs`
How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day. How often you run these commands is largely up to the user. `notify` should be run once a day (more often will result in
duplicate notifications). `check_revoked` is typically run at least once a day.
`sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine). `sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
`remove_old_acme_certs` can be ran more rarely, such as once every week. `remove_old_acme_certs` can be ran more rarely, such as once every week.
Example cron entries:: Example cron entries::
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
*/15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
@ -382,6 +384,20 @@ Example Celery configuration (To be placed in your configuration file)::
'expires': 180 'expires': 180
}, },
'schedule': crontab(hour="*"), 'schedule': crontab(hour="*"),
},
'notify_expirations': {
'task': 'lemur.common.celery.notify_expirations',
'options': {
'expires': 180
},
'schedule': crontab(hour=22, minute=0),
},
'notify_authority_expirations': {
'task': 'lemur.common.celery.notify_authority_expirations',
'options': {
'expires': 180
},
'schedule': crontab(hour=22, minute=0),
} }
} }

View File

@ -820,6 +820,42 @@ def notify_expirations():
return log_data return log_data
@celery.task(soft_time_limit=3600)
def notify_authority_expirations():
"""
This celery task notifies about expiring certificate authority certs
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "notify for certificate authority cert expiration",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_notification.authority_expirations()
except SoftTimeLimitExceeded:
log_data["message"] = "Notify expiring CA Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
metrics.send(f"{function}.success", "counter", 1)
return log_data
@celery.task(soft_time_limit=3600) @celery.task(soft_time_limit=3600)
def enable_autorotate_for_certs_attached_to_endpoint(): def enable_autorotate_for_certs_attached_to_endpoint():
""" """

View File

@ -10,6 +10,7 @@ from flask_script import Manager
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.extensions import sentry, metrics from lemur.extensions import sentry, metrics
from lemur.notifications.messaging import send_expiration_notifications from lemur.notifications.messaging import send_expiration_notifications
from lemur.notifications.messaging import send_authority_expiration_notifications
manager = Manager(usage="Handles notification related tasks.") manager = Manager(usage="Handles notification related tasks.")
@ -24,7 +25,7 @@ manager = Manager(usage="Handles notification related tasks.")
) )
def expirations(exclude): def expirations(exclude):
""" """
Runs Lemur's notification engine, that looks for expired certificates and sends Runs Lemur's notification engine, that looks for expiring certificates and sends
notifications out to those that have subscribed to them. notifications out to those that have subscribed to them.
Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur
@ -50,3 +51,29 @@ def expirations(exclude):
metrics.send( metrics.send(
"expiration_notification_job", "counter", 1, metric_tags={"status": status} "expiration_notification_job", "counter", 1, metric_tags={"status": status}
) )
def authority_expirations():
"""
Runs Lemur's notification engine, that looks for expiring certificate authority certificates and sends
notifications out to the security team and owner.
:return:
"""
status = FAILURE_METRIC_STATUS
try:
print("Starting to notify subscribers about expiring certificate authority certificates!")
success, failed = send_authority_expiration_notifications()
print(
"Finished notifying subscribers about expiring certificate authority certificates! "
"Sent: {success} Failed: {failed}".format(
success=success, failed=failed
)
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException()
metrics.send(
"authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
)

View File

@ -62,6 +62,34 @@ def get_certificates(exclude=None):
return certs return certs
def get_expiring_authority_certificates():
"""
Finds all certificate authority certificates that are eligible for expiration notifications.
:return:
"""
now = arrow.utcnow()
authority_expiration_intervals = current_app.config.get("LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS",
[365, 180])
max_not_after = now + timedelta(days=max(authority_expiration_intervals) + 1)
q = (
database.db.session.query(Certificate)
.filter(Certificate.not_after < max_not_after)
.filter(Certificate.notify == true())
.filter(Certificate.expired == false())
.filter(Certificate.revoked == false())
.filter(Certificate.root_authority_id.isnot(None))
.filter(Certificate.authority_id.is_(None))
)
certs = []
for c in windowed_query(q, Certificate.id, 10000):
days_remaining = (c.not_after - now).days
if days_remaining in authority_expiration_intervals:
certs.append(c)
return certs
def get_eligible_certificates(exclude=None): def get_eligible_certificates(exclude=None):
""" """
Finds all certificates that are eligible for certificate expiration notification. Finds all certificates that are eligible for certificate expiration notification.
@ -90,6 +118,25 @@ def get_eligible_certificates(exclude=None):
return certificates return certificates
def get_eligible_authority_certificates():
"""
Finds all certificate authority certificates that are eligible for certificate expiration notification.
Returns the set of all eligible CA certificates, grouped by owner and interval, with a list of applicable certs.
:return:
"""
certificates = defaultdict(dict)
all_certs = get_expiring_authority_certificates()
now = arrow.utcnow()
# group by owner
for owner, owner_certs in groupby(all_certs, lambda x: x.owner):
# group by expiration interval
for interval, interval_certs in groupby(owner_certs, lambda x: (x.not_after - now).days):
certificates[owner][interval] = list(interval_certs) # list(c for c in interval_certs)
return certificates
def send_plugin_notification(event_type, data, recipients, notification): def send_plugin_notification(event_type, data, recipients, notification):
""" """
Executes the plugin and handles failure. Executes the plugin and handles failure.
@ -176,6 +223,38 @@ def send_expiration_notifications(exclude):
return success, failure return success, failure
def send_authority_expiration_notifications():
"""
This function will check for upcoming certificate authority certificate expiration,
and send out notification emails at configured intervals.
"""
success = failure = 0
# security team gets all
security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
for owner, owner_cert_groups in get_eligible_authority_certificates().items():
for interval, certificates in owner_cert_groups.items():
notification_data = []
for certificate in certificates:
cert_data = certificate_notification_output_schema.dump(
certificate
).data
notification_data.append(cert_data)
email_recipients = security_email + [owner]
if send_default_notification(
"authority_expiration", notification_data, email_recipients,
notification_options=[{'name': 'interval', 'value': interval}]
):
success = len(email_recipients)
else:
failure = len(email_recipients)
return success, failure
def send_default_notification(notification_type, data, targets, notification_options=None): def send_default_notification(notification_type, data, targets, notification_options=None):
""" """
Sends a report to the specified target via the default notification plugin. Applicable for any notification_type. Sends a report to the specified target via the default notification plugin. Applicable for any notification_type.

View File

@ -0,0 +1,172 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
<title>Lemur</title>
</head>
<div style="margin:0;padding:0" bgcolor="#FFFFFF">
<table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr height="32px"></tr>
<tr align="center">
<td width="32px"></td>
<td>
<table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
<tbody>
<tr>
<td>
<table width="100%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
Lemur
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td>
<table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
<tbody>
<tr>
<td height="72px" colspan="3"></td>
</tr>
<tr>
<td width="32px"></td>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
Your CA certificate(s) are expiring in {{ message.options | interval }} days!
</td>
<td width="32px"></td>
</tr>
<tr>
<td height="18px" colspan="3"></td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td>
<table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
<tbody>
<tr height="16px">
<td width="32px" rowspan="3"></td>
<td></td>
<td width="32px" rowspan="3"></td>
</tr>
<tr>
<td>
<table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Hi,
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>This is a Lemur CA certificate expiration notice. The follow CA certificates are expiring soon;
please take manual action to renew them if necessary. You may also disable notifications via the
Notify toggle in Lemur if they are no longer in use.
<table border="0" cellspacing="0" cellpadding="0"
style="margin-top:48px;margin-bottom:48px">
<tbody>
{% for certificate in message.certificates %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.name }}</span>
<br>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
{{ certificate.endpoints | length }} Endpoints
<br>{{ certificate.owner }}
<br>{{ certificate.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
</span>
</td>
</tr>
{% if not loop.last %}
<tr valign="middle">
<td width="32px" height="24px"></td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Your action is required if the above CA certificates are still needed.
</td>
</tr>
<tr>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>Best,<br><span class="il">Lemur</span>
</td>
</tr>
<tr height="16px"></tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
<tbody>
<tr>
<td>*All expiration times are in UTC<br></td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
</tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
<tbody>
<tr>
<td>You received this mandatory email announcement to update you about
important changes to your <span class="il">TLS certificate</span>.
</td>
</tr>
<tr>
<td>
<div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span></div>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
<td width="32px"></td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</div>

View File

@ -5,6 +5,7 @@ import boto3
import pytest import pytest
from freezegun import freeze_time from freezegun import freeze_time
from moto import mock_ses from moto import mock_ses
from lemur.tests.factories import AuthorityFactory, CertificateFactory
@mock_ses @mock_ses
@ -125,3 +126,47 @@ def test_send_pending_failure_notification(notification_plugin, async_issuer_plu
verify_sender_email() verify_sender_email()
assert send_pending_failure_notification(pending_certificate) assert send_pending_failure_notification(pending_certificate)
def test_get_authority_certificates():
from lemur.notifications.messaging import get_expiring_authority_certificates
certificate_1 = create_cert_that_expires_in_days(180)
certificate_2 = create_cert_that_expires_in_days(365)
create_cert_that_expires_in_days(364)
create_cert_that_expires_in_days(366)
create_cert_that_expires_in_days(179)
create_cert_that_expires_in_days(181)
create_cert_that_expires_in_days(1)
assert set(get_expiring_authority_certificates()) == {certificate_1, certificate_2}
@mock_ses
def test_send_authority_expiration_notifications():
from lemur.notifications.messaging import send_authority_expiration_notifications
verify_sender_email()
create_cert_that_expires_in_days(180)
create_cert_that_expires_in_days(180) # two on the same day results in a single email
create_cert_that_expires_in_days(365)
create_cert_that_expires_in_days(364)
create_cert_that_expires_in_days(366)
create_cert_that_expires_in_days(179)
create_cert_that_expires_in_days(181)
create_cert_that_expires_in_days(1)
assert send_authority_expiration_notifications() == (2, 0)
def create_cert_that_expires_in_days(days):
now = arrow.utcnow()
not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future
authority = AuthorityFactory()
certificate = CertificateFactory()
certificate.not_after = not_after
certificate.notify = True
certificate.root_authority_id = authority.id
certificate.authority_id = None
return certificate