Add expiration summary email for security team

This commit is contained in:
Jasmine Schladen 2020-12-08 11:41:41 -08:00
parent 0ebaa78915
commit eab5532397
8 changed files with 421 additions and 4 deletions

View File

@ -287,6 +287,7 @@ Supported types:
* CA certificate expiration * CA certificate expiration
* Pending ACME certificate failure * Pending ACME certificate failure
* Certificate rotation * Certificate rotation
* Security certificate expiration summary
**Default notifications** **Default notifications**
@ -358,6 +359,18 @@ Whenever a cert is rotated, Lemur will send a notification via email to the cert
disabled by default; to enable it, you must set the option ``--notify`` (when using cron) or the configuration parameter disabled by default; to enable it, you must set the option ``--notify`` (when using cron) or the configuration parameter
``ENABLE_ROTATION_NOTIFICATION`` (when using celery). ``ENABLE_ROTATION_NOTIFICATION`` (when using celery).
**Security certificate expiration summary**
If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all
certificates with upcoming expiration date that matches one of the intervals configured in the
``LEMUR_SECURITY_TEAM_EMAIL_INTERVALS`` configuration parameter (with the same fallbacks as noted above).
Note that certificates will be included in this summary even if they do not have any associated notifications.
This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails.
NOTE: At present, this summary email essentially duplicates the certificate expiration notifications, since all
certificate expiration notifications are also sent to the security team. This issue will be fixed in the future.
**Email notifications** **Email notifications**
Templates for emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs. Templates for emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.

View File

@ -54,7 +54,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi
* pip * pip
* virtualenv (ideally virtualenvwrapper) * virtualenv (ideally virtualenvwrapper)
* node.js (for npm and building css/javascript) * node.js (for npm and building css/javascript)
+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_ * `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
Once you've got all that, the rest is simple: Once you've got all that, the rest is simple:

View File

@ -323,9 +323,9 @@ Periodic Tasks
Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
celery tasks or cron jobs that run these commands. celery tasks or cron jobs that run these commands.
There are currently three commands that could/should be run on a periodic basis: The following commands that could/should be run on a periodic basis:
- `notify expirations` and `notify authority_expirations` (see :ref:`NotificationOptions` for configuration info) - `notify expirations` `notify authority_expirations`, and `notify security_expiration_summary` (see :ref:`NotificationOptions` for configuration info)
- `check_revoked` - `check_revoked`
- `sync` - `sync`
@ -343,6 +343,7 @@ Example cron entries::
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify security_expiration_summary
*/15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
@ -398,6 +399,13 @@ Example Celery configuration (To be placed in your configuration file)::
'expires': 180 'expires': 180
}, },
'schedule': crontab(hour=22, minute=0), 'schedule': crontab(hour=22, minute=0),
},
'send_security_expiration_summary': {
'task': 'lemur.common.celery.send_security_expiration_summary',
'options': {
'expires': 180
},
'schedule': crontab(hour=22, minute=0),
} }
} }

View File

@ -857,6 +857,42 @@ def notify_authority_expirations():
return log_data return log_data
@celery.task(soft_time_limit=3600)
def send_security_expiration_summary():
"""
This celery task sends a summary about expiring certificates to the security team. TODO document
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "send summary for certificate expiration",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_notification.send_security_expiration_summary()
except SoftTimeLimitExceeded:
log_data["message"] = "Send summary for expiring certs Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
metrics.send(f"{function}.success", "counter", 1)
return log_data
@celery.task(soft_time_limit=3600) @celery.task(soft_time_limit=3600)
def enable_autorotate_for_certs_attached_to_endpoint(): def enable_autorotate_for_certs_attached_to_endpoint():
""" """

View File

@ -11,6 +11,7 @@ from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.extensions import sentry, metrics from lemur.extensions import sentry, metrics
from lemur.notifications.messaging import send_expiration_notifications from lemur.notifications.messaging import send_expiration_notifications
from lemur.notifications.messaging import send_authority_expiration_notifications from lemur.notifications.messaging import send_authority_expiration_notifications
from lemur.notifications.messaging import send_security_expiration_summary
manager = Manager(usage="Handles notification related tasks.") manager = Manager(usage="Handles notification related tasks.")
@ -73,3 +74,26 @@ def authority_expirations():
metrics.send( metrics.send(
"authority_expiration_notification_job", "counter", 1, metric_tags={"status": status} "authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
) )
def security_expiration_summary(exclude):
"""
Sends a summary email with info on all expiring certs (that match the configured expiry intervals).
:return:
"""
status = FAILURE_METRIC_STATUS
try:
print("Starting to notify security team about expiring certificates!")
success, failed = send_security_expiration_summary(exclude)
print(
"Finished notifying security team about expiring certificates! "
f"Sent: {success} Failed: {failed}"
)
status = SUCCESS_METRIC_STATUS
except Exception:
sentry.captureException()
metrics.send(
"security_expiration_notification_job", "counter", 1, metric_tags={"status": status}
)

View File

@ -63,6 +63,44 @@ def get_certificates(exclude=None):
return certs return certs
def get_certificates_for_security_summary_email(exclude=None):
"""
Finds all certificates that are eligible for expiration notifications for the security expiration summary.
:param exclude:
:return:
"""
now = arrow.utcnow()
expiration_summary_intervals = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL_INTERVALS", # first priority
current_app.config.get( # second priority
"LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS",
[30, 15, 2] # third priority
)
)
max_not_after = now + timedelta(days=max(expiration_summary_intervals) + 1)
q = (
database.db.session.query(Certificate)
.filter(Certificate.not_after <= max_not_after)
.filter(Certificate.notify == true())
.filter(Certificate.expired == false())
.filter(Certificate.revoked == false())
)
exclude_conditions = []
if exclude:
for e in exclude:
exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e)))
q = q.filter(and_(*exclude_conditions))
certs = []
for c in windowed_query(q, Certificate.id, 10000):
days_remaining = (c.not_after - now).days
if days_remaining in expiration_summary_intervals:
certs.append(c)
return certs
def get_expiring_authority_certificates(): def get_expiring_authority_certificates():
""" """
Finds all certificate authority certificates that are eligible for expiration notifications. Finds all certificate authority certificates that are eligible for expiration notifications.
@ -119,6 +157,18 @@ def get_eligible_certificates(exclude=None):
return certificates return certificates
def get_eligible_security_summary_certs(exclude=None):
certificates = defaultdict(list)
all_certs = get_certificates_for_security_summary_email(exclude=exclude)
now = arrow.utcnow()
# group by expiration interval
for interval, interval_certs in groupby(all_certs, lambda x: (x.not_after - now).days):
certificates[interval] = list(interval_certs)
return certificates
def get_eligible_authority_certificates(): def get_eligible_authority_certificates():
""" """
Finds all certificate authority certificates that are eligible for certificate expiration notification. Finds all certificate authority certificates that are eligible for certificate expiration notification.
@ -370,3 +420,59 @@ def needs_notification(certificate):
if days == interval: if days == interval:
notifications.append(notification) notifications.append(notification)
return notifications return notifications
def send_security_expiration_summary(exclude=None):
"""
Sends a report to the security team with a summary of all expiring certificates.
All expiring certificates are included here, regardless of notification configuration.
Certificates with notifications disabled are omitted.
:param exclude:
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
status = FAILURE_METRIC_STATUS
notification_plugin = plugins.get(
current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
)
notification_type = "expiration_summary"
log_data = {
"function": function,
"message": f"Sending expiration summary notification for to security team",
"notification_type": notification_type,
"notification_plugin": notification_plugin.slug,
}
intervals_and_certs = get_eligible_security_summary_certs(exclude)
security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
try:
current_app.logger.debug(log_data)
message_data = []
for interval, certs in intervals_and_certs.items():
cert_data = []
for certificate in certs:
cert_data.append(certificate_notification_output_schema.dump(certificate).data)
interval_data = {"interval": interval, "certificates": cert_data}
message_data.append(interval_data)
notification_plugin.send(notification_type, message_data, security_email, None)
status = SUCCESS_METRIC_STATUS
except Exception:
log_data["message"] = f"Unable to send {notification_type} notification for certificates " \
f"{intervals_and_certs} to targets {security_email}"
current_app.logger.error(log_data, exc_info=True)
sentry.captureException()
metrics.send(
"notification",
"counter",
1,
metric_tags={"status": status, "event_type": notification_type, "plugin": notification_plugin.slug},
)
if status == SUCCESS_METRIC_STATUS:
return True

View File

@ -0,0 +1,194 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
<title>Lemur</title>
</head>
<div style="margin:0;padding:0" bgcolor="#FFFFFF">
<table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr height="32px"></tr>
<tr align="center">
<td width="32px"></td>
<td>
<table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
<tbody>
<tr>
<td>
<table width="100%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td align="left"
style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
Lemur
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td>
<table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
<tbody>
<tr>
<td height="72px" colspan="3"></td>
</tr>
<tr>
<td width="32px"></td>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
Lemur certificate expiration summary
</td>
<td width="32px"></td>
</tr>
<tr>
<td height="18px" colspan="3"></td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td>
<table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
<tbody>
<tr height="16px">
<td width="32px" rowspan="3"></td>
<td></td>
<td width="32px" rowspan="3"></td>
</tr>
<tr>
<td>
<table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Hi,
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>This is a summary of all certificates expiring soon.
Only certificates matching the configured expiration intervals are included here.
Certificates with notifications disabled have been omitted.
<table border="0" cellspacing="0" cellpadding="0" style="margin-top:12px;margin-bottom:48px">
<tbody>
{% for interval_and_certs in message["certificates"] | sort(attribute="interval") %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#202020">
<br>Expiring in {{ interval_and_certs["interval"] }} days<br>
</span>
</td>
</tr>
<tr valign="middle">
<td width="32px" height="12"></td>
</tr>
{% for certificate in interval_and_certs["certificates"] %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:16px;color:#202020">
{{ certificate.name }}
</span>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
{% if certificate.endpoints | length > 0 %} <!-- highlight in red if > 0 -->
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#F44336">
<br>{{ certificate.endpoints | length }} Endpoints
</span>
{% else %}
<br>{{ certificate.endpoints | length }} Endpoints
{% endif %}
<br>{{ certificate.owner }}
<br>{{ certificate.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
</span>
</td>
</tr>
{% if not loop.last %}
<tr valign="middle">
<td width="32px" height="24px"></td>
</tr>
{% endif %}
{% endfor %}
{% endfor %}
</tbody>
</table>
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Please take action if any of the above certificates are still needed.
</td>
</tr>
<tr>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>Best,<br><span class="il">Lemur</span>
</td>
</tr>
<tr height="16px"></tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
<tbody>
<tr>
<td>*All expiration times are in UTC<br></td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
</tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
<tbody>
<tr>
<td>You received this mandatory email announcement to update you about
important changes to your <span class="il">TLS certificate</span>.
</td>
</tr>
<tr>
<td>
<div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span>
</div>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
<td width="32px"></td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</div>

View File

@ -5,7 +5,7 @@ import boto3
import pytest import pytest
from freezegun import freeze_time from freezegun import freeze_time
from moto import mock_ses from moto import mock_ses
from lemur.tests.factories import AuthorityFactory, CertificateFactory from lemur.tests.factories import AuthorityFactory, CertificateFactory, EndpointFactory
@mock_ses @mock_ses
@ -112,6 +112,26 @@ def test_send_expiration_notification_with_no_notifications(
assert send_expiration_notifications([]) == (0, 0) assert send_expiration_notifications([]) == (0, 0)
@mock_ses
def test_send_expiration_summary_notification(certificate, notification, notification_plugin):
from lemur.notifications.messaging import send_security_expiration_summary
verify_sender_email()
# we don't actually test the email contents, but adding an assortment of certs here is useful for step debugging
# to confirm the produced email body looks like we expect
create_cert_that_expires_in_days(2)
create_cert_that_expires_in_days(2)
create_cert_that_expires_in_days(2)
create_cert_that_expires_in_days(30)
create_cert_that_expires_in_days(30)
create_cert_that_expires_in_days(15)
create_cert_that_expires_in_days(20)
create_cert_that_expires_in_days(1)
create_cert_that_expires_in_days(100)
assert send_security_expiration_summary([])
@mock_ses @mock_ses
def test_send_rotation_notification(notification_plugin, certificate): def test_send_rotation_notification(notification_plugin, certificate):
from lemur.notifications.messaging import send_rotation_notification from lemur.notifications.messaging import send_rotation_notification
@ -170,3 +190,19 @@ def create_ca_cert_that_expires_in_days(days):
certificate.root_authority_id = authority.id certificate.root_authority_id = authority.id
certificate.authority_id = None certificate.authority_id = None
return certificate return certificate
def create_cert_that_expires_in_days(days):
from random import randrange
now = arrow.utcnow()
not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future
certificate = CertificateFactory()
certificate.not_after = not_after
certificate.notify = True
endpoints = []
for i in range(0, randrange(0, 5)):
endpoints.append(EndpointFactory())
certificate.endpoints = endpoints
return certificate