Merge pull request #3296 from jtschladen/expiration-summary-email
Expiration summary email for security team
This commit is contained in:
Jasmine Schladen
GitHub
commit
f941383cd2
|
|
@ -287,6 +287,7 @@ Supported types:
|
|||
* CA certificate expiration
|
||||
* Pending ACME certificate failure
|
||||
* Certificate rotation
|
||||
* Security certificate expiration summary
|
||||
|
||||
**Default notifications**
|
||||
|
||||
|
|
@ -358,6 +359,18 @@ Whenever a cert is rotated, Lemur will send a notification via email to the cert
|
|||
disabled by default; to enable it, you must set the option ``--notify`` (when using cron) or the configuration parameter
|
||||
``ENABLE_ROTATION_NOTIFICATION`` (when using celery).
|
||||
|
||||
**Security certificate expiration summary**
|
||||
|
||||
If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all
|
||||
certificates with upcoming expiration date that occurs within the number of days specified by the
|
||||
``LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS`` configuration parameter (with a fallback of 14 days).
|
||||
Note that certificates will be included in this summary even if they do not have any associated notifications.
|
||||
|
||||
This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails.
|
||||
|
||||
NOTE: At present, this summary email essentially duplicates the certificate expiration notifications, since all
|
||||
certificate expiration notifications are also sent to the security team. This issue will be fixed in the future.
|
||||
|
||||
**Email notifications**
|
||||
|
||||
Templates for emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs.
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi
|
|||
* pip
|
||||
* virtualenv (ideally virtualenvwrapper)
|
||||
* node.js (for npm and building css/javascript)
|
||||
+* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
|
||||
* `PostgreSQL <https://lemur.readthedocs.io/en/latest/quickstart/index.html#setup-postgres>`_
|
||||
|
||||
Once you've got all that, the rest is simple:
|
||||
|
||||
|
|
|
|||
|
|
@ -323,9 +323,9 @@ Periodic Tasks
|
|||
Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create
|
||||
celery tasks or cron jobs that run these commands.
|
||||
|
||||
There are currently three commands that could/should be run on a periodic basis:
|
||||
The following commands that could/should be run on a periodic basis:
|
||||
|
||||
- `notify expirations` and `notify authority_expirations` (see :ref:`NotificationOptions` for configuration info)
|
||||
- `notify expirations` `notify authority_expirations`, and `notify security_expiration_summary` (see :ref:`NotificationOptions` for configuration info)
|
||||
- `check_revoked`
|
||||
- `sync`
|
||||
|
||||
|
|
@ -343,6 +343,7 @@ Example cron entries::
|
|||
|
||||
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
|
||||
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
|
||||
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify security_expiration_summary
|
||||
*/15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
|
||||
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
|
||||
|
||||
|
|
@ -398,6 +399,13 @@ Example Celery configuration (To be placed in your configuration file)::
|
|||
'expires': 180
|
||||
},
|
||||
'schedule': crontab(hour=22, minute=0),
|
||||
},
|
||||
'send_security_expiration_summary': {
|
||||
'task': 'lemur.common.celery.send_security_expiration_summary',
|
||||
'options': {
|
||||
'expires': 180
|
||||
},
|
||||
'schedule': crontab(hour=22, minute=0),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -857,6 +857,42 @@ def notify_authority_expirations():
|
|||
return log_data
|
||||
|
||||
|
||||
@celery.task(soft_time_limit=3600)
|
||||
def send_security_expiration_summary():
|
||||
"""
|
||||
This celery task sends a summary about expiring certificates to the security team.
|
||||
:return:
|
||||
"""
|
||||
function = f"{__name__}.{sys._getframe().f_code.co_name}"
|
||||
task_id = None
|
||||
if celery.current_task:
|
||||
task_id = celery.current_task.request.id
|
||||
|
||||
log_data = {
|
||||
"function": function,
|
||||
"message": "send summary for certificate expiration",
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
if task_id and is_task_active(function, task_id, None):
|
||||
log_data["message"] = "Skipping task: Task is already active"
|
||||
current_app.logger.debug(log_data)
|
||||
return
|
||||
|
||||
current_app.logger.debug(log_data)
|
||||
try:
|
||||
cli_notification.security_expiration_summary(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
|
||||
except SoftTimeLimitExceeded:
|
||||
log_data["message"] = "Send summary for expiring certs Time limit exceeded."
|
||||
current_app.logger.error(log_data)
|
||||
sentry.captureException()
|
||||
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
|
||||
return
|
||||
|
||||
metrics.send(f"{function}.success", "counter", 1)
|
||||
return log_data
|
||||
|
||||
|
||||
@celery.task(soft_time_limit=3600)
|
||||
def enable_autorotate_for_certs_attached_to_endpoint():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
|
|||
from lemur.extensions import sentry, metrics
|
||||
from lemur.notifications.messaging import send_expiration_notifications
|
||||
from lemur.notifications.messaging import send_authority_expiration_notifications
|
||||
from lemur.notifications.messaging import send_security_expiration_summary
|
||||
|
||||
manager = Manager(usage="Handles notification related tasks.")
|
||||
|
||||
|
|
@ -73,3 +74,26 @@ def authority_expirations():
|
|||
metrics.send(
|
||||
"authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
|
||||
)
|
||||
|
||||
|
||||
def security_expiration_summary(exclude):
|
||||
"""
|
||||
Sends a summary email with info on all expiring certs (that match the configured expiry intervals).
|
||||
|
||||
:return:
|
||||
"""
|
||||
status = FAILURE_METRIC_STATUS
|
||||
try:
|
||||
print("Starting to notify security team about expiring certificates!")
|
||||
success, failed = send_security_expiration_summary(exclude)
|
||||
print(
|
||||
"Finished notifying security team about expiring certificates! "
|
||||
f"Sent: {success} Failed: {failed}"
|
||||
)
|
||||
status = SUCCESS_METRIC_STATUS
|
||||
except Exception:
|
||||
sentry.captureException()
|
||||
|
||||
metrics.send(
|
||||
"security_expiration_notification_job", "counter", 1, metric_tags={"status": status}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -63,6 +63,39 @@ def get_certificates(exclude=None):
|
|||
return certs
|
||||
|
||||
|
||||
def get_certificates_for_security_summary_email(exclude=None):
|
||||
"""
|
||||
Finds all certificates that are eligible for expiration notifications for the security expiration summary.
|
||||
:param exclude:
|
||||
:return:
|
||||
"""
|
||||
now = arrow.utcnow()
|
||||
threshold_days = current_app.config.get("LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS", 14)
|
||||
max_not_after = now + timedelta(days=threshold_days + 1)
|
||||
|
||||
q = (
|
||||
database.db.session.query(Certificate)
|
||||
.filter(Certificate.not_after <= max_not_after)
|
||||
.filter(Certificate.notify == true())
|
||||
.filter(Certificate.expired == false())
|
||||
.filter(Certificate.revoked == false())
|
||||
)
|
||||
|
||||
exclude_conditions = []
|
||||
if exclude:
|
||||
for e in exclude:
|
||||
exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e)))
|
||||
|
||||
q = q.filter(and_(*exclude_conditions))
|
||||
|
||||
certs = []
|
||||
for c in windowed_query(q, Certificate.id, 10000):
|
||||
days_remaining = (c.not_after - now).days
|
||||
if days_remaining <= threshold_days:
|
||||
certs.append(c)
|
||||
return certs
|
||||
|
||||
|
||||
def get_expiring_authority_certificates():
|
||||
"""
|
||||
Finds all certificate authority certificates that are eligible for expiration notifications.
|
||||
|
|
@ -119,6 +152,18 @@ def get_eligible_certificates(exclude=None):
|
|||
return certificates
|
||||
|
||||
|
||||
def get_eligible_security_summary_certs(exclude=None):
|
||||
certificates = defaultdict(list)
|
||||
all_certs = get_certificates_for_security_summary_email(exclude=exclude)
|
||||
now = arrow.utcnow()
|
||||
|
||||
# group by expiration interval
|
||||
for interval, interval_certs in groupby(all_certs, lambda x: (x.not_after - now).days):
|
||||
certificates[interval] = list(interval_certs)
|
||||
|
||||
return certificates
|
||||
|
||||
|
||||
def get_eligible_authority_certificates():
|
||||
"""
|
||||
Finds all certificate authority certificates that are eligible for certificate expiration notification.
|
||||
|
|
@ -370,3 +415,59 @@ def needs_notification(certificate):
|
|||
if days == interval:
|
||||
notifications.append(notification)
|
||||
return notifications
|
||||
|
||||
|
||||
def send_security_expiration_summary(exclude=None):
|
||||
"""
|
||||
Sends a report to the security team with a summary of all expiring certificates.
|
||||
All expiring certificates are included here, regardless of notification configuration.
|
||||
Certificates with notifications disabled are omitted.
|
||||
|
||||
:param exclude:
|
||||
:return:
|
||||
"""
|
||||
function = f"{__name__}.{sys._getframe().f_code.co_name}"
|
||||
status = FAILURE_METRIC_STATUS
|
||||
notification_plugin = plugins.get(
|
||||
current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
|
||||
)
|
||||
notification_type = "expiration_summary"
|
||||
log_data = {
|
||||
"function": function,
|
||||
"message": "Sending expiration summary notification for to security team",
|
||||
"notification_type": notification_type,
|
||||
"notification_plugin": notification_plugin.slug,
|
||||
}
|
||||
|
||||
intervals_and_certs = get_eligible_security_summary_certs(exclude)
|
||||
security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
|
||||
|
||||
try:
|
||||
current_app.logger.debug(log_data)
|
||||
|
||||
message_data = []
|
||||
|
||||
for interval, certs in intervals_and_certs.items():
|
||||
cert_data = []
|
||||
for certificate in certs:
|
||||
cert_data.append(certificate_notification_output_schema.dump(certificate).data)
|
||||
interval_data = {"interval": interval, "certificates": cert_data}
|
||||
message_data.append(interval_data)
|
||||
|
||||
notification_plugin.send(notification_type, message_data, security_email, None)
|
||||
status = SUCCESS_METRIC_STATUS
|
||||
except Exception:
|
||||
log_data["message"] = f"Unable to send {notification_type} notification for certificates " \
|
||||
f"{intervals_and_certs} to targets {security_email}"
|
||||
current_app.logger.error(log_data, exc_info=True)
|
||||
sentry.captureException()
|
||||
|
||||
metrics.send(
|
||||
"notification",
|
||||
"counter",
|
||||
1,
|
||||
metric_tags={"status": status, "event_type": notification_type, "plugin": notification_plugin.slug},
|
||||
)
|
||||
|
||||
if status == SUCCESS_METRIC_STATUS:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
|
||||
"http://www.w3.org/TR/html4/loose.dtd">
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
|
||||
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
|
||||
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
|
||||
|
||||
<title>Lemur</title>
|
||||
</head>
|
||||
|
||||
<div style="margin:0;padding:0" bgcolor="#FFFFFF">
|
||||
<table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr height="32px"></tr>
|
||||
<tr align="center">
|
||||
<td width="32px"></td>
|
||||
<td>
|
||||
<table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<table width="100%" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td align="left"
|
||||
style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
|
||||
Lemur
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="16"></tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
|
||||
style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td height="72px" colspan="3"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="32px"></td>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
|
||||
Lemur certificate expiration summary
|
||||
</td>
|
||||
<td width="32px"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td height="18px" colspan="3"></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
|
||||
style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
|
||||
<tbody>
|
||||
<tr height="16px">
|
||||
<td width="32px" rowspan="3"></td>
|
||||
<td></td>
|
||||
<td width="32px" rowspan="3"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
Hi,
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
<br>This is a summary of all certificates expiring soon.
|
||||
Only certificates matching the configured expiration intervals are included here.
|
||||
Certificates with notifications disabled have been omitted.
|
||||
<table border="0" cellspacing="0" cellpadding="0" style="margin-top:12px;margin-bottom:48px">
|
||||
<tbody>
|
||||
{% for interval_and_certs in message["certificates"] | sort(attribute="interval") %}
|
||||
<tr valign="middle">
|
||||
<td width="32px"></td>
|
||||
<td width="16px"></td>
|
||||
<td style="line-height:1.2">
|
||||
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#202020">
|
||||
<br>Expiring in {{ interval_and_certs["interval"] }} days<br>
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr valign="middle">
|
||||
<td width="32px" height="12"></td>
|
||||
</tr>
|
||||
{% for certificate in interval_and_certs["certificates"] %}
|
||||
<tr valign="middle">
|
||||
<td width="32px"></td>
|
||||
<td width="16px"></td>
|
||||
<td style="line-height:1.2">
|
||||
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:16px;color:#202020">
|
||||
{{ certificate.name }}
|
||||
</span>
|
||||
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
|
||||
{% if certificate.endpoints | length > 0 %} <!-- highlight in red if > 0 -->
|
||||
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#F44336">
|
||||
<br>{{ certificate.endpoints | length }} Endpoints
|
||||
</span>
|
||||
{% else %}
|
||||
<br>{{ certificate.endpoints | length }} Endpoints
|
||||
{% endif %}
|
||||
<br>{{ certificate.owner }}
|
||||
<br>{{ certificate.validityEnd | time }}
|
||||
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
{% if not loop.last %}
|
||||
<tr valign="middle">
|
||||
<td width="32px" height="24px"></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
Please take action if any of the above certificates are still needed.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
<br>Best,<br><span class="il">Lemur</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="16px"></tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>*All expiration times are in UTC<br></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="32px"></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="16"></tr>
|
||||
<tr>
|
||||
<td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>You received this mandatory email announcement to update you about
|
||||
important changes to your <span class="il">TLS certificate</span>.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
<td width="32px"></td>
|
||||
</tr>
|
||||
<tr height="32px"></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
|
@ -5,7 +5,7 @@ import boto3
|
|||
import pytest
|
||||
from freezegun import freeze_time
|
||||
from moto import mock_ses
|
||||
from lemur.tests.factories import AuthorityFactory, CertificateFactory
|
||||
from lemur.tests.factories import AuthorityFactory, CertificateFactory, EndpointFactory
|
||||
|
||||
|
||||
@mock_ses
|
||||
|
|
@ -112,6 +112,28 @@ def test_send_expiration_notification_with_no_notifications(
|
|||
assert send_expiration_notifications([]) == (0, 0)
|
||||
|
||||
|
||||
@mock_ses
|
||||
def test_send_expiration_summary_notification(certificate, notification, notification_plugin):
|
||||
from lemur.notifications.messaging import send_security_expiration_summary
|
||||
verify_sender_email()
|
||||
|
||||
# we don't actually test the email contents, but adding an assortment of certs here is useful for step debugging
|
||||
# to confirm the produced email body looks like we expect
|
||||
create_cert_that_expires_in_days(14)
|
||||
create_cert_that_expires_in_days(12)
|
||||
create_cert_that_expires_in_days(9)
|
||||
create_cert_that_expires_in_days(7)
|
||||
create_cert_that_expires_in_days(7)
|
||||
create_cert_that_expires_in_days(2)
|
||||
create_cert_that_expires_in_days(30)
|
||||
create_cert_that_expires_in_days(15)
|
||||
create_cert_that_expires_in_days(20)
|
||||
create_cert_that_expires_in_days(1)
|
||||
create_cert_that_expires_in_days(100)
|
||||
|
||||
assert send_security_expiration_summary([])
|
||||
|
||||
|
||||
@mock_ses
|
||||
def test_send_rotation_notification(notification_plugin, certificate):
|
||||
from lemur.notifications.messaging import send_rotation_notification
|
||||
|
|
@ -170,3 +192,19 @@ def create_ca_cert_that_expires_in_days(days):
|
|||
certificate.root_authority_id = authority.id
|
||||
certificate.authority_id = None
|
||||
return certificate
|
||||
|
||||
|
||||
def create_cert_that_expires_in_days(days):
|
||||
from random import randrange
|
||||
|
||||
now = arrow.utcnow()
|
||||
not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future
|
||||
|
||||
certificate = CertificateFactory()
|
||||
certificate.not_after = not_after
|
||||
certificate.notify = True
|
||||
endpoints = []
|
||||
for i in range(0, randrange(0, 5)):
|
||||
endpoints.append(EndpointFactory())
|
||||
certificate.endpoints = endpoints
|
||||
return certificate
|
||||
|
|
|
|||
Loading…
Reference in New Issue