Add email notifications for CA cert expiry
This commit is contained in:
Jasmine Schladen
parent
cbdaa4e3e4
commit
85d99ded73
|
|
@ -348,6 +348,15 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
|
|||
|
||||
LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
|
||||
|
||||
.. data:: LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS
|
||||
:noindex:
|
||||
|
||||
Notification interval set for CA certificate expiration notifications. If unspecified, the value [365, 180] is used (roughly one year and 6 months).
|
||||
|
||||
::
|
||||
|
||||
LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS = [365, 180]
|
||||
|
||||
|
||||
Celery Options
|
||||
---------------
|
||||
|
|
|
|||
|
|
@ -215,12 +215,13 @@ Notification
|
|||
------------
|
||||
|
||||
Lemur includes the ability to create Email notifications by **default**. These notifications
|
||||
currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and
|
||||
currently come in the form of expiration and rotation notices for all certificates, expiration notices for CA certificates,
|
||||
and ACME certificate creation failure notices. Lemur periodically checks certificate expiration dates and
|
||||
determines if a given certificate is eligible for notification. There are currently only two parameters used to
|
||||
determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
|
||||
of days the current date (UTC) is from that expiration date.
|
||||
|
||||
Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable.
|
||||
Certificate expiration notifications can also be configured for Slack or AWS SNS. Other notifications are not configurable.
|
||||
Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email.
|
||||
|
||||
There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for
|
||||
|
|
|
|||
|
|
@ -325,7 +325,7 @@ celery tasks or cron jobs that run these commands.
|
|||
|
||||
There are currently three commands that could/should be run on a periodic basis:
|
||||
|
||||
- `notify`
|
||||
- `notify expirations` and `notify authority_expirations`
|
||||
- `check_revoked`
|
||||
- `sync`
|
||||
|
||||
|
|
@ -334,13 +334,15 @@ If you are using LetsEncrypt, you must also run the following:
|
|||
- `fetch_all_pending_acme_certs`
|
||||
- `remove_old_acme_certs`
|
||||
|
||||
How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day.
|
||||
How often you run these commands is largely up to the user. `notify` should be run once a day (more often will result in
|
||||
duplicate notifications). `check_revoked` is typically run at least once a day.
|
||||
`sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine).
|
||||
`remove_old_acme_certs` can be ran more rarely, such as once every week.
|
||||
|
||||
Example cron entries::
|
||||
|
||||
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations
|
||||
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations
|
||||
*/15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all
|
||||
0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked
|
||||
|
||||
|
|
@ -382,6 +384,20 @@ Example Celery configuration (To be placed in your configuration file)::
|
|||
'expires': 180
|
||||
},
|
||||
'schedule': crontab(hour="*"),
|
||||
},
|
||||
'notify_expirations': {
|
||||
'task': 'lemur.common.celery.notify_expirations',
|
||||
'options': {
|
||||
'expires': 180
|
||||
},
|
||||
'schedule': crontab(hour=22, minute=0),
|
||||
},
|
||||
'notify_authority_expirations': {
|
||||
'task': 'lemur.common.celery.notify_authority_expirations',
|
||||
'options': {
|
||||
'expires': 180
|
||||
},
|
||||
'schedule': crontab(hour=22, minute=0),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -820,6 +820,42 @@ def notify_expirations():
|
|||
return log_data
|
||||
|
||||
|
||||
@celery.task(soft_time_limit=3600)
|
||||
def notify_authority_expirations():
|
||||
"""
|
||||
This celery task notifies about expiring certificate authority certs
|
||||
:return:
|
||||
"""
|
||||
function = f"{__name__}.{sys._getframe().f_code.co_name}"
|
||||
task_id = None
|
||||
if celery.current_task:
|
||||
task_id = celery.current_task.request.id
|
||||
|
||||
log_data = {
|
||||
"function": function,
|
||||
"message": "notify for certificate authority cert expiration",
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
if task_id and is_task_active(function, task_id, None):
|
||||
log_data["message"] = "Skipping task: Task is already active"
|
||||
current_app.logger.debug(log_data)
|
||||
return
|
||||
|
||||
current_app.logger.debug(log_data)
|
||||
try:
|
||||
cli_notification.authority_expirations()
|
||||
except SoftTimeLimitExceeded:
|
||||
log_data["message"] = "Notify expiring CA Time limit exceeded."
|
||||
current_app.logger.error(log_data)
|
||||
sentry.captureException()
|
||||
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
|
||||
return
|
||||
|
||||
metrics.send(f"{function}.success", "counter", 1)
|
||||
return log_data
|
||||
|
||||
|
||||
@celery.task(soft_time_limit=3600)
|
||||
def enable_autorotate_for_certs_attached_to_endpoint():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from flask_script import Manager
|
|||
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
|
||||
from lemur.extensions import sentry, metrics
|
||||
from lemur.notifications.messaging import send_expiration_notifications
|
||||
from lemur.notifications.messaging import send_authority_expiration_notifications
|
||||
|
||||
manager = Manager(usage="Handles notification related tasks.")
|
||||
|
||||
|
|
@ -24,7 +25,7 @@ manager = Manager(usage="Handles notification related tasks.")
|
|||
)
|
||||
def expirations(exclude):
|
||||
"""
|
||||
Runs Lemur's notification engine, that looks for expired certificates and sends
|
||||
Runs Lemur's notification engine, that looks for expiring certificates and sends
|
||||
notifications out to those that have subscribed to them.
|
||||
|
||||
Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur
|
||||
|
|
@ -50,3 +51,29 @@ def expirations(exclude):
|
|||
metrics.send(
|
||||
"expiration_notification_job", "counter", 1, metric_tags={"status": status}
|
||||
)
|
||||
|
||||
|
||||
def authority_expirations():
|
||||
"""
|
||||
Runs Lemur's notification engine, that looks for expiring certificate authority certificates and sends
|
||||
notifications out to the security team and owner.
|
||||
|
||||
:return:
|
||||
"""
|
||||
status = FAILURE_METRIC_STATUS
|
||||
try:
|
||||
print("Starting to notify subscribers about expiring certificate authority certificates!")
|
||||
success, failed = send_authority_expiration_notifications()
|
||||
print(
|
||||
"Finished notifying subscribers about expiring certificate authority certificates! "
|
||||
"Sent: {success} Failed: {failed}".format(
|
||||
success=success, failed=failed
|
||||
)
|
||||
)
|
||||
status = SUCCESS_METRIC_STATUS
|
||||
except Exception as e:
|
||||
sentry.captureException()
|
||||
|
||||
metrics.send(
|
||||
"authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,34 @@ def get_certificates(exclude=None):
|
|||
return certs
|
||||
|
||||
|
||||
def get_expiring_authority_certificates():
|
||||
"""
|
||||
Finds all certificate authority certificates that are eligible for expiration notifications.
|
||||
:return:
|
||||
"""
|
||||
now = arrow.utcnow()
|
||||
authority_expiration_intervals = current_app.config.get("LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS",
|
||||
[365, 180])
|
||||
max_not_after = now + timedelta(days=max(authority_expiration_intervals) + 1)
|
||||
|
||||
q = (
|
||||
database.db.session.query(Certificate)
|
||||
.filter(Certificate.not_after < max_not_after)
|
||||
.filter(Certificate.notify == true())
|
||||
.filter(Certificate.expired == false())
|
||||
.filter(Certificate.revoked == false())
|
||||
.filter(Certificate.root_authority_id.isnot(None))
|
||||
.filter(Certificate.authority_id.is_(None))
|
||||
)
|
||||
|
||||
certs = []
|
||||
for c in windowed_query(q, Certificate.id, 10000):
|
||||
days_remaining = (c.not_after - now).days
|
||||
if days_remaining in authority_expiration_intervals:
|
||||
certs.append(c)
|
||||
return certs
|
||||
|
||||
|
||||
def get_eligible_certificates(exclude=None):
|
||||
"""
|
||||
Finds all certificates that are eligible for certificate expiration notification.
|
||||
|
|
@ -90,6 +118,25 @@ def get_eligible_certificates(exclude=None):
|
|||
return certificates
|
||||
|
||||
|
||||
def get_eligible_authority_certificates():
|
||||
"""
|
||||
Finds all certificate authority certificates that are eligible for certificate expiration notification.
|
||||
Returns the set of all eligible CA certificates, grouped by owner and interval, with a list of applicable certs.
|
||||
:return:
|
||||
"""
|
||||
certificates = defaultdict(dict)
|
||||
all_certs = get_expiring_authority_certificates()
|
||||
now = arrow.utcnow()
|
||||
|
||||
# group by owner
|
||||
for owner, owner_certs in groupby(all_certs, lambda x: x.owner):
|
||||
# group by expiration interval
|
||||
for interval, interval_certs in groupby(owner_certs, lambda x: (x.not_after - now).days):
|
||||
certificates[owner][interval] = list(interval_certs) # list(c for c in interval_certs)
|
||||
|
||||
return certificates
|
||||
|
||||
|
||||
def send_plugin_notification(event_type, data, recipients, notification):
|
||||
"""
|
||||
Executes the plugin and handles failure.
|
||||
|
|
@ -176,6 +223,38 @@ def send_expiration_notifications(exclude):
|
|||
return success, failure
|
||||
|
||||
|
||||
def send_authority_expiration_notifications():
|
||||
"""
|
||||
This function will check for upcoming certificate authority certificate expiration,
|
||||
and send out notification emails at configured intervals.
|
||||
"""
|
||||
success = failure = 0
|
||||
|
||||
# security team gets all
|
||||
security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
|
||||
|
||||
for owner, owner_cert_groups in get_eligible_authority_certificates().items():
|
||||
for interval, certificates in owner_cert_groups.items():
|
||||
notification_data = []
|
||||
|
||||
for certificate in certificates:
|
||||
cert_data = certificate_notification_output_schema.dump(
|
||||
certificate
|
||||
).data
|
||||
notification_data.append(cert_data)
|
||||
|
||||
email_recipients = security_email + [owner]
|
||||
if send_default_notification(
|
||||
"authority_expiration", notification_data, email_recipients,
|
||||
notification_options=[{'name': 'interval', 'value': interval}]
|
||||
):
|
||||
success = len(email_recipients)
|
||||
else:
|
||||
failure = len(email_recipients)
|
||||
|
||||
return success, failure
|
||||
|
||||
|
||||
def send_default_notification(notification_type, data, targets, notification_options=None):
|
||||
"""
|
||||
Sends a report to the specified target via the default notification plugin. Applicable for any notification_type.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,172 @@
|
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
|
||||
"http://www.w3.org/TR/html4/loose.dtd">
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
|
||||
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
|
||||
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
|
||||
|
||||
<title>Lemur</title>
|
||||
</head>
|
||||
|
||||
<div style="margin:0;padding:0" bgcolor="#FFFFFF">
|
||||
<table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr height="32px"></tr>
|
||||
<tr align="center">
|
||||
<td width="32px"></td>
|
||||
<td>
|
||||
<table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<table width="100%" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
|
||||
Lemur
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="16"></tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
|
||||
style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td height="72px" colspan="3"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="32px"></td>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
|
||||
Your CA certificate(s) are expiring in {{ message.options | interval }} days!
|
||||
</td>
|
||||
<td width="32px"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td height="18px" colspan="3"></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
|
||||
style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
|
||||
<tbody>
|
||||
<tr height="16px">
|
||||
<td width="32px" rowspan="3"></td>
|
||||
<td></td>
|
||||
<td width="32px" rowspan="3"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
Hi,
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
<br>This is a Lemur CA certificate expiration notice. The follow CA certificates are expiring soon;
|
||||
please take manual action to renew them if necessary. You may also disable notifications via the
|
||||
Notify toggle in Lemur if they are no longer in use.
|
||||
<table border="0" cellspacing="0" cellpadding="0"
|
||||
style="margin-top:48px;margin-bottom:48px">
|
||||
<tbody>
|
||||
{% for certificate in message.certificates %}
|
||||
<tr valign="middle">
|
||||
<td width="32px"></td>
|
||||
<td width="16px"></td>
|
||||
<td style="line-height:1.2">
|
||||
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.name }}</span>
|
||||
<br>
|
||||
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
|
||||
{{ certificate.endpoints | length }} Endpoints
|
||||
<br>{{ certificate.owner }}
|
||||
<br>{{ certificate.validityEnd | time }}
|
||||
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
{% if not loop.last %}
|
||||
<tr valign="middle">
|
||||
<td width="32px" height="24px"></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
Your action is required if the above CA certificates are still needed.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
|
||||
<br>Best,<br><span class="il">Lemur</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="16px"></tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>*All expiration times are in UTC<br></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="32px"></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr height="16"></tr>
|
||||
<tr>
|
||||
<td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>You received this mandatory email announcement to update you about
|
||||
important changes to your <span class="il">TLS certificate</span>.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span></div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</td>
|
||||
<td width="32px"></td>
|
||||
</tr>
|
||||
<tr height="32px"></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
|
@ -5,6 +5,7 @@ import boto3
|
|||
import pytest
|
||||
from freezegun import freeze_time
|
||||
from moto import mock_ses
|
||||
from lemur.tests.factories import AuthorityFactory, CertificateFactory
|
||||
|
||||
|
||||
@mock_ses
|
||||
|
|
@ -125,3 +126,47 @@ def test_send_pending_failure_notification(notification_plugin, async_issuer_plu
|
|||
verify_sender_email()
|
||||
|
||||
assert send_pending_failure_notification(pending_certificate)
|
||||
|
||||
|
||||
def test_get_authority_certificates():
|
||||
from lemur.notifications.messaging import get_expiring_authority_certificates
|
||||
|
||||
certificate_1 = create_cert_that_expires_in_days(180)
|
||||
certificate_2 = create_cert_that_expires_in_days(365)
|
||||
create_cert_that_expires_in_days(364)
|
||||
create_cert_that_expires_in_days(366)
|
||||
create_cert_that_expires_in_days(179)
|
||||
create_cert_that_expires_in_days(181)
|
||||
create_cert_that_expires_in_days(1)
|
||||
|
||||
assert set(get_expiring_authority_certificates()) == {certificate_1, certificate_2}
|
||||
|
||||
|
||||
@mock_ses
|
||||
def test_send_authority_expiration_notifications():
|
||||
from lemur.notifications.messaging import send_authority_expiration_notifications
|
||||
verify_sender_email()
|
||||
|
||||
create_cert_that_expires_in_days(180)
|
||||
create_cert_that_expires_in_days(180) # two on the same day results in a single email
|
||||
create_cert_that_expires_in_days(365)
|
||||
create_cert_that_expires_in_days(364)
|
||||
create_cert_that_expires_in_days(366)
|
||||
create_cert_that_expires_in_days(179)
|
||||
create_cert_that_expires_in_days(181)
|
||||
create_cert_that_expires_in_days(1)
|
||||
|
||||
assert send_authority_expiration_notifications() == (2, 0)
|
||||
|
||||
|
||||
def create_cert_that_expires_in_days(days):
|
||||
now = arrow.utcnow()
|
||||
not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future
|
||||
|
||||
authority = AuthorityFactory()
|
||||
certificate = CertificateFactory()
|
||||
certificate.not_after = not_after
|
||||
certificate.notify = True
|
||||
certificate.root_authority_id = authority.id
|
||||
certificate.authority_id = None
|
||||
return certificate
|
||||
|
|
|
|||
Loading…
Reference in New Issue