From eab55323979aec08407838a479442c6acf159d43 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 8 Dec 2020 11:41:41 -0800 Subject: [PATCH 1/4] Add expiration summary email for security team --- docs/administration.rst | 13 ++ docs/developer/index.rst | 2 +- docs/production/index.rst | 12 +- lemur/common/celery.py | 36 ++++ lemur/notifications/cli.py | 24 +++ lemur/notifications/messaging.py | 106 ++++++++++ .../templates/expiration_summary.html | 194 ++++++++++++++++++ lemur/tests/test_messaging.py | 38 +++- 8 files changed, 421 insertions(+), 4 deletions(-) create mode 100644 lemur/plugins/lemur_email/templates/expiration_summary.html diff --git a/docs/administration.rst b/docs/administration.rst index 1415e598..4c755e7b 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -287,6 +287,7 @@ Supported types: * CA certificate expiration * Pending ACME certificate failure * Certificate rotation +* Security certificate expiration summary **Default notifications** @@ -358,6 +359,18 @@ Whenever a cert is rotated, Lemur will send a notification via email to the cert disabled by default; to enable it, you must set the option ``--notify`` (when using cron) or the configuration parameter ``ENABLE_ROTATION_NOTIFICATION`` (when using celery). +**Security certificate expiration summary** + +If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all +certificates with upcoming expiration date that matches one of the intervals configured in the +``LEMUR_SECURITY_TEAM_EMAIL_INTERVALS`` configuration parameter (with the same fallbacks as noted above). +Note that certificates will be included in this summary even if they do not have any associated notifications. + +This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails. + +NOTE: At present, this summary email essentially duplicates the certificate expiration notifications, since all +certificate expiration notifications are also sent to the security team. This issue will be fixed in the future. + **Email notifications** Templates for emails are located under `lemur/plugins/lemur_email/templates` and can be modified for your needs. diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 0033c3f4..bff53a96 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -54,7 +54,7 @@ of Lemur. You'll want to make sure you have a few things on your local system fi * pip * virtualenv (ideally virtualenvwrapper) * node.js (for npm and building css/javascript) -+* `PostgreSQL `_ +* `PostgreSQL `_ Once you've got all that, the rest is simple: diff --git a/docs/production/index.rst b/docs/production/index.rst index 106f6b99..fa0a7dec 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -323,9 +323,9 @@ Periodic Tasks Lemur contains a few tasks that are run and scheduled basis, currently the recommend way to run these tasks is to create celery tasks or cron jobs that run these commands. -There are currently three commands that could/should be run on a periodic basis: +The following commands that could/should be run on a periodic basis: -- `notify expirations` and `notify authority_expirations` (see :ref:`NotificationOptions` for configuration info) +- `notify expirations` `notify authority_expirations`, and `notify security_expiration_summary` (see :ref:`NotificationOptions` for configuration info) - `check_revoked` - `sync` @@ -343,6 +343,7 @@ Example cron entries:: 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations + 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify security_expiration_summary */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked @@ -398,6 +399,13 @@ Example Celery configuration (To be placed in your configuration file):: 'expires': 180 }, 'schedule': crontab(hour=22, minute=0), + }, + 'send_security_expiration_summary': { + 'task': 'lemur.common.celery.send_security_expiration_summary', + 'options': { + 'expires': 180 + }, + 'schedule': crontab(hour=22, minute=0), } } diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 9dc4bd0a..574d6ec5 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -857,6 +857,42 @@ def notify_authority_expirations(): return log_data +@celery.task(soft_time_limit=3600) +def send_security_expiration_summary(): + """ + This celery task sends a summary about expiring certificates to the security team. TODO document + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "message": "send summary for certificate expiration", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + + current_app.logger.debug(log_data) + try: + cli_notification.send_security_expiration_summary() + except SoftTimeLimitExceeded: + log_data["message"] = "Send summary for expiring certs Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) + return + + metrics.send(f"{function}.success", "counter", 1) + return log_data + + @celery.task(soft_time_limit=3600) def enable_autorotate_for_certs_attached_to_endpoint(): """ diff --git a/lemur/notifications/cli.py b/lemur/notifications/cli.py index 3c29693b..d86028c0 100644 --- a/lemur/notifications/cli.py +++ b/lemur/notifications/cli.py @@ -11,6 +11,7 @@ from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS from lemur.extensions import sentry, metrics from lemur.notifications.messaging import send_expiration_notifications from lemur.notifications.messaging import send_authority_expiration_notifications +from lemur.notifications.messaging import send_security_expiration_summary manager = Manager(usage="Handles notification related tasks.") @@ -73,3 +74,26 @@ def authority_expirations(): metrics.send( "authority_expiration_notification_job", "counter", 1, metric_tags={"status": status} ) + + +def security_expiration_summary(exclude): + """ + Sends a summary email with info on all expiring certs (that match the configured expiry intervals). + + :return: + """ + status = FAILURE_METRIC_STATUS + try: + print("Starting to notify security team about expiring certificates!") + success, failed = send_security_expiration_summary(exclude) + print( + "Finished notifying security team about expiring certificates! " + f"Sent: {success} Failed: {failed}" + ) + status = SUCCESS_METRIC_STATUS + except Exception: + sentry.captureException() + + metrics.send( + "security_expiration_notification_job", "counter", 1, metric_tags={"status": status} + ) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 75d829b1..e121e09a 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -63,6 +63,44 @@ def get_certificates(exclude=None): return certs +def get_certificates_for_security_summary_email(exclude=None): + """ + Finds all certificates that are eligible for expiration notifications for the security expiration summary. + :param exclude: + :return: + """ + now = arrow.utcnow() + expiration_summary_intervals = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL_INTERVALS", # first priority + current_app.config.get( # second priority + "LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", + [30, 15, 2] # third priority + ) + ) + max_not_after = now + timedelta(days=max(expiration_summary_intervals) + 1) + + q = ( + database.db.session.query(Certificate) + .filter(Certificate.not_after <= max_not_after) + .filter(Certificate.notify == true()) + .filter(Certificate.expired == false()) + .filter(Certificate.revoked == false()) + ) + + exclude_conditions = [] + if exclude: + for e in exclude: + exclude_conditions.append(~Certificate.name.ilike("%{}%".format(e))) + + q = q.filter(and_(*exclude_conditions)) + + certs = [] + for c in windowed_query(q, Certificate.id, 10000): + days_remaining = (c.not_after - now).days + if days_remaining in expiration_summary_intervals: + certs.append(c) + return certs + + def get_expiring_authority_certificates(): """ Finds all certificate authority certificates that are eligible for expiration notifications. @@ -119,6 +157,18 @@ def get_eligible_certificates(exclude=None): return certificates +def get_eligible_security_summary_certs(exclude=None): + certificates = defaultdict(list) + all_certs = get_certificates_for_security_summary_email(exclude=exclude) + now = arrow.utcnow() + + # group by expiration interval + for interval, interval_certs in groupby(all_certs, lambda x: (x.not_after - now).days): + certificates[interval] = list(interval_certs) + + return certificates + + def get_eligible_authority_certificates(): """ Finds all certificate authority certificates that are eligible for certificate expiration notification. @@ -370,3 +420,59 @@ def needs_notification(certificate): if days == interval: notifications.append(notification) return notifications + + +def send_security_expiration_summary(exclude=None): + """ + Sends a report to the security team with a summary of all expiring certificates. + All expiring certificates are included here, regardless of notification configuration. + Certificates with notifications disabled are omitted. + + :param exclude: + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + status = FAILURE_METRIC_STATUS + notification_plugin = plugins.get( + current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification") + ) + notification_type = "expiration_summary" + log_data = { + "function": function, + "message": f"Sending expiration summary notification for to security team", + "notification_type": notification_type, + "notification_plugin": notification_plugin.slug, + } + + intervals_and_certs = get_eligible_security_summary_certs(exclude) + security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") + + try: + current_app.logger.debug(log_data) + + message_data = [] + + for interval, certs in intervals_and_certs.items(): + cert_data = [] + for certificate in certs: + cert_data.append(certificate_notification_output_schema.dump(certificate).data) + interval_data = {"interval": interval, "certificates": cert_data} + message_data.append(interval_data) + + notification_plugin.send(notification_type, message_data, security_email, None) + status = SUCCESS_METRIC_STATUS + except Exception: + log_data["message"] = f"Unable to send {notification_type} notification for certificates " \ + f"{intervals_and_certs} to targets {security_email}" + current_app.logger.error(log_data, exc_info=True) + sentry.captureException() + + metrics.send( + "notification", + "counter", + 1, + metric_tags={"status": status, "event_type": notification_type, "plugin": notification_plugin.slug}, + ) + + if status == SUCCESS_METRIC_STATUS: + return True diff --git a/lemur/plugins/lemur_email/templates/expiration_summary.html b/lemur/plugins/lemur_email/templates/expiration_summary.html new file mode 100644 index 00000000..d2e98196 --- /dev/null +++ b/lemur/plugins/lemur_email/templates/expiration_summary.html @@ -0,0 +1,194 @@ + + + + + + + + Lemur + + +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ Lemur +
+
+ + + + + + + + + + + + + + +
+ Lemur certificate expiration summary +
+
+ + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ Hi, +
+
This is a summary of all certificates expiring soon. + Only certificates matching the configured expiration intervals are included here. + Certificates with notifications disabled have been omitted. + + + {% for interval_and_certs in message["certificates"] | sort(attribute="interval") %} + + + + + + + + + {% for certificate in interval_and_certs["certificates"] %} + + + + + + {% if not loop.last %} + + + + {% endif %} + {% endfor %} + {% endfor %} + +
+ +
Expiring in {{ interval_and_certs["interval"] }} days
+ +
+ + {{ certificate.name }} + + + {% if certificate.endpoints | length > 0 %} + +
{{ certificate.endpoints | length }} Endpoints + + {% else %} +
{{ certificate.endpoints | length }} Endpoints + {% endif %} +
{{ certificate.owner }} +
{{ certificate.validityEnd | time }} + Details + +
+
+ Please take action if any of the above certificates are still needed. +
+
Best,
Lemur +
+ + + + + + +
*All expiration times are in UTC
+
+
+
+ + + + + + + + + +
You received this mandatory email announcement to update you about + important changes to your TLS certificate. +
+
© 2020 Lemur +
+
+
+
+
diff --git a/lemur/tests/test_messaging.py b/lemur/tests/test_messaging.py index 0845d468..85032c00 100644 --- a/lemur/tests/test_messaging.py +++ b/lemur/tests/test_messaging.py @@ -5,7 +5,7 @@ import boto3 import pytest from freezegun import freeze_time from moto import mock_ses -from lemur.tests.factories import AuthorityFactory, CertificateFactory +from lemur.tests.factories import AuthorityFactory, CertificateFactory, EndpointFactory @mock_ses @@ -112,6 +112,26 @@ def test_send_expiration_notification_with_no_notifications( assert send_expiration_notifications([]) == (0, 0) +@mock_ses +def test_send_expiration_summary_notification(certificate, notification, notification_plugin): + from lemur.notifications.messaging import send_security_expiration_summary + verify_sender_email() + + # we don't actually test the email contents, but adding an assortment of certs here is useful for step debugging + # to confirm the produced email body looks like we expect + create_cert_that_expires_in_days(2) + create_cert_that_expires_in_days(2) + create_cert_that_expires_in_days(2) + create_cert_that_expires_in_days(30) + create_cert_that_expires_in_days(30) + create_cert_that_expires_in_days(15) + create_cert_that_expires_in_days(20) + create_cert_that_expires_in_days(1) + create_cert_that_expires_in_days(100) + + assert send_security_expiration_summary([]) + + @mock_ses def test_send_rotation_notification(notification_plugin, certificate): from lemur.notifications.messaging import send_rotation_notification @@ -170,3 +190,19 @@ def create_ca_cert_that_expires_in_days(days): certificate.root_authority_id = authority.id certificate.authority_id = None return certificate + + +def create_cert_that_expires_in_days(days): + from random import randrange + + now = arrow.utcnow() + not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future + + certificate = CertificateFactory() + certificate.not_after = not_after + certificate.notify = True + endpoints = [] + for i in range(0, randrange(0, 5)): + endpoints.append(EndpointFactory()) + certificate.endpoints = endpoints + return certificate From 065e4d3964d5e1f7276b7a1511e7879266a6cfb7 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 8 Dec 2020 11:46:55 -0800 Subject: [PATCH 2/4] Add new configuration for summary email intervals --- docs/administration.rst | 2 +- lemur/notifications/messaging.py | 7 +------ lemur/tests/test_messaging.py | 5 ++--- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 4c755e7b..fb576b9c 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -363,7 +363,7 @@ disabled by default; to enable it, you must set the option ``--notify`` (when us If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all certificates with upcoming expiration date that matches one of the intervals configured in the -``LEMUR_SECURITY_TEAM_EMAIL_INTERVALS`` configuration parameter (with the same fallbacks as noted above). +``LEMUR_EXPIRATION_SUMMARY_EMAIL_INTERVALS`` configuration parameter (with a fallback of 14 days). Note that certificates will be included in this summary even if they do not have any associated notifications. This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails. diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index e121e09a..b472a5f7 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -70,12 +70,7 @@ def get_certificates_for_security_summary_email(exclude=None): :return: """ now = arrow.utcnow() - expiration_summary_intervals = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL_INTERVALS", # first priority - current_app.config.get( # second priority - "LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", - [30, 15, 2] # third priority - ) - ) + expiration_summary_intervals = current_app.config.get("LEMUR_EXPIRATION_SUMMARY_EMAIL_INTERVALS", [14]) max_not_after = now + timedelta(days=max(expiration_summary_intervals) + 1) q = ( diff --git a/lemur/tests/test_messaging.py b/lemur/tests/test_messaging.py index 85032c00..c1c9b24a 100644 --- a/lemur/tests/test_messaging.py +++ b/lemur/tests/test_messaging.py @@ -119,10 +119,9 @@ def test_send_expiration_summary_notification(certificate, notification, notific # we don't actually test the email contents, but adding an assortment of certs here is useful for step debugging # to confirm the produced email body looks like we expect + for i in range(1, 6): + create_cert_that_expires_in_days(14) create_cert_that_expires_in_days(2) - create_cert_that_expires_in_days(2) - create_cert_that_expires_in_days(2) - create_cert_that_expires_in_days(30) create_cert_that_expires_in_days(30) create_cert_that_expires_in_days(15) create_cert_that_expires_in_days(20) From 15d43d6c270b5c81470f6834570c9962428bfea2 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 8 Dec 2020 12:09:53 -0800 Subject: [PATCH 3/4] Fix lint errors --- lemur/notifications/messaging.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index b472a5f7..9a55ab3a 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -75,10 +75,10 @@ def get_certificates_for_security_summary_email(exclude=None): q = ( database.db.session.query(Certificate) - .filter(Certificate.not_after <= max_not_after) - .filter(Certificate.notify == true()) - .filter(Certificate.expired == false()) - .filter(Certificate.revoked == false()) + .filter(Certificate.not_after <= max_not_after) + .filter(Certificate.notify == true()) + .filter(Certificate.expired == false()) + .filter(Certificate.revoked == false()) ) exclude_conditions = [] @@ -434,7 +434,7 @@ def send_security_expiration_summary(exclude=None): notification_type = "expiration_summary" log_data = { "function": function, - "message": f"Sending expiration summary notification for to security team", + "message": "Sending expiration summary notification for to security team", "notification_type": notification_type, "notification_plugin": notification_plugin.slug, } From 512b1acfddbdc2558717e53791303651a63c4f8d Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 8 Dec 2020 18:29:48 -0800 Subject: [PATCH 4/4] PR feedback: use days threshold instead of interval set, etc. --- docs/administration.rst | 4 ++-- lemur/common/celery.py | 4 ++-- lemur/notifications/messaging.py | 6 +++--- lemur/tests/test_messaging.py | 7 +++++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index fb576b9c..bd0b5f96 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -362,8 +362,8 @@ disabled by default; to enable it, you must set the option ``--notify`` (when us **Security certificate expiration summary** If you enable the Celery or cron task to send this notification type, Lemur will send a summary of all -certificates with upcoming expiration date that matches one of the intervals configured in the -``LEMUR_EXPIRATION_SUMMARY_EMAIL_INTERVALS`` configuration parameter (with a fallback of 14 days). +certificates with upcoming expiration date that occurs within the number of days specified by the +``LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS`` configuration parameter (with a fallback of 14 days). Note that certificates will be included in this summary even if they do not have any associated notifications. This notification type also supports the same ``--exclude`` and ``EXCLUDE_CN_FROM_NOTIFICATION`` options as expiration emails. diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 574d6ec5..578592dc 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -860,7 +860,7 @@ def notify_authority_expirations(): @celery.task(soft_time_limit=3600) def send_security_expiration_summary(): """ - This celery task sends a summary about expiring certificates to the security team. TODO document + This celery task sends a summary about expiring certificates to the security team. :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" @@ -881,7 +881,7 @@ def send_security_expiration_summary(): current_app.logger.debug(log_data) try: - cli_notification.send_security_expiration_summary() + cli_notification.security_expiration_summary(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) except SoftTimeLimitExceeded: log_data["message"] = "Send summary for expiring certs Time limit exceeded." current_app.logger.error(log_data) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 9a55ab3a..1d7bda4c 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -70,8 +70,8 @@ def get_certificates_for_security_summary_email(exclude=None): :return: """ now = arrow.utcnow() - expiration_summary_intervals = current_app.config.get("LEMUR_EXPIRATION_SUMMARY_EMAIL_INTERVALS", [14]) - max_not_after = now + timedelta(days=max(expiration_summary_intervals) + 1) + threshold_days = current_app.config.get("LEMUR_EXPIRATION_SUMMARY_EMAIL_THRESHOLD_DAYS", 14) + max_not_after = now + timedelta(days=threshold_days + 1) q = ( database.db.session.query(Certificate) @@ -91,7 +91,7 @@ def get_certificates_for_security_summary_email(exclude=None): certs = [] for c in windowed_query(q, Certificate.id, 10000): days_remaining = (c.not_after - now).days - if days_remaining in expiration_summary_intervals: + if days_remaining <= threshold_days: certs.append(c) return certs diff --git a/lemur/tests/test_messaging.py b/lemur/tests/test_messaging.py index c1c9b24a..d897f931 100644 --- a/lemur/tests/test_messaging.py +++ b/lemur/tests/test_messaging.py @@ -119,8 +119,11 @@ def test_send_expiration_summary_notification(certificate, notification, notific # we don't actually test the email contents, but adding an assortment of certs here is useful for step debugging # to confirm the produced email body looks like we expect - for i in range(1, 6): - create_cert_that_expires_in_days(14) + create_cert_that_expires_in_days(14) + create_cert_that_expires_in_days(12) + create_cert_that_expires_in_days(9) + create_cert_that_expires_in_days(7) + create_cert_that_expires_in_days(7) create_cert_that_expires_in_days(2) create_cert_that_expires_in_days(30) create_cert_that_expires_in_days(15)