diff --git a/docs/administration.rst b/docs/administration.rst index 3ef484be..94b15829 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -348,6 +348,15 @@ Lemur supports sending certificate expiration notifications through SES and SMTP LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2] +.. data:: LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS + :noindex: + + Notification interval set for CA certificate expiration notifications. If unspecified, the value [365, 180] is used (roughly one year and 6 months). + + :: + + LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS = [365, 180] + Celery Options --------------- diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index c2a8c48a..ad45692e 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -215,12 +215,13 @@ Notification ------------ Lemur includes the ability to create Email notifications by **default**. These notifications -currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and +currently come in the form of expiration and rotation notices for all certificates, expiration notices for CA certificates, +and ACME certificate creation failure notices. Lemur periodically checks certificate expiration dates and determines if a given certificate is eligible for notification. There are currently only two parameters used to determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number of days the current date (UTC) is from that expiration date. -Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable. +Certificate expiration notifications can also be configured for Slack or AWS SNS. Other notifications are not configurable. Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email. There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for diff --git a/docs/production/index.rst b/docs/production/index.rst index c6f561ca..d6e925a4 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -325,7 +325,7 @@ celery tasks or cron jobs that run these commands. There are currently three commands that could/should be run on a periodic basis: -- `notify` +- `notify expirations` and `notify authority_expirations` - `check_revoked` - `sync` @@ -334,13 +334,15 @@ If you are using LetsEncrypt, you must also run the following: - `fetch_all_pending_acme_certs` - `remove_old_acme_certs` -How often you run these commands is largely up to the user. `notify` and `check_revoked` are typically run at least once a day. +How often you run these commands is largely up to the user. `notify` should be run once a day (more often will result in +duplicate notifications). `check_revoked` is typically run at least once a day. `sync` is typically run every 15 minutes. `fetch_all_pending_acme_certs` should be ran frequently (Every minute is fine). `remove_old_acme_certs` can be ran more rarely, such as once every week. Example cron entries:: 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify expirations + 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur notify authority_expirations */15 * * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur source sync -s all 0 22 * * * lemuruser export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; /www/lemur/bin/lemur certificate check_revoked @@ -382,6 +384,20 @@ Example Celery configuration (To be placed in your configuration file):: 'expires': 180 }, 'schedule': crontab(hour="*"), + }, + 'notify_expirations': { + 'task': 'lemur.common.celery.notify_expirations', + 'options': { + 'expires': 180 + }, + 'schedule': crontab(hour=22, minute=0), + }, + 'notify_authority_expirations': { + 'task': 'lemur.common.celery.notify_authority_expirations', + 'options': { + 'expires': 180 + }, + 'schedule': crontab(hour=22, minute=0), } } diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f9d58bd9..f428927e 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -820,6 +820,42 @@ def notify_expirations(): return log_data +@celery.task(soft_time_limit=3600) +def notify_authority_expirations(): + """ + This celery task notifies about expiring certificate authority certs + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "message": "notify for certificate authority cert expiration", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + + current_app.logger.debug(log_data) + try: + cli_notification.authority_expirations() + except SoftTimeLimitExceeded: + log_data["message"] = "Notify expiring CA Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) + return + + metrics.send(f"{function}.success", "counter", 1) + return log_data + + @celery.task(soft_time_limit=3600) def enable_autorotate_for_certs_attached_to_endpoint(): """ diff --git a/lemur/notifications/cli.py b/lemur/notifications/cli.py index a2848117..7012e9c8 100644 --- a/lemur/notifications/cli.py +++ b/lemur/notifications/cli.py @@ -10,6 +10,7 @@ from flask_script import Manager from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS from lemur.extensions import sentry, metrics from lemur.notifications.messaging import send_expiration_notifications +from lemur.notifications.messaging import send_authority_expiration_notifications manager = Manager(usage="Handles notification related tasks.") @@ -24,7 +25,7 @@ manager = Manager(usage="Handles notification related tasks.") ) def expirations(exclude): """ - Runs Lemur's notification engine, that looks for expired certificates and sends + Runs Lemur's notification engine, that looks for expiring certificates and sends notifications out to those that have subscribed to them. Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur @@ -50,3 +51,29 @@ def expirations(exclude): metrics.send( "expiration_notification_job", "counter", 1, metric_tags={"status": status} ) + + +def authority_expirations(): + """ + Runs Lemur's notification engine, that looks for expiring certificate authority certificates and sends + notifications out to the security team and owner. + + :return: + """ + status = FAILURE_METRIC_STATUS + try: + print("Starting to notify subscribers about expiring certificate authority certificates!") + success, failed = send_authority_expiration_notifications() + print( + "Finished notifying subscribers about expiring certificate authority certificates! " + "Sent: {success} Failed: {failed}".format( + success=success, failed=failed + ) + ) + status = SUCCESS_METRIC_STATUS + except Exception as e: + sentry.captureException() + + metrics.send( + "authority_expiration_notification_job", "counter", 1, metric_tags={"status": status} + ) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 2658e1a0..9b299a24 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -62,6 +62,34 @@ def get_certificates(exclude=None): return certs +def get_expiring_authority_certificates(): + """ + Finds all certificate authority certificates that are eligible for expiration notifications. + :return: + """ + now = arrow.utcnow() + authority_expiration_intervals = current_app.config.get("LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS", + [365, 180]) + max_not_after = now + timedelta(days=max(authority_expiration_intervals) + 1) + + q = ( + database.db.session.query(Certificate) + .filter(Certificate.not_after < max_not_after) + .filter(Certificate.notify == true()) + .filter(Certificate.expired == false()) + .filter(Certificate.revoked == false()) + .filter(Certificate.root_authority_id.isnot(None)) + .filter(Certificate.authority_id.is_(None)) + ) + + certs = [] + for c in windowed_query(q, Certificate.id, 10000): + days_remaining = (c.not_after - now).days + if days_remaining in authority_expiration_intervals: + certs.append(c) + return certs + + def get_eligible_certificates(exclude=None): """ Finds all certificates that are eligible for certificate expiration notification. @@ -90,6 +118,25 @@ def get_eligible_certificates(exclude=None): return certificates +def get_eligible_authority_certificates(): + """ + Finds all certificate authority certificates that are eligible for certificate expiration notification. + Returns the set of all eligible CA certificates, grouped by owner and interval, with a list of applicable certs. + :return: + """ + certificates = defaultdict(dict) + all_certs = get_expiring_authority_certificates() + now = arrow.utcnow() + + # group by owner + for owner, owner_certs in groupby(all_certs, lambda x: x.owner): + # group by expiration interval + for interval, interval_certs in groupby(owner_certs, lambda x: (x.not_after - now).days): + certificates[owner][interval] = list(interval_certs) # list(c for c in interval_certs) + + return certificates + + def send_plugin_notification(event_type, data, recipients, notification): """ Executes the plugin and handles failure. @@ -176,6 +223,38 @@ def send_expiration_notifications(exclude): return success, failure +def send_authority_expiration_notifications(): + """ + This function will check for upcoming certificate authority certificate expiration, + and send out notification emails at configured intervals. + """ + success = failure = 0 + + # security team gets all + security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") + + for owner, owner_cert_groups in get_eligible_authority_certificates().items(): + for interval, certificates in owner_cert_groups.items(): + notification_data = [] + + for certificate in certificates: + cert_data = certificate_notification_output_schema.dump( + certificate + ).data + notification_data.append(cert_data) + + email_recipients = security_email + [owner] + if send_default_notification( + "authority_expiration", notification_data, email_recipients, + notification_options=[{'name': 'interval', 'value': interval}] + ): + success = len(email_recipients) + else: + failure = len(email_recipients) + + return success, failure + + def send_default_notification(notification_type, data, targets, notification_options=None): """ Sends a report to the specified target via the default notification plugin. Applicable for any notification_type. diff --git a/lemur/plugins/lemur_email/templates/authority_expiration.html b/lemur/plugins/lemur_email/templates/authority_expiration.html new file mode 100644 index 00000000..2c077bf5 --- /dev/null +++ b/lemur/plugins/lemur_email/templates/authority_expiration.html @@ -0,0 +1,172 @@ + + + + + + + + Lemur + + +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ Lemur +
+
+ + + + + + + + + + + + + + +
+ Your CA certificate(s) are expiring in {{ message.options | interval }} days! +
+
+ + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ Hi, +
+
This is a Lemur CA certificate expiration notice. The follow CA certificates are expiring soon; + please take manual action to renew them if necessary. You may also disable notifications via the + Notify toggle in Lemur if they are no longer in use. + + + {% for certificate in message.certificates %} + + + + + + {% if not loop.last %} + + + + {% endif %} + {% endfor %} + +
+ {{ certificate.name }} +
+ + {{ certificate.endpoints | length }} Endpoints +
{{ certificate.owner }} +
{{ certificate.validityEnd | time }} + Details + +
+
+ Your action is required if the above CA certificates are still needed. +
+
Best,
Lemur +
+ + + + + + +
*All expiration times are in UTC
+
+
+
+ + + + + + + + + +
You received this mandatory email announcement to update you about + important changes to your TLS certificate. +
+
© 2020 Lemur
+
+
+
+
diff --git a/lemur/tests/test_messaging.py b/lemur/tests/test_messaging.py index 13b6c9b3..9a9a5ad3 100644 --- a/lemur/tests/test_messaging.py +++ b/lemur/tests/test_messaging.py @@ -5,6 +5,7 @@ import boto3 import pytest from freezegun import freeze_time from moto import mock_ses +from lemur.tests.factories import AuthorityFactory, CertificateFactory @mock_ses @@ -125,3 +126,47 @@ def test_send_pending_failure_notification(notification_plugin, async_issuer_plu verify_sender_email() assert send_pending_failure_notification(pending_certificate) + + +def test_get_authority_certificates(): + from lemur.notifications.messaging import get_expiring_authority_certificates + + certificate_1 = create_cert_that_expires_in_days(180) + certificate_2 = create_cert_that_expires_in_days(365) + create_cert_that_expires_in_days(364) + create_cert_that_expires_in_days(366) + create_cert_that_expires_in_days(179) + create_cert_that_expires_in_days(181) + create_cert_that_expires_in_days(1) + + assert set(get_expiring_authority_certificates()) == {certificate_1, certificate_2} + + +@mock_ses +def test_send_authority_expiration_notifications(): + from lemur.notifications.messaging import send_authority_expiration_notifications + verify_sender_email() + + create_cert_that_expires_in_days(180) + create_cert_that_expires_in_days(180) # two on the same day results in a single email + create_cert_that_expires_in_days(365) + create_cert_that_expires_in_days(364) + create_cert_that_expires_in_days(366) + create_cert_that_expires_in_days(179) + create_cert_that_expires_in_days(181) + create_cert_that_expires_in_days(1) + + assert send_authority_expiration_notifications() == (2, 0) + + +def create_cert_that_expires_in_days(days): + now = arrow.utcnow() + not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future + + authority = AuthorityFactory() + certificate = CertificateFactory() + certificate.not_after = not_after + certificate.notify = True + certificate.root_authority_id = authority.id + certificate.authority_id = None + return certificate