Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into source_options

Browse Source
This commit is contained in:
sirferl
2020-12-08 10:57:40 +01:00
committed by GitHub
parent 5e33eee5c8 affd077afe
commit b07af654e1
16 changed files with 545 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
lemur/certificates/service.py
View File

@ -864,3 +864,12 @@ def cleanup_after_revoke(certificate):
database.update(certificate)
return error_message
def get_issued_cert_count_for_authority(authority):
"""
Returns the count of certs issued by the specified authority.
:return:
"""
return database.db.session.query(Certificate).filter(Certificate.authority_id == authority.id).count()

41
lemur/common/celery.py
View File

@ -656,11 +656,12 @@ def certificate_rotate(**kwargs):
current_app.logger.debug(log_data)
try:
notify = current_app.config.get("ENABLE_ROTATION_NOTIFICATION", None)
if region:
log_data["region"] = region
cli_certificate.rotate_region(None, None, None, None, True, region)
cli_certificate.rotate_region(None, None, None, notify, True, region)
else:
cli_certificate.rotate(None, None, None, None, True)
cli_certificate.rotate(None, None, None, notify, True)
except SoftTimeLimitExceeded:
log_data["message"] = "Certificate rotate: Time limit exceeded."
current_app.logger.error(log_data)
@ -820,6 +821,42 @@ def notify_expirations():
return log_data
@celery.task(soft_time_limit=3600)
def notify_authority_expirations():
"""
This celery task notifies about expiring certificate authority certs
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
task_id = None
if celery.current_task:
task_id = celery.current_task.request.id
log_data = {
"function": function,
"message": "notify for certificate authority cert expiration",
"task_id": task_id,
}
if task_id and is_task_active(function, task_id, None):
log_data["message"] = "Skipping task: Task is already active"
current_app.logger.debug(log_data)
return
current_app.logger.debug(log_data)
try:
cli_notification.authority_expirations()
except SoftTimeLimitExceeded:
log_data["message"] = "Notify expiring CA Time limit exceeded."
current_app.logger.error(log_data)
sentry.captureException()
metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
return
metrics.send(f"{function}.success", "counter", 1)
return log_data
@celery.task(soft_time_limit=3600)
def enable_autorotate_for_certs_attached_to_endpoint():
"""

31
lemur/notifications/cli.py
View File

@ -10,6 +10,7 @@ from flask_script import Manager
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.extensions import sentry, metrics
from lemur.notifications.messaging import send_expiration_notifications
from lemur.notifications.messaging import send_authority_expiration_notifications
manager = Manager(usage="Handles notification related tasks.")
@ -24,7 +25,7 @@ manager = Manager(usage="Handles notification related tasks.")
)
def expirations(exclude):
"""
Runs Lemur's notification engine, that looks for expired certificates and sends
Runs Lemur's notification engine, that looks for expiring certificates and sends
notifications out to those that have subscribed to them.
Every certificate receives notifications by default. When expiration notifications are handled outside of Lemur
@ -39,9 +40,7 @@ def expirations(exclude):
print("Starting to notify subscribers about expiring certificates!")
success, failed = send_expiration_notifications(exclude)
print(
"Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}".format(
success=success, failed=failed
)
f"Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}"
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
@ -50,3 +49,27 @@ def expirations(exclude):
metrics.send(
"expiration_notification_job", "counter", 1, metric_tags={"status": status}
)
def authority_expirations():
"""
Runs Lemur's notification engine, that looks for expiring certificate authority certificates and sends
notifications out to the security team and owner.
:return:
"""
status = FAILURE_METRIC_STATUS
try:
print("Starting to notify subscribers about expiring certificate authority certificates!")
success, failed = send_authority_expiration_notifications()
print(
"Finished notifying subscribers about expiring certificate authority certificates! "
f"Sent: {success} Failed: {failed}"
)
status = SUCCESS_METRIC_STATUS
except Exception as e:
sentry.captureException()
metrics.send(
"authority_expiration_notification_job", "counter", 1, metric_tags={"status": status}
)

84
lemur/notifications/messaging.py
View File

@ -19,9 +19,10 @@ from sqlalchemy import and_
from sqlalchemy.sql.expression import false, true
from lemur import database
from lemur.certificates import service as certificates_service
from lemur.certificates.models import Certificate
from lemur.certificates.schemas import certificate_notification_output_schema
from lemur.common.utils import windowed_query
from lemur.common.utils import windowed_query, is_selfsigned
from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS
from lemur.extensions import metrics, sentry
from lemur.pending_certificates.schemas import pending_certificate_output_schema
@ -62,6 +63,34 @@ def get_certificates(exclude=None):
return certs
def get_expiring_authority_certificates():
"""
Finds all certificate authority certificates that are eligible for expiration notifications.
:return:
"""
now = arrow.utcnow()
authority_expiration_intervals = current_app.config.get("LEMUR_AUTHORITY_CERT_EXPIRATION_EMAIL_INTERVALS",
[365, 180])
max_not_after = now + timedelta(days=max(authority_expiration_intervals) + 1)
q = (
database.db.session.query(Certificate)
.filter(Certificate.not_after < max_not_after)
.filter(Certificate.notify == true())
.filter(Certificate.expired == false())
.filter(Certificate.revoked == false())
.filter(Certificate.root_authority_id.isnot(None))
.filter(Certificate.authority_id.is_(None))
)
certs = []
for c in windowed_query(q, Certificate.id, 10000):
days_remaining = (c.not_after - now).days
if days_remaining in authority_expiration_intervals:
certs.append(c)
return certs
def get_eligible_certificates(exclude=None):
"""
Finds all certificates that are eligible for certificate expiration notification.
@ -90,6 +119,25 @@ def get_eligible_certificates(exclude=None):
return certificates
def get_eligible_authority_certificates():
"""
Finds all certificate authority certificates that are eligible for certificate expiration notification.
Returns the set of all eligible CA certificates, grouped by owner and interval, with a list of applicable certs.
:return:
"""
certificates = defaultdict(dict)
all_certs = get_expiring_authority_certificates()
now = arrow.utcnow()
# group by owner
for owner, owner_certs in groupby(all_certs, lambda x: x.owner):
# group by expiration interval
for interval, interval_certs in groupby(owner_certs, lambda x: (x.not_after - now).days):
certificates[owner][interval] = list(interval_certs)
return certificates
def send_plugin_notification(event_type, data, recipients, notification):
"""
Executes the plugin and handles failure.
@ -176,6 +224,40 @@ def send_expiration_notifications(exclude):
return success, failure
def send_authority_expiration_notifications():
"""
This function will check for upcoming certificate authority certificate expiration,
and send out notification emails at configured intervals.
"""
success = failure = 0
# security team gets all
security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
for owner, owner_cert_groups in get_eligible_authority_certificates().items():
for interval, certificates in owner_cert_groups.items():
notification_data = []
for certificate in certificates:
cert_data = certificate_notification_output_schema.dump(
certificate
).data
cert_data['self_signed'] = is_selfsigned(certificate.parsed_cert)
cert_data['issued_cert_count'] = certificates_service.get_issued_cert_count_for_authority(certificate.root_authority)
notification_data.append(cert_data)
email_recipients = security_email + [owner]
if send_default_notification(
"authority_expiration", notification_data, email_recipients,
notification_options=[{'name': 'interval', 'value': interval}]
):
success = len(email_recipients)
else:
failure = len(email_recipients)
return success, failure
def send_default_notification(notification_type, data, targets, notification_options=None):
"""
Sends a report to the specified target via the default notification plugin. Applicable for any notification_type.

3
lemur/plugins/lemur_email/plugin.py
View File

@ -108,7 +108,8 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
if not targets:
return
subject = "Lemur: {0} Notification".format(notification_type.capitalize())
readable_notification_type = ' '.join(map(lambda x: x.capitalize(), notification_type.split('_')))
subject = f"Lemur: {readable_notification_type} Notification"
body = render_html(notification_type, options, message)

179
lemur/plugins/lemur_email/templates/authority_expiration.html Normal file
View File

@ -0,0 +1,179 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
"http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="initial-scale=1.0"> <!-- So that mobile webkit will display zoomed in -->
<meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
<title>Lemur</title>
</head>
<div style="margin:0;padding:0" bgcolor="#FFFFFF">
<table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr height="32px"></tr>
<tr align="center">
<td width="32px"></td>
<td>
<table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
<tbody>
<tr>
<td>
<table width="100%" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
Lemur
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td>
<table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
<tbody>
<tr>
<td height="72px" colspan="3"></td>
</tr>
<tr>
<td width="32px"></td>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
Your CA certificate(s) are expiring in {{ message.options | interval }} days!
</td>
<td width="32px"></td>
</tr>
<tr>
<td height="18px" colspan="3"></td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td>
<table bgcolor="#FAFAFA" width="100%" border="0" cellspacing="0" cellpadding="0"
style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
<tbody>
<tr height="16px">
<td width="32px" rowspan="3"></td>
<td></td>
<td width="32px" rowspan="3"></td>
</tr>
<tr>
<td>
<table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Hi,
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>This is a Lemur CA certificate expiration notice. The following CA certificates are expiring soon;
please take manual action to renew them if necessary. Note that rotating a root CA requires
advanced planing and the respective trustStores need to be updated. A sub-CA, on the other hand,
does not require any changes to the trustStore. You may also disable notifications via the
Notify toggle in Lemur if they are no longer in use.
<table border="0" cellspacing="0" cellpadding="0"
style="margin-top:48px;margin-bottom:48px">
<tbody>
{% for certificate in message.certificates %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.name }}</span>
<br>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
{% if certificate.self_signed %}
<b>Root</b>
{% else %}
Intermediate
{% endif %} CA
<br>{{ certificate.issued_cert_count }} issued certificates
<br>{{ certificate.owner }}
<br>{{ certificate.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
</span>
</td>
</tr>
{% if not loop.last %}
<tr valign="middle">
<td width="32px" height="24px"></td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</td>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
Your action is required if the above CA certificates are still needed.
</td>
</tr>
<tr>
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>Best,<br><span class="il">Lemur</span>
</td>
</tr>
<tr height="16px"></tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
<tbody>
<tr>
<td>*All expiration times are in UTC<br></td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</td>
</tr>
<tr height="16"></tr>
<tr>
<td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
</tr>
<tr>
<td>
<table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
<tbody>
<tr>
<td>You received this mandatory email announcement to update you about
important changes to your <span class="il">TLS certificate</span>.
</td>
</tr>
<tr>
<td>
<div style="direction:ltr;text-align:left">© 2020 <span class="il">Lemur</span></div>
</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
</td>
<td width="32px"></td>
</tr>
<tr height="32px"></tr>
</tbody>
</table>
</div>

2
lemur/plugins/lemur_entrust/plugin.py
View File

@ -394,7 +394,7 @@ class EntrustSourcePlugin(SourcePlugin):
"external_id": str(certificate["trackingId"]),
"csr": certificate["csr"],
"owner": certificate["tracking"]["requesterEmail"],
"description": f"Type: Entrust {certificate['certType']}\nExtended Key Usage: {certificate['eku']}"
"description": f"Imported by Lemur; Type: Entrust {certificate['certType']}\nExtended Key Usage: {certificate['eku']}"
}
certs.append(cert)
processed_certs += 1

14
lemur/tests/test_certificates.py
View File

@ -1377,3 +1377,17 @@ def test_boolean_filter(client):
headers=VALID_ADMIN_HEADER_TOKEN,
)
assert resp.status_code == 200
def test_issued_cert_count_for_authority(authority):
from lemur.tests.factories import CertificateFactory
from lemur.certificates.service import get_issued_cert_count_for_authority
assert get_issued_cert_count_for_authority(authority) == 0
# create a few certs issued by the authority
CertificateFactory(authority=authority, name="test_issued_cert_count_for_authority1")
CertificateFactory(authority=authority, name="test_issued_cert_count_for_authority2")
CertificateFactory(authority=authority, name="test_issued_cert_count_for_authority3")
assert get_issued_cert_count_for_authority(authority) == 3

45
lemur/tests/test_messaging.py
View File

@ -5,6 +5,7 @@ import boto3
import pytest
from freezegun import freeze_time
from moto import mock_ses
from lemur.tests.factories import AuthorityFactory, CertificateFactory
@mock_ses
@ -125,3 +126,47 @@ def test_send_pending_failure_notification(notification_plugin, async_issuer_plu
verify_sender_email()
assert send_pending_failure_notification(pending_certificate)
def test_get_authority_certificates():
from lemur.notifications.messaging import get_expiring_authority_certificates
certificate_1 = create_ca_cert_that_expires_in_days(180)
certificate_2 = create_ca_cert_that_expires_in_days(365)
create_ca_cert_that_expires_in_days(364)
create_ca_cert_that_expires_in_days(366)
create_ca_cert_that_expires_in_days(179)
create_ca_cert_that_expires_in_days(181)
create_ca_cert_that_expires_in_days(1)
assert set(get_expiring_authority_certificates()) == {certificate_1, certificate_2}
@mock_ses
def test_send_authority_expiration_notifications():
from lemur.notifications.messaging import send_authority_expiration_notifications
verify_sender_email()
create_ca_cert_that_expires_in_days(180)
create_ca_cert_that_expires_in_days(180) # two on the same day results in a single email
create_ca_cert_that_expires_in_days(365)
create_ca_cert_that_expires_in_days(364)
create_ca_cert_that_expires_in_days(366)
create_ca_cert_that_expires_in_days(179)
create_ca_cert_that_expires_in_days(181)
create_ca_cert_that_expires_in_days(1)
assert send_authority_expiration_notifications() == (2, 0)
def create_ca_cert_that_expires_in_days(days):
now = arrow.utcnow()
not_after = now + timedelta(days=days, hours=1) # a bit more than specified since we'll check in the future
authority = AuthorityFactory()
certificate = CertificateFactory()
certificate.not_after = not_after
certificate.notify = True
certificate.root_authority_id = authority.id
certificate.authority_id = None
return certificate