Merge branch 'master' of github.com:Netflix/lemur into cname_01

This commit is contained in:
csine-nflx
2020-10-28 14:03:23 -07:00
67 changed files with 2412 additions and 465 deletions

View File

@ -20,6 +20,15 @@ class NotificationPlugin(Plugin):
def send(self, notification_type, message, targets, options, **kwargs):
raise NotImplementedError
def filter_recipients(self, options, excluded_recipients):
"""
Given a set of options (which should include configured recipient info), filters out recipients that
we do NOT want to notify.
For any notification types where recipients can't be dynamically modified, this returns an empty list.
"""
return []
class ExpirationNotificationPlugin(NotificationPlugin):
"""
@ -50,5 +59,5 @@ class ExpirationNotificationPlugin(NotificationPlugin):
def options(self):
return self.default_options + self.additional_options
def send(self, notification_type, message, targets, options, **kwargs):
def send(self, notification_type, message, excluded_targets, options, **kwargs):
raise NotImplementedError

View File

@ -32,6 +32,7 @@ from lemur.extensions import metrics, sentry
from lemur.plugins import lemur_acme as acme
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns
from lemur.authorities import service as authorities_service
from retrying import retry
@ -240,6 +241,7 @@ class AcmeHandler(object):
existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR"))
if existing_key and existing_regr:
current_app.logger.debug("Reusing existing ACME account")
# Reuse the same account for each certificate issuance
key = jose.JWK.json_loads(existing_key)
regr = messages.RegistrationResource.json_loads(existing_regr)
@ -253,6 +255,7 @@ class AcmeHandler(object):
# Create an account for each certificate issuance
key = jose.JWKRSA(key=generate_private_key("RSA2048"))
current_app.logger.debug("Creating a new ACME account")
current_app.logger.debug(
"Connecting with directory at {0}".format(directory_url)
)
@ -262,6 +265,27 @@ class AcmeHandler(object):
registration = client.new_account_and_tos(
messages.NewRegistration.from_data(email=email)
)
# if store_account is checked, add the private_key and registration resources to the options
if options['store_account']:
new_options = json.loads(authority.options)
# the key returned by fields_to_partial_json is missing the key type, so we add it manually
key_dict = key.fields_to_partial_json()
key_dict["kty"] = "RSA"
acme_private_key = {
"name": "acme_private_key",
"value": json.dumps(key_dict)
}
new_options.append(acme_private_key)
acme_regr = {
"name": "acme_regr",
"value": json.dumps({"body": {}, "uri": registration.uri})
}
new_options.append(acme_regr)
authorities_service.update_options(authority.id, options=json.dumps(new_options))
current_app.logger.debug("Connected: {0}".format(registration.uri))
return client, registration
@ -467,6 +491,13 @@ class ACMEIssuerPlugin(IssuerPlugin):
"validation": "/^-----BEGIN CERTIFICATE-----/",
"helpMessage": "Certificate to use",
},
{
"name": "store_account",
"type": "bool",
"required": False,
"helpMessage": "Disable to create a new account for each ACME request",
"default": False,
}
]
def __init__(self, *args, **kwargs):

View File

@ -1,8 +1,10 @@
import unittest
from unittest.mock import patch, Mock
import josepy as jose
from cryptography.x509 import DNSName
from lemur.plugins.lemur_acme import plugin
from lemur.common.utils import generate_private_key
from mock import MagicMock
@ -165,11 +167,65 @@ class TestAcme(unittest.TestCase):
with self.assertRaises(Exception):
self.acme.setup_acme_client(mock_authority)
@patch("lemur.plugins.lemur_acme.plugin.jose.JWK.json_loads")
@patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_setup_acme_client_success(self, mock_current_app, mock_acme):
def test_setup_acme_client_success_load_account_from_authority(self, mock_current_app, mock_acme, mock_key_json_load):
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
mock_authority.id = 2
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
'{"name": "store_account", "value": true},' \
'{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, ' \
'{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]'
mock_client = Mock()
mock_acme.return_value = mock_client
mock_current_app.config = {}
mock_key_json_load.return_value = jose.JWKRSA(key=generate_private_key("RSA2048"))
result_client, result_registration = self.acme.setup_acme_client(mock_authority)
mock_acme.new_account_and_tos.assert_not_called()
assert result_client
assert not result_registration
@patch("lemur.plugins.lemur_acme.plugin.jose.JWKRSA.fields_to_partial_json")
@patch("lemur.plugins.lemur_acme.plugin.authorities_service")
@patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_setup_acme_client_success_store_new_account(self, mock_current_app, mock_acme, mock_authorities_service,
mock_key_generation):
mock_authority = Mock()
mock_authority.id = 2
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
'{"name": "store_account", "value": true}]'
mock_client = Mock()
mock_registration = Mock()
mock_registration.uri = "http://test.com"
mock_client.register = mock_registration
mock_client.agree_to_tos = Mock(return_value=True)
mock_client.new_account_and_tos.return_value = mock_registration
mock_acme.return_value = mock_client
mock_current_app.config = {}
mock_key_generation.return_value = {"n": "PwIOkViO"}
mock_authorities_service.update_options = Mock(return_value=True)
self.acme.setup_acme_client(mock_authority)
mock_authorities_service.update_options.assert_called_with(2, options='[{"name": "mock_name", "value": "mock_value"}, '
'{"name": "store_account", "value": true}, '
'{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, '
'{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]')
@patch("lemur.plugins.lemur_acme.plugin.authorities_service")
@patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2")
@patch("lemur.plugins.lemur_acme.plugin.current_app")
def test_setup_acme_client_success(self, mock_current_app, mock_acme, mock_authorities_service):
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
'{"name": "store_account", "value": false}]'
mock_client = Mock()
mock_registration = Mock()
mock_registration.uri = "http://test.com"
@ -178,6 +234,7 @@ class TestAcme(unittest.TestCase):
mock_acme.return_value = mock_client
mock_current_app.config = {}
result_client, result_registration = self.acme.setup_acme_client(mock_authority)
mock_authorities_service.update_options.assert_not_called()
assert result_client
assert result_registration

View File

@ -32,13 +32,14 @@
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
.. moduleauthor:: Harm Weites <harm@weites.com>
"""
from acme.errors import ClientError
from flask import current_app
from lemur.extensions import sentry, metrics
from lemur.plugins import lemur_aws as aws
from lemur.extensions import sentry, metrics
from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin
from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
from lemur.plugins.lemur_aws import iam, s3, elb, ec2
from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns
def get_region_from_dns(dns):
@ -406,3 +407,51 @@ class S3DestinationPlugin(ExportDestinationPlugin):
self.get_option("encrypt", options),
account_number=self.get_option("accountNumber", options),
)
class SNSNotificationPlugin(ExpirationNotificationPlugin):
title = "AWS SNS"
slug = "aws-sns"
description = "Sends notifications to AWS SNS"
version = aws.VERSION
author = "Jasmine Schladen <jschladen@netflix.com>"
author_url = "https://github.com/Netflix/lemur"
additional_options = [
{
"name": "accountNumber",
"type": "str",
"required": True,
"validation": "[0-9]{12}",
"helpMessage": "A valid AWS account number with permission to access the SNS topic",
},
{
"name": "region",
"type": "str",
"required": True,
"validation": "[0-9a-z\\-]{1,25}",
"helpMessage": "Region in which the SNS topic is located, e.g. \"us-east-1\"",
},
{
"name": "topicName",
"type": "str",
"required": True,
# base topic name is 1-256 characters (alphanumeric plus underscore and hyphen)
"validation": "^[a-zA-Z0-9_\\-]{1,256}$",
"helpMessage": "The name of the topic to use for expiration notifications",
}
]
def send(self, notification_type, message, excluded_targets, options, **kwargs):
"""
While we receive a `targets` parameter here, it is unused, as the SNS topic is pre-configured in the
plugin configuration, and can't reasonably be changed dynamically.
"""
topic_arn = f"arn:aws:sns:{self.get_option('region', options)}:" \
f"{self.get_option('accountNumber', options)}:" \
f"{self.get_option('topicName', options)}"
current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}")
sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options))

View File

@ -0,0 +1,55 @@
"""
.. module: lemur.plugins.lemur_aws.sts
:platform: Unix
:copyright: (c) 2020 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Jasmine Schladen <jschladen@netflix.com>
"""
import json
import arrow
import boto3
from flask import current_app
def publish(topic_arn, certificates, notification_type, **kwargs):
sns_client = boto3.client("sns", **kwargs)
message_ids = {}
for certificate in certificates:
message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type)
return message_ids
def publish_single(sns_client, topic_arn, certificate, notification_type):
response = sns_client.publish(
TopicArn=topic_arn,
Message=format_message(certificate, notification_type),
)
response_code = response["ResponseMetadata"]["HTTPStatusCode"]
if response_code != 200:
raise Exception(f"Failed to publish {notification_type} notification to SNS topic {topic_arn}. "
f"SNS response: {response_code} {response}")
current_app.logger.info(f"AWS SNS message published to topic [{topic_arn}] with message ID {response['MessageId']}")
current_app.logger.debug(f"AWS SNS message published to topic [{topic_arn}]: [{response}]")
return response["MessageId"]
def create_certificate_url(name):
return "https://{hostname}/#/certificates/{name}".format(
hostname=current_app.config.get("LEMUR_HOSTNAME"), name=name
)
def format_message(certificate, notification_type):
json_message = {
"notification_type": notification_type,
"certificate_name": certificate["name"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"), # 2047-12-T22:00:00
"endpoints_detected": len(certificate["endpoints"]),
"details": create_certificate_url(certificate["name"])
}
return json.dumps(json_message)

View File

@ -0,0 +1,120 @@
import json
from datetime import timedelta
import arrow
import boto3
from moto import mock_sns, mock_sqs, mock_ses
from lemur.certificates.schemas import certificate_notification_output_schema
from lemur.plugins.lemur_aws.sns import format_message
from lemur.plugins.lemur_aws.sns import publish
from lemur.tests.factories import NotificationFactory, CertificateFactory
from lemur.tests.test_messaging import verify_sender_email
@mock_sns()
def test_format(certificate, endpoint):
data = [certificate_notification_output_schema.dump(certificate).data]
for certificate in data:
expected_message = {
"notification_type": "expiration",
"certificate_name": certificate["name"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"),
"endpoints_detected": 0,
"details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
}
assert expected_message == json.loads(format_message(certificate, "expiration"))
@mock_sns()
@mock_sqs()
def create_and_subscribe_to_topic():
sns_client = boto3.client("sns", region_name="us-east-1")
topic_arn = sns_client.create_topic(Name='lemursnstest')["TopicArn"]
sqs_client = boto3.client("sqs", region_name="us-east-1")
queue = sqs_client.create_queue(QueueName="lemursnstestqueue")
queue_url = queue["QueueUrl"]
queue_arn = sqs_client.get_queue_attributes(QueueUrl=queue_url)["Attributes"]["QueueArn"]
sns_client.subscribe(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_arn)
return [topic_arn, sqs_client, queue_url]
@mock_sns()
@mock_sqs()
def test_publish(certificate, endpoint):
data = [certificate_notification_output_schema.dump(certificate).data]
topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic()
message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1")
assert len(message_ids) == len(data)
received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
for certificate in data:
expected_message_id = message_ids[certificate["name"]]
actual_message = next(
(m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None)
assert json.loads(actual_message["Body"])["Message"] == format_message(certificate, "expiration")
def get_options():
return [
{"name": "interval", "value": 10},
{"name": "unit", "value": "days"},
{"name": "region", "value": "us-east-1"},
{"name": "accountNumber", "value": "123456789012"},
{"name": "topicName", "value": "lemursnstest"},
]
@mock_sns()
@mock_sqs()
@mock_ses() # because email notifications are also sent
def test_send_expiration_notification():
from lemur.notifications.messaging import send_expiration_notifications
verify_sender_email() # emails are sent to owner and security; SNS only used for configured notification
topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic()
notification = NotificationFactory(plugin_name="aws-sns")
notification.options = get_options()
now = arrow.utcnow()
in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future
certificate = CertificateFactory()
certificate.not_after = in_ten_days
certificate.notifications.append(notification)
assert send_expiration_notifications([]) == (3, 0) # owner, SNS, and security
received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
assert len(received_messages) == 1
expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration")
actual_message = json.loads(received_messages[0]["Body"])["Message"]
assert actual_message == expected_message
# Currently disabled as the SNS plugin doesn't support this type of notification
# def test_send_rotation_notification(endpoint, source_plugin):
# from lemur.notifications.messaging import send_rotation_notification
# from lemur.deployment.service import rotate_certificate
#
# notification = NotificationFactory(plugin_name="aws-sns")
# notification.options = get_options()
#
# new_certificate = CertificateFactory()
# rotate_certificate(endpoint, new_certificate)
# assert endpoint.certificate == new_certificate
#
# assert send_rotation_notification(new_certificate)
# Currently disabled as the SNS plugin doesn't support this type of notification
# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin):
# from lemur.notifications.messaging import send_pending_failure_notification
#
# assert send_pending_failure_notification(pending_certificate)

View File

@ -18,9 +18,10 @@ import json
import arrow
import pem
import requests
import sys
from cryptography import x509
from flask import current_app
from lemur.common.utils import validate_conf
from flask import current_app, g
from lemur.common.utils import validate_conf, convert_pkcs7_bytes_to_pem
from lemur.extensions import metrics
from lemur.plugins import lemur_digicert as digicert
from lemur.plugins.bases import IssuerPlugin, SourcePlugin
@ -36,7 +37,13 @@ def log_status_code(r, *args, **kwargs):
:param kwargs:
:return:
"""
log_data = {
"reason": (r.reason if r.reason else ""),
"status_code": r.status_code,
"url": (r.url if r.url else ""),
}
metrics.send("digicert_status_code_{}".format(r.status_code), "counter", 1)
current_app.logger.info(log_data)
def signature_hash(signing_algorithm):
@ -129,6 +136,9 @@ def map_fields(options, csr):
data["validity_years"] = determine_validity_years(options.get("validity_years"))
elif options.get("validity_end"):
data["custom_expiration_date"] = determine_end_date(options.get("validity_end")).format("YYYY-MM-DD")
# check if validity got truncated. If resultant validity is not equal to requested validity, it just got truncated
if data["custom_expiration_date"] != options.get("validity_end").format("YYYY-MM-DD"):
log_validity_truncation(options, f"{__name__}.{sys._getframe().f_code.co_name}")
else:
data["validity_years"] = determine_validity_years(0)
@ -154,6 +164,9 @@ def map_cis_fields(options, csr):
validity_end = determine_end_date(arrow.utcnow().shift(years=options["validity_years"]))
elif options.get("validity_end"):
validity_end = determine_end_date(options.get("validity_end"))
# check if validity got truncated. If resultant validity is not equal to requested validity, it just got truncated
if validity_end != options.get("validity_end"):
log_validity_truncation(options, f"{__name__}.{sys._getframe().f_code.co_name}")
else:
validity_end = determine_end_date(False)
@ -164,11 +177,10 @@ def map_cis_fields(options, csr):
"csr": csr,
"signature_hash": signature_hash(options.get("signing_algorithm")),
"validity": {
"valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z"
"valid_to": validity_end.format("YYYY-MM-DDTHH:mm:ss") + "Z"
},
"organization": {
"name": options["organization"],
"units": [options["organizational_unit"]],
},
}
# possibility to default to a SIGNING_ALGORITHM for a given profile
@ -179,6 +191,18 @@ def map_cis_fields(options, csr):
return data
def log_validity_truncation(options, function):
log_data = {
"cn": options["common_name"],
"creator": g.user.username
}
metrics.send("digicert_validity_truncated", "counter", 1, metric_tags=log_data)
log_data["function"] = function
log_data["message"] = "Digicert Plugin truncated the validity of certificate"
current_app.logger.info(log_data)
def handle_response(response):
"""
Handle the DigiCert API response and any errors it might have experienced.
@ -186,7 +210,7 @@ def handle_response(response):
:return:
"""
if response.status_code > 399:
raise Exception(response.json()["errors"][0]["message"])
raise Exception("DigiCert rejected request with the error:" + response.json()["errors"][0]["message"])
return response.json()
@ -197,10 +221,17 @@ def handle_cis_response(response):
:param response:
:return:
"""
if response.status_code > 399:
raise Exception(response.text)
if response.status_code == 404:
raise Exception("DigiCert: order not in issued state")
elif response.status_code == 406:
raise Exception("DigiCert: wrong header request format")
elif response.status_code > 399:
raise Exception("DigiCert rejected request with the error:" + response.text)
return response.json()
if response.url.endswith("download"):
return response.content
else:
return response.json()
@retry(stop_max_attempt_number=10, wait_fixed=10000)
@ -216,15 +247,16 @@ def get_certificate_id(session, base_url, order_id):
@retry(stop_max_attempt_number=10, wait_fixed=10000)
def get_cis_certificate(session, base_url, order_id):
"""Retrieve certificate order id from Digicert API."""
certificate_url = "{0}/platform/cis/certificate/{1}".format(base_url, order_id)
session.headers.update({"Accept": "application/x-pem-file"})
"""Retrieve certificate order id from Digicert API, including the chain"""
certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id)
session.headers.update({"Accept": "application/x-pkcs7-certificates"})
response = session.get(certificate_url)
response_content = handle_cis_response(response)
if response.status_code == 404:
raise Exception("Order not in issued state.")
return response.content
cert_chain_pem = convert_pkcs7_bytes_to_pem(response_content)
if len(cert_chain_pem) < 3:
raise Exception("Missing the certificate chain")
return cert_chain_pem
class DigiCertSourcePlugin(SourcePlugin):
@ -428,7 +460,6 @@ class DigiCertCISSourcePlugin(SourcePlugin):
"DIGICERT_CIS_API_KEY",
"DIGICERT_CIS_URL",
"DIGICERT_CIS_ROOTS",
"DIGICERT_CIS_INTERMEDIATES",
"DIGICERT_CIS_PROFILE_NAMES",
]
validate_conf(current_app, required_vars)
@ -503,7 +534,6 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
"DIGICERT_CIS_API_KEY",
"DIGICERT_CIS_URL",
"DIGICERT_CIS_ROOTS",
"DIGICERT_CIS_INTERMEDIATES",
"DIGICERT_CIS_PROFILE_NAMES",
]
@ -533,22 +563,15 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
data = handle_cis_response(response)
# retrieve certificate
certificate_pem = get_cis_certificate(self.session, base_url, data["id"])
certificate_chain_pem = get_cis_certificate(self.session, base_url, data["id"])
self.session.headers.pop("Accept")
end_entity = pem.parse(certificate_pem)[0]
end_entity = certificate_chain_pem[0]
intermediate = certificate_chain_pem[1]
if "ECC" in issuer_options["key_type"]:
return (
"\n".join(str(end_entity).splitlines()),
current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name),
data["id"],
)
# By default return RSA
return (
"\n".join(str(end_entity).splitlines()),
current_app.config.get("DIGICERT_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name),
"\n".join(str(intermediate).splitlines()),
data["id"],
)

View File

@ -121,9 +121,9 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority):
"csr": CSR_STR,
"additional_dns_names": names,
"signature_hash": "sha256",
"organization": {"name": "Example, Inc.", "units": ["Example Org"]},
"organization": {"name": "Example, Inc."},
"validity": {
"valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM") + "Z"
"valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:mm:ss") + "Z"
},
"profile_name": None,
}
@ -157,9 +157,9 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho
"csr": CSR_STR,
"additional_dns_names": names,
"signature_hash": "sha256",
"organization": {"name": "Example, Inc.", "units": ["Example Org"]},
"organization": {"name": "Example, Inc."},
"validity": {
"valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z"
"valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:mm:ss") + "Z"
},
"profile_name": None,
}

View File

@ -17,16 +17,19 @@ from lemur.plugins.bases import ExpirationNotificationPlugin
from lemur.plugins import lemur_email as email
from lemur.plugins.lemur_email.templates.config import env
from lemur.plugins.utils import get_plugin_option
def render_html(template_name, message):
def render_html(template_name, options, certificates):
"""
Renders the html for our email notification.
:param template_name:
:param message:
:param options:
:param certificates:
:return:
"""
message = {"options": options, "certificates": certificates}
template = env.get_template("{}.html".format(template_name))
return template.render(
dict(message=message, hostname=current_app.config.get("LEMUR_HOSTNAME"))
@ -100,8 +103,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
subject = "Lemur: {0} Notification".format(notification_type.capitalize())
data = {"options": options, "certificates": message}
body = render_html(notification_type, data)
body = render_html(notification_type, options, message)
s_type = current_app.config.get("LEMUR_EMAIL_SENDER", "ses").lower()
@ -110,3 +112,13 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
elif s_type == "smtp":
send_via_smtp(subject, body, targets)
@staticmethod
def filter_recipients(options, excluded_recipients, **kwargs):
notification_recipients = get_plugin_option("recipients", options)
if notification_recipients:
notification_recipients = notification_recipients.split(",")
# removing owner and security_email from notification_recipient
notification_recipients = [i for i in notification_recipients if i not in excluded_recipients]
return notification_recipients

View File

@ -83,12 +83,12 @@
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.name }}</span>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.certificates.name }}</span>
<br>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
<br>{{ certificate.owner }}
<br>{{ certificate.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
<br>{{ message.certificates.owner }}
<br>{{ message.certificates.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ message.certificates.name }}" target="_blank">Details</a>
</span>
</td>
</tr>
@ -110,12 +110,12 @@
<td width="32px"></td>
<td width="16px"></td>
<td style="line-height:1.2">
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.replacedBy[0].name }}</span>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.certificates.name }}</span>
<br>
<span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
<br>{{ certificate.replacedBy[0].owner }}
<br>{{ certificate.replacedBy[0].validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ certificate.replacedBy[0].name }}" target="_blank">Details</a>
<br>{{ message.certificates.owner }}
<br>{{ message.certificates.validityEnd | time }}
<a href="https://{{ hostname }}/#/certificates/{{ message.certificates.name }}" target="_blank">Details</a>
</span>
</td>
</tr>
@ -133,7 +133,7 @@
<table border="0" cellspacing="0" cellpadding="0"
style="margin-top:48px;margin-bottom:48px">
<tbody>
{% for endpoint in certificate.endpoints %}
{% for endpoint in message.certificates.endpoints %}
<tr valign="middle">
<td width="32px"></td>
<td width="16px"></td>

View File

@ -1,36 +1,90 @@
import os
from lemur.plugins.lemur_email.templates.config import env
from datetime import timedelta
import arrow
from moto import mock_ses
from lemur.certificates.schemas import certificate_notification_output_schema
from lemur.plugins.lemur_email.plugin import render_html
from lemur.tests.factories import CertificateFactory
from lemur.tests.test_messaging import verify_sender_email
dir_path = os.path.dirname(os.path.realpath(__file__))
def test_render(certificate, endpoint):
from lemur.certificates.schemas import certificate_notification_output_schema
def get_options():
return [
{"name": "interval", "value": 10},
{"name": "unit", "value": "days"},
{"name": "recipients", "value": "person1@example.com,person2@example.com"},
]
def test_render_expiration(certificate, endpoint):
new_cert = CertificateFactory()
new_cert.replaces.append(certificate)
data = {
"certificates": [certificate_notification_output_schema.dump(certificate).data],
"options": [
{"name": "interval", "value": 10},
{"name": "unit", "value": "days"},
],
}
assert render_html("expiration", get_options(), [certificate_notification_output_schema.dump(certificate).data])
template = env.get_template("{}.html".format("expiration"))
body = template.render(dict(message=data, hostname="lemur.test.example.com"))
template = env.get_template("{}.html".format("rotation"))
def test_render_rotation(certificate, endpoint):
certificate.endpoints.append(endpoint)
body = template.render(
dict(
certificate=certificate_notification_output_schema.dump(certificate).data,
hostname="lemur.test.example.com",
)
)
assert render_html("rotation", get_options(), certificate_notification_output_schema.dump(certificate).data)
def test_render_rotation_failure(pending_certificate):
assert render_html("failed", get_options(), certificate_notification_output_schema.dump(pending_certificate).data)
@mock_ses
def test_send_expiration_notification():
from lemur.notifications.messaging import send_expiration_notifications
from lemur.tests.factories import CertificateFactory
from lemur.tests.factories import NotificationFactory
now = arrow.utcnow()
in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future
certificate = CertificateFactory()
notification = NotificationFactory(plugin_name="email-notification")
certificate.not_after = in_ten_days
certificate.notifications.append(notification)
certificate.notifications[0].options = get_options()
verify_sender_email()
assert send_expiration_notifications([]) == (3, 0) # owner, recipients (only counted as 1), and security
@mock_ses
def test_send_rotation_notification(endpoint, source_plugin):
from lemur.notifications.messaging import send_rotation_notification
from lemur.deployment.service import rotate_certificate
new_certificate = CertificateFactory()
rotate_certificate(endpoint, new_certificate)
assert endpoint.certificate == new_certificate
verify_sender_email()
assert send_rotation_notification(new_certificate)
@mock_ses
def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin):
from lemur.notifications.messaging import send_pending_failure_notification
verify_sender_email()
assert send_pending_failure_notification(pending_certificate)
def test_filter_recipients(certificate, endpoint):
from lemur.plugins.lemur_email.plugin import EmailNotificationPlugin
options = [{"name": "recipients", "value": "security@example.com,bob@example.com,joe@example.com"}]
assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@example.com", "bob@example.com",
"joe@example.com"]
assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com"]) == ["bob@example.com",
"joe@example.com"]
assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com", "bob@example.com",
"joe@example.com"]) == []

View File

@ -0,0 +1,5 @@
"""Set the version information."""
try:
VERSION = __import__("pkg_resources").get_distribution(__name__).version
except Exception as e:
VERSION = "unknown"

View File

@ -0,0 +1,266 @@
import arrow
import requests
import json
import sys
from flask import current_app
from lemur.plugins import lemur_entrust as entrust
from lemur.plugins.bases import IssuerPlugin, SourcePlugin
from lemur.extensions import metrics
from lemur.common.utils import validate_conf
def log_status_code(r, *args, **kwargs):
"""
Is a request hook that logs all status codes to the ENTRUST api.
:param r:
:param args:
:param kwargs:
:return:
"""
log_data = {
"reason": (r.reason if r.reason else ""),
"status_code": r.status_code,
"url": (r.url if r.url else ""),
}
metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1)
current_app.logger.info(log_data)
def determine_end_date(end_date):
"""
Determine appropriate end date
:param end_date:
:return: validity_end as string
"""
# ENTRUST only allows 13 months of max certificate duration
max_validity_end = arrow.utcnow().shift(years=1, months=+1)
if not end_date:
end_date = max_validity_end
elif end_date > max_validity_end:
end_date = max_validity_end
return end_date.format('YYYY-MM-DD')
def process_options(options):
"""
Processes and maps the incoming issuer options to fields/options that
Entrust understands
:param options:
:return: dict of valid entrust options
"""
# if there is a config variable ENTRUST_PRODUCT_<upper(authority.name)>
# take the value as Cert product-type
# else default to "STANDARD_SSL"
authority = options.get("authority").name.upper()
# STANDARD_SSL (cn=domain, san=www.domain),
# ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]),
# WILDCARD_SSL (unlimited sans, and wildcard)
product_type = current_app.config.get(f"ENTRUST_PRODUCT_{authority}", "STANDARD_SSL")
if options.get("validity_end"):
validity_end = determine_end_date(options.get("validity_end"))
else:
validity_end = determine_end_date(False)
tracking_data = {
"requesterName": current_app.config.get("ENTRUST_NAME"),
"requesterEmail": current_app.config.get("ENTRUST_EMAIL"),
"requesterPhone": current_app.config.get("ENTRUST_PHONE")
}
data = {
"signingAlg": "SHA-2",
"eku": "SERVER_AND_CLIENT_AUTH",
"certType": product_type,
"certExpiryDate": validity_end,
# "keyType": "RSA", Entrust complaining about this parameter
"tracking": tracking_data
}
return data
def handle_response(my_response):
"""
Helper function for parsing responses from the Entrust API.
:param content:
:return: :raise Exception:
"""
msg = {
200: "The request had the validateOnly flag set to true and validation was successful.",
201: "Certificate created",
202: "Request accepted and queued for approval",
400: "Invalid request parameters",
404: "Unknown jobId",
429: "Too many requests"
}
try:
d = json.loads(my_response.content)
except ValueError:
# catch an empty jason object here
d = {'response': 'No detailed message'}
s = my_response.status_code
if s > 399:
raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}")
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Response",
"status": s,
"response": d
}
current_app.logger.info(log_data)
if d == {'response': 'No detailed message'}:
# status if no data
return s
else:
# return data from the response
return d
class EntrustIssuerPlugin(IssuerPlugin):
title = "Entrust"
slug = "entrust-issuer"
description = "Enables the creation of certificates by ENTRUST"
version = entrust.VERSION
author = "sirferl"
author_url = "https://github.com/sirferl/lemur"
def __init__(self, *args, **kwargs):
"""Initialize the issuer with the appropriate details."""
required_vars = [
"ENTRUST_API_CERT",
"ENTRUST_API_KEY",
"ENTRUST_API_USER",
"ENTRUST_API_PASS",
"ENTRUST_URL",
"ENTRUST_ROOT",
"ENTRUST_NAME",
"ENTRUST_EMAIL",
"ENTRUST_PHONE",
]
validate_conf(current_app, required_vars)
self.session = requests.Session()
cert_file = current_app.config.get("ENTRUST_API_CERT")
key_file = current_app.config.get("ENTRUST_API_KEY")
user = current_app.config.get("ENTRUST_API_USER")
password = current_app.config.get("ENTRUST_API_PASS")
self.session.cert = (cert_file, key_file)
self.session.auth = (user, password)
self.session.hooks = dict(response=log_status_code)
# self.session.config['keep_alive'] = False
super(EntrustIssuerPlugin, self).__init__(*args, **kwargs)
def create_certificate(self, csr, issuer_options):
"""
Creates an Entrust certificate.
:param csr:
:param issuer_options:
:return: :raise Exception:
"""
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Requesting options",
"options": issuer_options
}
current_app.logger.info(log_data)
url = current_app.config.get("ENTRUST_URL") + "/certificates"
data = process_options(issuer_options)
data["csr"] = csr
try:
response = self.session.post(url, json=data, timeout=(15, 40))
except requests.exceptions.Timeout:
raise Exception("Timeout for POST")
except requests.exceptions.RequestException as e:
raise Exception(f"Error for POST {e}")
response_dict = handle_response(response)
external_id = response_dict['trackingId']
cert = response_dict['endEntityCert']
if len(response_dict['chainCerts']) < 2:
# certificate signed by CA directly, no ICA included ini the chain
chain = None
else:
chain = response_dict['chainCerts'][1]
log_data["message"] = "Received Chain"
log_data["options"] = f"chain: {chain}"
current_app.logger.info(log_data)
return cert, chain, external_id
def revoke_certificate(self, certificate, comments):
"""Revoke an Entrust certificate."""
base_url = current_app.config.get("ENTRUST_URL")
# make certificate revoke request
revoke_url = f"{base_url}/certificates/{certificate.external_id}/revocations"
if not comments or comments == '':
comments = "revoked via API"
data = {
"crlReason": "superseded", # enum (keyCompromise, affiliationChanged, superseded, cessationOfOperation)
"revocationComment": comments
}
response = self.session.post(revoke_url, json=data)
metrics.send("entrust_revoke_certificate", "counter", 1)
return handle_response(response)
def deactivate_certificate(self, certificate):
"""Deactivates an Entrust certificate."""
base_url = current_app.config.get("ENTRUST_URL")
deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations"
response = self.session.post(deactivate_url)
metrics.send("entrust_deactivate_certificate", "counter", 1)
return handle_response(response)
@staticmethod
def create_authority(options):
"""Create an authority.
Creates an authority, this authority is then used by Lemur to
allow a user to specify which Certificate Authority they want
to sign their certificate.
:param options:
:return:
"""
entrust_root = current_app.config.get("ENTRUST_ROOT")
entrust_issuing = current_app.config.get("ENTRUST_ISSUING")
role = {"username": "", "password": "", "name": "entrust"}
current_app.logger.info(f"Creating Auth: {options} {entrust_issuing}")
# body, chain, role
return entrust_root, "", [role]
def get_ordered_certificate(self, order_id):
raise NotImplementedError("Not implemented\n", self, order_id)
def canceled_ordered_certificate(self, pending_cert, **kwargs):
raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs)
class EntrustSourcePlugin(SourcePlugin):
title = "Entrust"
slug = "entrust-source"
description = "Enables the collection of certificates"
version = entrust.VERSION
author = "sirferl"
author_url = "https://github.com/sirferl/lemur"
def get_certificates(self, options, **kwargs):
# Not needed for ENTRUST
raise NotImplementedError("Not implemented\n", self, options, **kwargs)
def get_endpoints(self, options, **kwargs):
# There are no endpoints in ENTRUST
raise NotImplementedError("Not implemented\n", self, options, **kwargs)

View File

@ -0,0 +1 @@
from lemur.tests.conftest import * # noqa

View File

@ -0,0 +1,62 @@
from unittest.mock import patch, Mock
import arrow
from cryptography import x509
from lemur.plugins.lemur_entrust import plugin
from freezegun import freeze_time
def config_mock(*args):
values = {
"ENTRUST_API_CERT": "-----BEGIN CERTIFICATE-----abc-----END CERTIFICATE-----",
"ENTRUST_API_KEY": False,
"ENTRUST_API_USER": "test",
"ENTRUST_API_PASS": "password",
"ENTRUST_URL": "http",
"ENTRUST_ROOT": None,
"ENTRUST_NAME": "test",
"ENTRUST_EMAIL": "test@lemur.net",
"ENTRUST_PHONE": "0123456",
"ENTRUST_PRODUCT_ENTRUST": "ADVANTAGE_SSL"
}
return values[args[0]]
@patch("lemur.plugins.lemur_digicert.plugin.current_app")
def test_determine_end_date(mock_current_app):
with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):
assert arrow.get(2017, 12, 3).format('YYYY-MM-DD') == plugin.determine_end_date(0) # 1 year + 1 month
assert arrow.get(2017, 3, 5).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2017, 3, 5))
assert arrow.get(2017, 12, 3).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2020, 5, 7))
@patch("lemur.plugins.lemur_entrust.plugin.current_app")
def test_process_options(mock_current_app, authority):
mock_current_app.config.get = Mock(side_effect=config_mock)
plugin.determine_end_date = Mock(return_value=arrow.get(2017, 11, 5).format('YYYY-MM-DD'))
authority.name = "Entrust"
names = [u"one.example.com", u"two.example.com", u"three.example.com"]
options = {
"common_name": "example.com",
"owner": "bob@example.com",
"description": "test certificate",
"extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
"organization": "Example, Inc.",
"organizational_unit": "Example Org",
"validity_end": arrow.utcnow().shift(years=1, months=+1),
"authority": authority,
}
expected = {
"signingAlg": "SHA-2",
"eku": "SERVER_AND_CLIENT_AUTH",
"certType": "ADVANTAGE_SSL",
"certExpiryDate": arrow.get(2017, 11, 5).format('YYYY-MM-DD'),
"tracking": {
"requesterName": mock_current_app.config.get("ENTRUST_NAME"),
"requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"),
"requesterPhone": mock_current_app.config.get("ENTRUST_PHONE")
}
}
assert expected == plugin.process_options(options)

View File

@ -58,26 +58,19 @@ def create_rotation_attachments(certificate):
"title": certificate["name"],
"title_link": create_certificate_url(certificate["name"]),
"fields": [
{"title": "Owner", "value": certificate["owner"], "short": True},
{
{"title": "Owner", "value": certificate["owner"], "short": True},
{
"title": "Expires",
"value": arrow.get(certificate["validityEnd"]).format(
"dddd, MMMM D, YYYY"
),
"short": True,
},
{
"title": "Replaced By",
"value": len(certificate["replaced"][0]["name"]),
"short": True,
},
{
"title": "Endpoints Rotated",
"value": len(certificate["endpoints"]),
"short": True,
},
}
"title": "Expires",
"value": arrow.get(certificate["validityEnd"]).format(
"dddd, MMMM D, YYYY"
),
"short": True,
},
{
"title": "Endpoints Rotated",
"value": len(certificate["endpoints"]),
"short": True,
},
],
}
@ -119,6 +112,9 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
"""
A typical check can be performed using the notify command:
`lemur notify`
While we receive a `targets` parameter here, it is unused, as Slack webhooks do not allow
dynamic re-targeting of messages. The webhook itself specifies a channel.
"""
attachments = None
if notification_type == "expiration":
@ -131,7 +127,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
raise Exception("Unable to create message attachments")
body = {
"text": "Lemur {0} Notification".format(notification_type.capitalize()),
"text": f"Lemur {notification_type.capitalize()} Notification",
"attachments": attachments,
"channel": self.get_option("recipients", options),
"username": self.get_option("username", options),
@ -140,8 +136,8 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
r = requests.post(self.get_option("webhook", options), json.dumps(body))
if r.status_code not in [200]:
raise Exception("Failed to send message")
raise Exception(f"Failed to send message. Slack response: {r.status_code} {body}")
current_app.logger.error(
"Slack response: {0} Message Body: {1}".format(r.status_code, body)
current_app.logger.info(
f"Slack response: {r.status_code} Message Body: {body}"
)

View File

@ -1,3 +1,12 @@
from datetime import timedelta
import arrow
from moto import mock_ses
from lemur.tests.factories import NotificationFactory, CertificateFactory
from lemur.tests.test_messaging import verify_sender_email
def test_formatting(certificate):
from lemur.plugins.lemur_slack.plugin import create_expiration_attachments
from lemur.certificates.schemas import certificate_notification_output_schema
@ -21,3 +30,52 @@ def test_formatting(certificate):
}
assert attachment == create_expiration_attachments(data)[0]
def get_options():
return [
{"name": "interval", "value": 10},
{"name": "unit", "value": "days"},
{"name": "webhook", "value": "https://slack.com/api/api.test"},
]
@mock_ses() # because email notifications are also sent
def test_send_expiration_notification():
from lemur.notifications.messaging import send_expiration_notifications
verify_sender_email() # emails are sent to owner and security; Slack only used for configured notification
notification = NotificationFactory(plugin_name="slack-notification")
notification.options = get_options()
now = arrow.utcnow()
in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future
certificate = CertificateFactory()
certificate.not_after = in_ten_days
certificate.notifications.append(notification)
assert send_expiration_notifications([]) == (3, 0) # owner, Slack, and security
# Currently disabled as the Slack plugin doesn't support this type of notification
# def test_send_rotation_notification(endpoint, source_plugin):
# from lemur.notifications.messaging import send_rotation_notification
# from lemur.deployment.service import rotate_certificate
#
# notification = NotificationFactory(plugin_name="slack-notification")
# notification.options = get_options()
#
# new_certificate = CertificateFactory()
# rotate_certificate(endpoint, new_certificate)
# assert endpoint.certificate == new_certificate
#
# assert send_rotation_notification(new_certificate, notification_plugin=notification.plugin)
# Currently disabled as the Slack plugin doesn't support this type of notification
# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin):
# from lemur.notifications.messaging import send_pending_failure_notification
#
# assert send_pending_failure_notification(pending_certificate, notification_plugin=plugins.get("slack-notification"))