Merge branch 'master' into url_context_path

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-dist: xenial
+dist: bionic
 
 node_js:
   - "6.2.0"
@@ -47,4 +47,4 @@ after_success:
 
 notifications:
   email:
-    ccastrapel@netflix.com
+    lemur@netflix.com

docs/administration.rst

@@ -28,6 +28,13 @@ Basic Configuration
 
         LOG_FILE = "/logs/lemur/lemur-test.log"
 
+.. data:: LOG_UPGRADE_FILE
+    :noindex:
+
+    ::
+
+        LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log"
+
 .. data:: DEBUG
     :noindex:
 
@@ -269,7 +276,7 @@ Certificates marked as inactive will **not** be notified of upcoming expiration.
 silence the expiration. If a certificate is active and is expiring the above will be notified according to the `LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS` or
 30, 15, 2 days before expiration if no intervals are set.
 
-Lemur supports sending certification expiration notifications through SES and SMTP.
+Lemur supports sending certificate expiration notifications through SES and SMTP.
 
 
 .. data:: LEMUR_EMAIL_SENDER
@@ -285,6 +292,25 @@ Lemur supports sending certification expiration notifications through SES and SM
         you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
 
 
+.. data:: LEMUR_SES_SOURCE_ARN
+    :noindex:
+
+    Specifies an ARN to use as the SourceArn when sending emails via SES.
+
+    .. note::
+        This parameter is only required if you're using a sending authorization with SES.
+        See: `Using sending authorization with Amazon SES <https://docs.aws.amazon.com/ses/latest/DeveloperGuide/sending-authorization.html>`_
+
+
+.. data:: LEMUR_SES_REGION
+    :noindex:
+
+    Specifies a region for sending emails via SES.
+
+    .. note::
+        This parameter defaults to us-east-1 and is only required if you wish to use a different region.
+
+
 .. data:: LEMUR_EMAIL
     :noindex:
 
@@ -664,6 +690,20 @@ If you are not using a metric provider you do not need to configure any of these
 Plugin Specific Options
 -----------------------
 
+ACME Plugin
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. data:: ACME_DNS_PROVIDER_TYPES
+    :noindex:
+
+        Dictionary of ACME DNS Providers and their requirements.
+
+.. data:: ACME_ENABLE_DELEGATED_CNAME
+    :noindex:
+
+        Enables delegated DNS domain validation using CNAMES.  When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges.
+
+
 Active Directory Certificate Services Plugin
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1436,7 +1476,7 @@ Slack
     Adds support for slack notifications.
 
 
-AWS
+AWS (Source)
 ----
 
 :Authors:
@@ -1449,7 +1489,7 @@ AWS
     Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment.
 
 
-AWS
+AWS (Destination)
 ----
 
 :Authors:
@@ -1462,6 +1502,19 @@ AWS
     Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment.
 
 
+AWS (SNS Notification)
+-----
+
+:Authors:
+    Jasmine Schladen <jschladen@netflix.com>
+:Type:
+    Notification
+:Description:
+    Adds support for SNS notifications. SNS notifications (like other notification plugins) are currently only supported
+    for certificate expiration. Configuration requires a region, account number, and SNS topic name; these elements
+    are then combined to build the topic ARN. Lemur must have access to publish messages to the specified SNS topic.
+
+
 Kubernetes
 ----------
 

docs/developer/plugins/index.rst

@@ -215,18 +215,21 @@ Notification
 ------------
 
 Lemur includes the ability to create Email notifications by **default**. These notifications
-currently come in the form of expiration notices. Lemur periodically checks certifications expiration dates and
+currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and
 determines if a given certificate is eligible for notification. There are currently only two parameters used to
 determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number
 of days the current date (UTC) is from that expiration date.
 
-There are currently two objects that available for notification plugins the first is `NotficationPlugin`. This is the base object for
+Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable.
-any notification within Lemur. Currently the only support notification type is an certificate expiration notification. If you
+Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email.
+
+There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for
+any notification within Lemur. Currently the only supported notification type is a certificate expiration notification. If you
 are trying to create a new notification type (audit, failed logins, etc.) this would be the object to base your plugin on.
 You would also then need to build additional code to trigger the new notification type.
 
-The second is `ExpirationNotificationPlugin`, this object inherits from `NotificationPlugin` object.
+The second is `ExpirationNotificationPlugin`, which inherits from the `NotificationPlugin` object.
-You will most likely want to base your plugin on, if you want to add new channels for expiration notices (Slack, HipChat, Jira, etc.). It adds default options that are required by
+You will most likely want to base your plugin on this object if you want to add new channels for expiration notices (HipChat, Jira, etc.). It adds default options that are required by
 all expiration notifications (interval, unit). This interface expects for the child to define the following function::
 
     def send(self, notification_type, message, targets, options, **kwargs):

lemur/certificates/service.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+import re
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
@@ -560,20 +561,21 @@ def query_common_name(common_name, args):
     :return:
     """
     owner = args.pop("owner")
-    if not owner:
-        owner = "%"
-
     # only not expired certificates
     current_time = arrow.utcnow()
 
-    result = (
+    query = Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\
-        Certificate.query.filter(Certificate.cn.ilike(common_name))
+        .filter(not_(Certificate.revoked))\
-        .filter(Certificate.owner.ilike(owner))
+        .filter(not_(Certificate.replaced.any()))  # ignore rotated certificates to avoid duplicates
-        .filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))
-        .all()
-    )
 
-    return result
+    if owner:
+        query = query.filter(Certificate.owner.ilike(owner))
+
+    if common_name != "%":
+        # if common_name is a wildcard ('%'), no need to include it in the query
+        query = query.filter(Certificate.cn.ilike(common_name))
+
+    return query.all()
 
 
 def create_csr(**csr_config):
@@ -778,6 +780,19 @@ def reissue_certificate(certificate, replace=None, user=None):
     if replace:
         primitives["replaces"] = [certificate]
 
+    # Modify description to include the certificate ID being reissued and mention that this is created by Lemur
+    # as part of reissue
+    reissue_message_prefix = "Reissued by Lemur for cert ID "
+    reissue_message = re.compile(f"{reissue_message_prefix}([0-9]+)")
+    if primitives["description"]:
+        match = reissue_message.search(primitives["description"])
+        if match:
+            primitives["description"] = primitives["description"].replace(match.group(1), str(certificate.id))
+        else:
+            primitives["description"] = f"{reissue_message_prefix}{certificate.id}, {primitives['description']}"
+    else:
+        primitives["description"] = f"{reissue_message_prefix}{certificate.id}"
+
     new_cert = create(**primitives)
 
     return new_cert

lemur/certificates/views.py

@@ -1155,6 +1155,7 @@ class NotificationCertificatesList(AuthenticatedResource):
         )
         parser.add_argument("creator", type=str, location="args")
         parser.add_argument("show", type=str, location="args")
+        parser.add_argument("showExpired", type=int, location="args")
 
         args = parser.parse_args()
         args["notification_id"] = notification_id

lemur/manage.py

@@ -120,6 +120,7 @@ METRIC_PROVIDERS = []
 
 LOG_LEVEL = "DEBUG"
 LOG_FILE = "lemur.log"
+LOG_UPGRADE_FILE = "db_upgrade.log"
 
 
 # Database

lemur/migrations/versions/1db4f82bc780_.py

@@ -10,11 +10,21 @@ Create Date: 2018-08-03 12:56:44.565230
 revision = "1db4f82bc780"
 down_revision = "3adfdd6598df"
 
-import logging
-
 from alembic import op
 
-log = logging.getLogger(__name__)
+from flask import current_app
+from logging import Formatter, FileHandler, getLogger
+
+log = getLogger(__name__)
+handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
+handler.setFormatter(
+    Formatter(
+        "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+    )
+)
+handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.addHandler(handler)
 
 
 def upgrade():

lemur/migrations/versions/c301c59688d2_.py

@@ -7,8 +7,9 @@ the rest of the keys, the certificate body is parsed to determine
 the exact key_type information.
 
 Each individual DB change is explicitly committed, and the respective
-log is added to a file named db_upgrade.log in the current working
+log is added to a file configured in LOG_UPGRADE_FILE or, by default,
-directory. Any error encountered while parsing a certificate will
+to a file named db_upgrade.log in the current working directory.
+Any error encountered while parsing a certificate will
 also be logged along with the certificate ID. If faced with any issue
 while running this upgrade, there is no harm in re-running the upgrade.
 Each run processes only rows for which key_type information is not yet
@@ -31,15 +32,28 @@ down_revision = '434c29e40511'
 
 from alembic import op
 from sqlalchemy.sql import text
-from lemur.common import utils
 import time
 import datetime
+from flask import current_app
 
-log_file = open('db_upgrade.log', 'a')
+from logging import Formatter, FileHandler, getLogger
+
+from lemur.common import utils
+
+log = getLogger(__name__)
+handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
+handler.setFormatter(
+    Formatter(
+        "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+    )
+)
+handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.addHandler(handler)
 
 
 def upgrade():
-    log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
+    log.info("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
     start_time = time.time()
 
     # Update RSA keys using the key length information
@@ -50,8 +64,7 @@ def upgrade():
     # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs.
     update_key_type()
 
-    log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- Total %s seconds ---\n" % (time.time() - start_time))
-    log_file.close()
 
 
 def downgrade():
@@ -69,18 +82,18 @@ def downgrade():
 
 
 def update_key_type_rsa(bits):
-    log_file.write("Processing certificate with key type RSA %s\n" % bits)
+    log.info("Processing certificate with key type RSA %s\n" % bits)
 
     stmt = text(
         f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null"
     )
-    log_file.write("Query: %s\n" % stmt)
+    log.info("Query: %s\n" % stmt)
 
     start_time = time.time()
     op.execute(stmt)
     commit()
 
-    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- %s seconds ---\n" % (time.time() - start_time))
 
 
 def update_key_type():
@@ -95,9 +108,9 @@ def update_key_type():
         try:
             cert_key_type = utils.get_key_type_from_certificate(body)
         except ValueError as e:
-            log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
+            log.error("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
         else:
-            log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
+            log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
             stmt = text(
                 "update certificates set key_type=:key_type where id=:id"
             )
@@ -106,7 +119,7 @@ def update_key_type():
 
             commit()
 
-    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- %s seconds ---\n" % (time.time() - start_time))
 
 
 def commit():

lemur/notifications/messaging.py

@@ -30,7 +30,7 @@ from lemur.plugins.utils import get_plugin_option
 
 def get_certificates(exclude=None):
     """
-    Finds all certificates that are eligible for notifications.
+    Finds all certificates that are eligible for expiration notifications.
     :param exclude:
     :return:
     """
@@ -42,6 +42,7 @@ def get_certificates(exclude=None):
         .filter(Certificate.not_after <= max)
         .filter(Certificate.notify == True)
         .filter(Certificate.expired == False)
+        .filter(Certificate.revoked == False)
     )  # noqa
 
     exclude_conditions = []
@@ -62,7 +63,8 @@ def get_certificates(exclude=None):
 
 def get_eligible_certificates(exclude=None):
     """
-    Finds all certificates that are eligible for certificate expiration.
+    Finds all certificates that are eligible for certificate expiration notification.
+    Returns the set of all eligible certificates, grouped by owner, with a list of applicable notifications.
     :param exclude:
     :return:
     """
@@ -87,29 +89,30 @@ def get_eligible_certificates(exclude=None):
     return certificates
 
 
-def send_notification(event_type, data, targets, notification):
+def send_plugin_notification(event_type, data, recipients, notification):
     """
     Executes the plugin and handles failure.
 
     :param event_type:
     :param data:
-    :param targets:
+    :param recipients:
     :param notification:
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
     log_data = {
         "function": function,
-        "message": f"Sending expiration notification for to targets {targets}",
+        "message": f"Sending expiration notification for to recipients {recipients}",
         "notification_type": "expiration",
-        "certificate_targets": targets,
+        "certificate_targets": recipients,
     }
     status = FAILURE_METRIC_STATUS
     try:
-        notification.plugin.send(event_type, data, targets, notification.options)
+        current_app.logger.debug(log_data)
+        notification.plugin.send(event_type, data, recipients, notification.options)
         status = SUCCESS_METRIC_STATUS
     except Exception as e:
-        log_data["message"] = f"Unable to send expiration notification to targets {targets}"
+        log_data["message"] = f"Unable to send {event_type} notification to recipients {recipients}"
         current_app.logger.error(log_data, exc_info=True)
         sentry.captureException()
 
@@ -150,36 +153,27 @@ def send_expiration_notifications(exclude):
                 notification_data.append(cert_data)
                 security_data.append(cert_data)
 
-            if send_notification(
+            if send_default_notification(
-                "expiration", notification_data, [owner], notification
+                    "expiration", notification_data, [owner], notification.options
             ):
                 success += 1
             else:
                 failure += 1
 
-            notification_recipient = get_plugin_option(
+            recipients = notification.plugin.filter_recipients(notification.options, security_email + [owner])
-                "recipients", notification.options
-            )
-            if notification_recipient:
-                notification_recipient = notification_recipient.split(",")
-                # removing owner and security_email from notification_recipient
-                notification_recipient = [i for i in notification_recipient if i not in security_email and i != owner]
 
-            if (
+            if send_plugin_notification(
-                notification_recipient
-            ):
-                if send_notification(
                 "expiration",
                 notification_data,
-                    notification_recipient,
+                recipients,
                 notification,
             ):
                 success += 1
             else:
                 failure += 1
 
-            if send_notification(
+            if send_default_notification(
-                "expiration", security_data, security_email, notification
+                "expiration", security_data, security_email, notification.options
             ):
                 success += 1
             else:
@@ -188,37 +182,36 @@ def send_expiration_notifications(exclude):
     return success, failure
 
 
-def send_rotation_notification(certificate, notification_plugin=None):
+def send_default_notification(notification_type, data, targets, notification_options=None):
     """
-    Sends a report to certificate owners when their certificate has been
+    Sends a report to the specified target via the default notification plugin. Applicable for any notification_type.
-    rotated.
+    At present, "default" means email, as the other notification plugins do not support dynamically configured targets.
 
-    :param certificate:
+    :param notification_type:
-    :param notification_plugin:
+    :param data:
+    :param targets:
+    :param notification_options:
     :return:
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
     log_data = {
         "function": function,
-        "message": f"Sending rotation notification for certificate {certificate.name}",
+        "message": f"Sending notification for certificate data {data}",
-        "notification_type": "rotation",
+        "notification_type": notification_type,
-        "certificate_name": certificate.name,
-        "certificate_owner": certificate.owner,
     }
     status = FAILURE_METRIC_STATUS
-    if not notification_plugin:
     notification_plugin = plugins.get(
         current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
     )
 
-    data = certificate_notification_output_schema.dump(certificate).data
-
     try:
-        notification_plugin.send("rotation", data, [data["owner"]], [])
+        current_app.logger.debug(log_data)
+        # we need the notification.options here because the email templates utilize the interval/unit info
+        notification_plugin.send(notification_type, data, targets, notification_options)
         status = SUCCESS_METRIC_STATUS
     except Exception as e:
-        log_data["message"] = f"Unable to send rotation notification for certificate {certificate.name} " \
+        log_data["message"] = f"Unable to send {notification_type} notification for certificate data {data} " \
-                              f"to owner {data['owner']}"
+                              f"to target {targets}"
         current_app.logger.error(log_data, exc_info=True)
         sentry.captureException()
 
@@ -226,82 +219,49 @@ def send_rotation_notification(certificate, notification_plugin=None):
         "notification",
         "counter",
         1,
-        metric_tags={"status": status, "event_type": "rotation"},
+        metric_tags={"status": status, "event_type": notification_type},
     )
 
     if status == SUCCESS_METRIC_STATUS:
         return True
 
 
+def send_rotation_notification(certificate):
+    data = certificate_notification_output_schema.dump(certificate).data
+    return send_default_notification("rotation", data, [data["owner"]])
+
+
 def send_pending_failure_notification(
-    pending_cert, notify_owner=True, notify_security=True, notification_plugin=None
+    pending_cert, notify_owner=True, notify_security=True
 ):
     """
     Sends a report to certificate owners when their pending certificate failed to be created.
 
     :param pending_cert:
-    :param notification_plugin:
+    :param notify_owner:
+    :param notify_security:
     :return:
     """
-    function = f"{__name__}.{sys._getframe().f_code.co_name}"
-    log_data = {
-        "function": function,
-        "message": f"Sending pending failure notification for pending certificate {pending_cert}",
-        "notification_type": "failed",
-        "certificate_name": pending_cert.name,
-        "certificate_owner": pending_cert.owner,
-    }
-    status = FAILURE_METRIC_STATUS
-
-    if not notification_plugin:
-        notification_plugin = plugins.get(
-            current_app.config.get(
-                "LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification"
-            )
-        )
 
     data = pending_certificate_output_schema.dump(pending_cert).data
     data["security_email"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
 
+    notify_owner_success = False
     if notify_owner:
-        try:
+        notify_owner_success = send_default_notification("failed", data, [data["owner"]], pending_cert)
-            notification_plugin.send("failed", data, [data["owner"]], pending_cert)
-            status = SUCCESS_METRIC_STATUS
-        except Exception as e:
-            log_data["recipient"] = data["owner"]
-            log_data["message"] = f"Unable to send pending failure notification for certificate {pending_cert.name} " \
-                                  f"to owner {pending_cert.owner}"
-            current_app.logger.error(log_data, exc_info=True)
-            sentry.captureException()
 
+    notify_security_success = False
     if notify_security:
-        try:
+        notify_security_success = send_default_notification("failed", data, data["security_email"], pending_cert)
-            notification_plugin.send(
-                "failed", data, data["security_email"], pending_cert
-            )
-            status = SUCCESS_METRIC_STATUS
-        except Exception as e:
-            log_data["recipient"] = data["security_email"]
-            log_data["message"] = f"Unable to send pending failure notification for certificate {pending_cert.name} " \
-                                  f"to security email {pending_cert.owner}"
-            current_app.logger.error(log_data, exc_info=True)
-            sentry.captureException()
 
-    metrics.send(
+    return notify_owner_success or notify_security_success
-        "notification",
-        "counter",
-        1,
-        metric_tags={"status": status, "event_type": "failed"},
-    )
-
-    if status == SUCCESS_METRIC_STATUS:
-        return True
 
 
 def needs_notification(certificate):
     """
-    Determine if notifications for a given certificate should
+    Determine if notifications for a given certificate should currently be sent.
-    currently be sent
+    For each notification configured for the cert, verifies it is active, properly configured,
+    and that the configured expiration period is currently met.
 
     :param certificate:
     :return:
@@ -331,7 +291,6 @@ def needs_notification(certificate):
             raise Exception(
                 f"Invalid base unit for expiration interval: {unit}"
             )
-
         if days == interval:
             notifications.append(notification)
     return notifications

lemur/plugins/bases/notification.py

@@ -20,6 +20,15 @@ class NotificationPlugin(Plugin):
     def send(self, notification_type, message, targets, options, **kwargs):
         raise NotImplementedError
 
+    def filter_recipients(self, options, excluded_recipients):
+        """
+        Given a set of options (which should include configured recipient info), filters out recipients that
+        we do NOT want to notify.
+
+        For any notification types where recipients can't be dynamically modified, this returns an empty list.
+        """
+        return []
+
 
 class ExpirationNotificationPlugin(NotificationPlugin):
     """
@@ -50,5 +59,5 @@ class ExpirationNotificationPlugin(NotificationPlugin):
     def options(self):
         return self.default_options + self.additional_options
 
-    def send(self, notification_type, message, targets, options, **kwargs):
+    def send(self, notification_type, message, excluded_targets, options, **kwargs):
         raise NotImplementedError

lemur/plugins/lemur_acme/plugin.py

@@ -16,6 +16,7 @@ import json
 import time
 
 import OpenSSL.crypto
+import dns.resolver
 import josepy as jose
 from acme import challenges, errors, messages
 from acme.client import BackwardsCompatibleClientV2, ClientNetwork
@@ -23,7 +24,6 @@ from acme.errors import PollError, TimeoutError, WildcardUnsupportedError
 from acme.messages import Error as AcmeError
 from botocore.exceptions import ClientError
 from flask import current_app
-
 from lemur.authorizations import service as authorization_service
 from lemur.common.utils import generate_private_key
 from lemur.dns_providers import service as dns_provider_service
@@ -37,8 +37,9 @@ from retrying import retry
 
 
 class AuthorizationRecord(object):
-    def __init__(self, host, authz, dns_challenge, change_id):
+    def __init__(self, domain, target_domain, authz, dns_challenge, change_id):
-        self.host = host
+        self.domain = domain
+        self.target_domain = target_domain
         self.authz = authz
         self.dns_challenge = dns_challenge
         self.change_id = change_id
@@ -91,19 +92,18 @@ class AcmeHandler(object):
         self,
         acme_client,
         account_number,
-        host,
+        domain,
+        target_domain,
         dns_provider,
         order,
         dns_provider_options,
     ):
-        current_app.logger.debug("Starting DNS challenge for {0}".format(host))
+        current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.")
 
         change_ids = []
-        dns_challenges = self.get_dns_challenges(host, order.authorizations)
+        dns_challenges = self.get_dns_challenges(domain, order.authorizations)
-        host_to_validate, _ = self.strip_wildcard(host)
+        host_to_validate, _ = self.strip_wildcard(target_domain)
-        host_to_validate = self.maybe_add_extension(
+        host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
-            host_to_validate, dns_provider_options
-        )
 
         if not dns_challenges:
             sentry.captureException()
@@ -111,15 +111,20 @@ class AcmeHandler(object):
             raise Exception("Unable to determine DNS challenges from authorizations")
 
         for dns_challenge in dns_challenges:
+
+            # Only prepend '_acme-challenge' if not using CNAME redirection
+            if domain == target_domain:
+                host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
+
             change_id = dns_provider.create_txt_record(
-                dns_challenge.validation_domain_name(host_to_validate),
+                host_to_validate,
                 dns_challenge.validation(acme_client.client.net.key),
                 account_number,
             )
             change_ids.append(change_id)
 
         return AuthorizationRecord(
-            host, order.authorizations, dns_challenges, change_ids
+            domain, target_domain, order.authorizations, dns_challenges, change_ids
         )
 
     def complete_dns_challenge(self, acme_client, authz_record):
@@ -128,11 +133,11 @@ class AcmeHandler(object):
                 authz_record.authz[0].body.identifier.value
             )
         )
-        dns_providers = self.dns_providers_for_domain.get(authz_record.host)
+        dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
         if not dns_providers:
             metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1)
             raise Exception(
-                "No DNS providers found for domain: {}".format(authz_record.host)
+                "No DNS providers found for domain: {}".format(authz_record.target_domain)
             )
 
         for dns_provider in dns_providers:
@@ -160,7 +165,7 @@ class AcmeHandler(object):
 
                 verified = response.simple_verify(
                     dns_challenge.chall,
-                    authz_record.host,
+                    authz_record.target_domain,
                     acme_client.client.net.key.public_key(),
                 )
 
@@ -311,12 +316,24 @@ class AcmeHandler(object):
         authorizations = []
 
         for domain in order_info.domains:
-            if not self.dns_providers_for_domain.get(domain):
+
+            # If CNAME exists, set host to the target address
+            target_domain = domain
+            if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False):
+                cname_result, _ = self.strip_wildcard(domain)
+                cname_result = challenges.DNS01().validation_domain_name(cname_result)
+                cname_result = self.get_cname(cname_result)
+                if cname_result:
+                    target_domain = cname_result
+                    self.autodetect_dns_providers(target_domain)
+
+            if not self.dns_providers_for_domain.get(target_domain):
                 metrics.send(
                     "get_authorizations_no_dns_provider_for_domain", "counter", 1
                 )
-                raise Exception("No DNS providers found for domain: {}".format(domain))
+                raise Exception("No DNS providers found for domain: {}".format(target_domain))
-            for dns_provider in self.dns_providers_for_domain[domain]:
+
+            for dns_provider in self.dns_providers_for_domain[target_domain]:
                 dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
                 dns_provider_options = json.loads(dns_provider.credentials)
                 account_number = dns_provider_options.get("account_id")
@@ -324,6 +341,7 @@ class AcmeHandler(object):
                     acme_client,
                     account_number,
                     domain,
+                    target_domain,
                     dns_provider_plugin,
                     order,
                     dns_provider.options,
@@ -358,7 +376,7 @@ class AcmeHandler(object):
         for authz_record in authorizations:
             dns_challenges = authz_record.dns_challenge
             for dns_challenge in dns_challenges:
-                dns_providers = self.dns_providers_for_domain.get(authz_record.host)
+                dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
                 for dns_provider in dns_providers:
                     # Grab account number (For Route53)
                     dns_provider_plugin = self.get_dns_provider(
@@ -366,14 +384,14 @@ class AcmeHandler(object):
                     )
                     dns_provider_options = json.loads(dns_provider.credentials)
                     account_number = dns_provider_options.get("account_id")
-                    host_to_validate, _ = self.strip_wildcard(authz_record.host)
+                    host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
-                    host_to_validate = self.maybe_add_extension(
+                    host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
-                        host_to_validate, dns_provider_options
+                    if authz_record.domain == authz_record.target_domain:
-                    )
+                        host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate)
                     dns_provider_plugin.delete_txt_record(
                         authz_record.change_id,
                         account_number,
-                        dns_challenge.validation_domain_name(host_to_validate),
+                        host_to_validate,
                         dns_challenge.validation(acme_client.client.net.key),
                     )
 
@@ -392,23 +410,26 @@ class AcmeHandler(object):
         :return:
         """
         for authz_record in authorizations:
-            dns_providers = self.dns_providers_for_domain.get(authz_record.host)
+            dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
             for dns_provider in dns_providers:
                 # Grab account number (For Route53)
                 dns_provider_options = json.loads(dns_provider.credentials)
                 account_number = dns_provider_options.get("account_id")
                 dns_challenges = authz_record.dns_challenge
-                host_to_validate, _ = self.strip_wildcard(authz_record.host)
+                host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
                 host_to_validate = self.maybe_add_extension(
                     host_to_validate, dns_provider_options
                 )
+
                 dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
                 for dns_challenge in dns_challenges:
+                    if authz_record.domain == authz_record.target_domain:
+                        host_to_validate = dns_challenge.validation_domain_name(host_to_validate),
                     try:
                         dns_provider_plugin.delete_txt_record(
                             authz_record.change_id,
                             account_number,
-                            dns_challenge.validation_domain_name(host_to_validate),
+                            host_to_validate,
                             dns_challenge.validation(acme_client.client.net.key),
                         )
                     except Exception as e:
@@ -431,6 +452,18 @@ class AcmeHandler(object):
             raise UnknownProvider("No such DNS provider: {}".format(type))
         return provider
 
+    def get_cname(self, domain):
+        """
+        :param domain: Domain name to look up a CNAME for.
+        :return: First CNAME target or False if no CNAME record exists.
+        """
+        try:
+            result = dns.resolver.query(domain, 'CNAME')
+            if len(result) > 0:
+                return str(result[0].target).rstrip('.')
+        except dns.exception.DNSException:
+            return False
+
 
 class ACMEIssuerPlugin(IssuerPlugin):
     title = "Acme"

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -49,7 +49,7 @@ class TestAcme(unittest.TestCase):
         self.assertEqual(expected, result)
 
     def test_authz_record(self):
-        a = plugin.AuthorizationRecord("host", "authz", "challenge", "id")
+        a = plugin.AuthorizationRecord("domain", "host", "authz", "challenge", "id")
         self.assertEqual(type(a), plugin.AuthorizationRecord)
 
     @patch("acme.client.Client")
@@ -79,7 +79,7 @@ class TestAcme(unittest.TestCase):
         iterator = iter(values)
         iterable.__iter__.return_value = iterator
         result = self.acme.start_dns_challenge(
-            mock_acme, "accountid", "host", mock_dns_provider, mock_order, {}
+            mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {}
         )
         self.assertEqual(type(result), plugin.AuthorizationRecord)
 
@@ -97,7 +97,7 @@ class TestAcme(unittest.TestCase):
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True)
         mock_authz.authz = []
-        mock_authz.host = "www.test.com"
+        mock_authz.target_domain = "www.test.com"
         mock_authz_record = Mock()
         mock_authz_record.body.identifier.value = "test"
         mock_authz.authz.append(mock_authz_record)
@@ -121,7 +121,7 @@ class TestAcme(unittest.TestCase):
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False)
         mock_authz.authz = []
-        mock_authz.host = "www.test.com"
+        mock_authz.target_domain = "www.test.com"
         mock_authz_record = Mock()
         mock_authz_record.body.identifier.value = "test"
         mock_authz.authz.append(mock_authz_record)
@@ -270,11 +270,9 @@ class TestAcme(unittest.TestCase):
             result, [options["common_name"], "test2.netflix.net"]
         )
 
-    @patch(
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", return_value="test")
-        "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge",
+    @patch("lemur.plugins.lemur_acme.plugin.current_app", return_value=False)
-        return_value="test",
+    def test_get_authorizations(self, mock_current_app, mock_start_dns_challenge):
-    )
-    def test_get_authorizations(self, mock_start_dns_challenge):
         mock_order = Mock()
         mock_order.body.identifiers = []
         mock_domain = Mock()

lemur/plugins/lemur_aws/plugin.py

@@ -32,13 +32,14 @@
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 .. moduleauthor:: Harm Weites <harm@weites.com>
 """
+
 from acme.errors import ClientError
 from flask import current_app
-from lemur.extensions import sentry, metrics
 
-from lemur.plugins import lemur_aws as aws
+from lemur.extensions import sentry, metrics
+from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin
 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
-from lemur.plugins.lemur_aws import iam, s3, elb, ec2
+from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns
 
 
 def get_region_from_dns(dns):
@@ -406,3 +407,51 @@ class S3DestinationPlugin(ExportDestinationPlugin):
                 self.get_option("encrypt", options),
                 account_number=self.get_option("accountNumber", options),
             )
+
+
+class SNSNotificationPlugin(ExpirationNotificationPlugin):
+    title = "AWS SNS"
+    slug = "aws-sns"
+    description = "Sends notifications to AWS SNS"
+    version = aws.VERSION
+
+    author = "Jasmine Schladen <jschladen@netflix.com>"
+    author_url = "https://github.com/Netflix/lemur"
+
+    additional_options = [
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "required": True,
+            "validation": "[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access the SNS topic",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "required": True,
+            "validation": "[0-9a-z\\-]{1,25}",
+            "helpMessage": "Region in which the SNS topic is located, e.g. \"us-east-1\"",
+        },
+        {
+            "name": "topicName",
+            "type": "str",
+            "required": True,
+            # base topic name is 1-256 characters (alphanumeric plus underscore and hyphen)
+            "validation": "^[a-zA-Z0-9_\\-]{1,256}$",
+            "helpMessage": "The name of the topic to use for expiration notifications",
+        }
+    ]
+
+    def send(self, notification_type, message, excluded_targets, options, **kwargs):
+        """
+        While we receive a `targets` parameter here, it is unused, as the SNS topic is pre-configured in the
+        plugin configuration, and can't reasonably be changed dynamically.
+        """
+
+        topic_arn = f"arn:aws:sns:{self.get_option('region', options)}:" \
+                    f"{self.get_option('accountNumber', options)}:" \
+                    f"{self.get_option('topicName', options)}"
+
+        current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}")
+        sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options))

lemur/plugins/lemur_aws/sns.py

@@ -0,0 +1,58 @@
+"""
+.. module: lemur.plugins.lemur_aws.sts
+    :platform: Unix
+    :copyright: (c) 2020 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Jasmine Schladen <jschladen@netflix.com>
+"""
+import json
+
+import arrow
+import boto3
+from flask import current_app
+
+
+def publish(topic_arn, certificates, notification_type, **kwargs):
+    sns_client = boto3.client("sns", **kwargs)
+    message_ids = {}
+    subject = "Lemur: {0} Notification".format(notification_type.capitalize())
+    for certificate in certificates:
+        message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject)
+
+    return message_ids
+
+
+def publish_single(sns_client, topic_arn, certificate, notification_type, subject):
+    response = sns_client.publish(
+        TopicArn=topic_arn,
+        Message=format_message(certificate, notification_type),
+        Subject=subject,
+    )
+
+    response_code = response["ResponseMetadata"]["HTTPStatusCode"]
+    if response_code != 200:
+        raise Exception(f"Failed to publish {notification_type} notification to SNS topic {topic_arn}. "
+                        f"SNS response: {response_code} {response}")
+
+    current_app.logger.info(f"AWS SNS message published to topic [{topic_arn}] with message ID {response['MessageId']}")
+    current_app.logger.debug(f"AWS SNS message published to topic [{topic_arn}]: [{response}]")
+
+    return response["MessageId"]
+
+
+def create_certificate_url(name):
+    return "https://{hostname}/#/certificates/{name}".format(
+        hostname=current_app.config.get("LEMUR_HOSTNAME"), name=name
+    )
+
+
+def format_message(certificate, notification_type):
+    json_message = {
+        "notification_type": notification_type,
+        "certificate_name": certificate["name"],
+        "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),  # 2047-12-31T22:00:00
+        "endpoints_detected": len(certificate["endpoints"]),
+        "owner": certificate["owner"],
+        "details": create_certificate_url(certificate["name"])
+    }
+    return json.dumps(json_message)

lemur/plugins/lemur_aws/tests/test_sns.py

@@ -0,0 +1,123 @@
+import json
+from datetime import timedelta
+
+import arrow
+import boto3
+from moto import mock_sns, mock_sqs, mock_ses
+
+from lemur.certificates.schemas import certificate_notification_output_schema
+from lemur.plugins.lemur_aws.sns import format_message
+from lemur.plugins.lemur_aws.sns import publish
+from lemur.tests.factories import NotificationFactory, CertificateFactory
+from lemur.tests.test_messaging import verify_sender_email
+
+
+@mock_sns()
+def test_format(certificate, endpoint):
+    data = [certificate_notification_output_schema.dump(certificate).data]
+
+    for certificate in data:
+        expected_message = {
+            "notification_type": "expiration",
+            "certificate_name": certificate["name"],
+            "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
+            "endpoints_detected": 0,
+            "owner": certificate["owner"],
+            "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
+        }
+        assert expected_message == json.loads(format_message(certificate, "expiration"))
+
+
+@mock_sns()
+@mock_sqs()
+def create_and_subscribe_to_topic():
+    sns_client = boto3.client("sns", region_name="us-east-1")
+    topic_arn = sns_client.create_topic(Name='lemursnstest')["TopicArn"]
+
+    sqs_client = boto3.client("sqs", region_name="us-east-1")
+    queue = sqs_client.create_queue(QueueName="lemursnstestqueue")
+    queue_url = queue["QueueUrl"]
+    queue_arn = sqs_client.get_queue_attributes(QueueUrl=queue_url)["Attributes"]["QueueArn"]
+    sns_client.subscribe(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_arn)
+
+    return [topic_arn, sqs_client, queue_url]
+
+
+@mock_sns()
+@mock_sqs()
+def test_publish(certificate, endpoint):
+    data = [certificate_notification_output_schema.dump(certificate).data]
+
+    topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic()
+
+    message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1")
+    assert len(message_ids) == len(data)
+    received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
+
+    for certificate in data:
+        expected_message_id = message_ids[certificate["name"]]
+        actual_message = next(
+            (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None)
+        actual_json = json.loads(actual_message["Body"])
+        assert actual_json["Message"] == format_message(certificate, "expiration")
+        assert actual_json["Subject"] == "Lemur: Expiration Notification"
+
+
+def get_options():
+    return [
+        {"name": "interval", "value": 10},
+        {"name": "unit", "value": "days"},
+        {"name": "region", "value": "us-east-1"},
+        {"name": "accountNumber", "value": "123456789012"},
+        {"name": "topicName", "value": "lemursnstest"},
+    ]
+
+
+@mock_sns()
+@mock_sqs()
+@mock_ses()  # because email notifications are also sent
+def test_send_expiration_notification():
+    from lemur.notifications.messaging import send_expiration_notifications
+
+    verify_sender_email()  # emails are sent to owner and security; SNS only used for configured notification
+    topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic()
+
+    notification = NotificationFactory(plugin_name="aws-sns")
+    notification.options = get_options()
+
+    now = arrow.utcnow()
+    in_ten_days = now + timedelta(days=10, hours=1)  # a bit more than 10 days since we'll check in the future
+
+    certificate = CertificateFactory()
+    certificate.not_after = in_ten_days
+    certificate.notifications.append(notification)
+
+    assert send_expiration_notifications([]) == (3, 0)  # owner, SNS, and security
+
+    received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
+    assert len(received_messages) == 1
+    expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration")
+    actual_message = json.loads(received_messages[0]["Body"])["Message"]
+    assert actual_message == expected_message
+
+
+# Currently disabled as the SNS plugin doesn't support this type of notification
+# def test_send_rotation_notification(endpoint, source_plugin):
+#     from lemur.notifications.messaging import send_rotation_notification
+#     from lemur.deployment.service import rotate_certificate
+#
+#     notification = NotificationFactory(plugin_name="aws-sns")
+#     notification.options = get_options()
+#
+#     new_certificate = CertificateFactory()
+#     rotate_certificate(endpoint, new_certificate)
+#     assert endpoint.certificate == new_certificate
+#
+#     assert send_rotation_notification(new_certificate)
+
+
+# Currently disabled as the SNS plugin doesn't support this type of notification
+# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin):
+#     from lemur.notifications.messaging import send_pending_failure_notification
+#
+#     assert send_pending_failure_notification(pending_certificate)

lemur/plugins/lemur_digicert/plugin.py

@@ -234,7 +234,7 @@ def handle_cis_response(response):
         return response.json()
 
 
-@retry(stop_max_attempt_number=10, wait_fixed=10000)
+@retry(stop_max_attempt_number=10, wait_fixed=1000)
 def get_certificate_id(session, base_url, order_id):
     """Retrieve certificate order id from Digicert API."""
     order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id)
@@ -245,7 +245,7 @@ def get_certificate_id(session, base_url, order_id):
     return response_data["certificate"]["id"]
 
 
-@retry(stop_max_attempt_number=10, wait_fixed=10000)
+@retry(stop_max_attempt_number=10, wait_fixed=1000)
 def get_cis_certificate(session, base_url, order_id):
     """Retrieve certificate order id from Digicert API, including the chain"""
     certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id)

lemur/plugins/lemur_email/plugin.py

@@ -17,6 +17,7 @@ from lemur.plugins.bases import ExpirationNotificationPlugin
 from lemur.plugins import lemur_email as email
 
 from lemur.plugins.lemur_email.templates.config import env
+from lemur.plugins.utils import get_plugin_option
 
 
 def render_html(template_name, options, certificates):
@@ -37,7 +38,7 @@ def render_html(template_name, options, certificates):
 
 def send_via_smtp(subject, body, targets):
     """
-    Attempts to deliver email notification via SES service.
+    Attempts to deliver email notification via SMTP.
 
     :param subject:
     :param body:
@@ -54,21 +55,26 @@ def send_via_smtp(subject, body, targets):
 
 def send_via_ses(subject, body, targets):
     """
-    Attempts to deliver email notification via SMTP.
+    Attempts to deliver email notification via SES service.
     :param subject:
     :param body:
     :param targets:
     :return:
     """
-    client = boto3.client("ses", region_name="us-east-1")
+    ses_region = current_app.config.get("LEMUR_SES_REGION", "us-east-1")
-    client.send_email(
+    client = boto3.client("ses", region_name=ses_region)
-        Source=current_app.config.get("LEMUR_EMAIL"),
+    source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN")
-        Destination={"ToAddresses": targets},
+    args = {
-        Message={
+        "Source": current_app.config.get("LEMUR_EMAIL"),
+        "Destination": {"ToAddresses": targets},
+        "Message": {
             "Subject": {"Data": subject, "Charset": "UTF-8"},
             "Body": {"Html": {"Data": body, "Charset": "UTF-8"}},
         },
-    )
+    }
+    if source_arn:
+        args["SourceArn"] = source_arn
+    client.send_email(**args)
 
 
 class EmailNotificationPlugin(ExpirationNotificationPlugin):
@@ -111,3 +117,13 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
 
         elif s_type == "smtp":
             send_via_smtp(subject, body, targets)
+
+    @staticmethod
+    def filter_recipients(options, excluded_recipients, **kwargs):
+        notification_recipients = get_plugin_option("recipients", options)
+        if notification_recipients:
+            notification_recipients = notification_recipients.split(",")
+            # removing owner and security_email from notification_recipient
+            notification_recipients = [i for i in notification_recipients if i not in excluded_recipients]
+
+        return notification_recipients

lemur/plugins/lemur_email/tests/test_email.py

@@ -2,26 +2,21 @@ import os
 from datetime import timedelta
 
 import arrow
-import boto3
 from moto import mock_ses
 
 from lemur.certificates.schemas import certificate_notification_output_schema
 from lemur.plugins.lemur_email.plugin import render_html
 from lemur.tests.factories import CertificateFactory
+from lemur.tests.test_messaging import verify_sender_email
 
 dir_path = os.path.dirname(os.path.realpath(__file__))
 
 
-@mock_ses
-def verify_sender_email():
-    ses_client = boto3.client("ses", region_name="us-east-1")
-    ses_client.verify_email_identity(EmailAddress="lemur@example.com")
-
-
 def get_options():
     return [
         {"name": "interval", "value": 10},
         {"name": "unit", "value": "days"},
+        {"name": "recipients", "value": "person1@example.com,person2@example.com"},
     ]
 
 
@@ -59,7 +54,7 @@ def test_send_expiration_notification():
     certificate.notifications[0].options = get_options()
 
     verify_sender_email()
-    assert send_expiration_notifications([]) == (2, 0)
+    assert send_expiration_notifications([]) == (3, 0)  # owner, recipients (only counted as 1), and security
 
 
 @mock_ses
@@ -81,3 +76,15 @@ def test_send_pending_failure_notification(user, pending_certificate, async_issu
 
     verify_sender_email()
     assert send_pending_failure_notification(pending_certificate)
+
+
+def test_filter_recipients(certificate, endpoint):
+    from lemur.plugins.lemur_email.plugin import EmailNotificationPlugin
+
+    options = [{"name": "recipients", "value": "security@example.com,bob@example.com,joe@example.com"}]
+    assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@example.com", "bob@example.com",
+                                                                      "joe@example.com"]
+    assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com"]) == ["bob@example.com",
+                                                                                            "joe@example.com"]
+    assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com", "bob@example.com",
+                                                               "joe@example.com"]) == []

lemur/plugins/lemur_entrust/plugin.py

@@ -1,9 +1,9 @@
-
 import arrow
 import requests
 import json
 import sys
 from flask import current_app
+from retrying import retry
 
 from lemur.plugins import lemur_entrust as entrust
 from lemur.plugins.bases import IssuerPlugin, SourcePlugin
@@ -78,7 +78,6 @@ def process_options(options):
         "eku": "SERVER_AND_CLIENT_AUTH",
         "certType": product_type,
         "certExpiryDate": validity_end,
-        # "keyType": "RSA", Entrust complaining about this parameter
         "tracking": tracking_data
     }
     return data
@@ -87,7 +86,7 @@ def process_options(options):
 def handle_response(my_response):
     """
     Helper function for parsing responses from the Entrust API.
-    :param content:
+    :param my_response:
     :return: :raise Exception:
     """
     msg = {
@@ -100,27 +99,47 @@ def handle_response(my_response):
     }
 
     try:
-        d = json.loads(my_response.content)
+        data = json.loads(my_response.content)
     except ValueError:
         # catch an empty jason object here
-        d = {'response': 'No detailed message'}
+        data = {'response': 'No detailed message'}
-    s = my_response.status_code
+    status_code = my_response.status_code
-    if s > 399:
+    if status_code > 399:
-        raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}")
+        raise Exception(f"ENTRUST error: {msg.get(status_code, status_code)}\n{data['errors']}")
 
     log_data = {
         "function": f"{__name__}.{sys._getframe().f_code.co_name}",
         "message": "Response",
-        "status": s,
+        "status": status_code,
-        "response": d
+        "response": data
     }
     current_app.logger.info(log_data)
-    if d == {'response': 'No detailed message'}:
+    if data == {'response': 'No detailed message'}:
         # status if no data
-        return s
+        return status_code
     else:
         #  return data from the response
-        return d
+        return data
+
+
+@retry(stop_max_attempt_number=3, wait_fixed=5000)
+def order_and_download_certificate(session, url, data):
+    """
+    Helper function to place a certificacte order and download it
+    :param session:
+    :param url: Entrust endpoint url
+    :param data: CSR, and the required order details, such as validity length
+    :return: the cert chain
+    :raise Exception:
+    """
+    try:
+        response = session.post(url, json=data, timeout=(15, 40))
+    except requests.exceptions.Timeout:
+        raise Exception("Timeout for POST")
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Error for POST {e}")
+
+    return handle_response(response)
 
 
 class EntrustIssuerPlugin(IssuerPlugin):
@@ -178,14 +197,8 @@ class EntrustIssuerPlugin(IssuerPlugin):
         data = process_options(issuer_options)
         data["csr"] = csr
 
-        try:
+        response_dict = order_and_download_certificate(self.session, url, data)
-            response = self.session.post(url, json=data, timeout=(15, 40))
-        except requests.exceptions.Timeout:
-            raise Exception("Timeout for POST")
-        except requests.exceptions.RequestException as e:
-            raise Exception(f"Error for POST {e}")
 
-        response_dict = handle_response(response)
         external_id = response_dict['trackingId']
         cert = response_dict['endEntityCert']
         if len(response_dict['chainCerts']) < 2:
@@ -200,6 +213,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
 
         return cert, chain, external_id
 
+    @retry(stop_max_attempt_number=3, wait_fixed=1000)
     def revoke_certificate(self, certificate, comments):
         """Revoke an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
@@ -216,6 +230,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
         metrics.send("entrust_revoke_certificate", "counter", 1)
         return handle_response(response)
 
+    @retry(stop_max_attempt_number=3, wait_fixed=1000)
     def deactivate_certificate(self, certificate):
         """Deactivates an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
@@ -244,7 +259,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
     def get_ordered_certificate(self, order_id):
         raise NotImplementedError("Not implemented\n", self, order_id)
 
-    def canceled_ordered_certificate(self, pending_cert, **kwargs):
+    def cancel_ordered_certificate(self, pending_cert, **kwargs):
         raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs)
 
 

lemur/plugins/lemur_slack/plugin.py

@@ -112,6 +112,9 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
         """
         A typical check can be performed using the notify command:
         `lemur notify`
+
+        While we receive a `targets` parameter here, it is unused, as Slack webhooks do not allow
+        dynamic re-targeting of messages. The webhook itself specifies a channel.
         """
         attachments = None
         if notification_type == "expiration":
@@ -124,7 +127,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
             raise Exception("Unable to create message attachments")
 
         body = {
-            "text": "Lemur {0} Notification".format(notification_type.capitalize()),
+            "text": f"Lemur {notification_type.capitalize()} Notification",
             "attachments": attachments,
             "channel": self.get_option("recipients", options),
             "username": self.get_option("username", options),
@@ -133,8 +136,8 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
         r = requests.post(self.get_option("webhook", options), json.dumps(body))
 
         if r.status_code not in [200]:
-            raise Exception("Failed to send message")
+            raise Exception(f"Failed to send message. Slack response: {r.status_code} {body}")
 
-        current_app.logger.error(
+        current_app.logger.info(
-            "Slack response: {0} Message Body: {1}".format(r.status_code, body)
+            f"Slack response: {r.status_code} Message Body: {body}"
         )

lemur/plugins/lemur_slack/tests/test_slack.py

@@ -1,8 +1,10 @@
 from datetime import timedelta
 
 import arrow
+from moto import mock_ses
 
 from lemur.tests.factories import NotificationFactory, CertificateFactory
+from lemur.tests.test_messaging import verify_sender_email
 
 
 def test_formatting(certificate):
@@ -38,9 +40,12 @@ def get_options():
     ]
 
 
+@mock_ses()  # because email notifications are also sent
 def test_send_expiration_notification():
     from lemur.notifications.messaging import send_expiration_notifications
 
+    verify_sender_email()  # emails are sent to owner and security; Slack only used for configured notification
+
     notification = NotificationFactory(plugin_name="slack-notification")
     notification.options = get_options()
 
@@ -51,7 +56,7 @@ def test_send_expiration_notification():
     certificate.not_after = in_ten_days
     certificate.notifications.append(notification)
 
-    assert send_expiration_notifications([]) == (2, 0)
+    assert send_expiration_notifications([]) == (3, 0)  # owner, Slack, and security
 
 
 # Currently disabled as the Slack plugin doesn't support this type of notification

lemur/static/app/angular/notifications/services.js

@@ -27,7 +27,7 @@ angular.module('lemur')
     };
 
     NotificationService.getCertificates = function (notification) {
-      notification.getList('certificates').then(function (certificates) {
+      notification.getList('certificates', {showExpired: 0}).then(function (certificates) {
         notification.certificates = certificates;
       });
     };
@@ -40,7 +40,7 @@ angular.module('lemur')
 
 
     NotificationService.loadMoreCertificates = function (notification, page) {
-      notification.getList('certificates', {page: page}).then(function (certificates) {
+      notification.getList('certificates', {page: page, showExpired: 0}).then(function (certificates) {
         _.each(certificates, function (certificate) {
           notification.roles.push(certificate);
         });

lemur/tests/test_certificates.py

@@ -802,6 +802,7 @@ def test_reissue_certificate(
     assert new_cert.organization != certificate.organization
     # Check for default value since authority does not have cab_compliant option set
     assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION
+    assert new_cert.description.startswith(f"Reissued by Lemur for cert ID {certificate.id}")
 
     # update cab_compliant option to false for crypto_authority to maintain subject details
     update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]')

lemur/tests/test_messaging.py

@@ -1,11 +1,18 @@
 from datetime import timedelta
 
 import arrow
+import boto3
 import pytest
 from freezegun import freeze_time
 from moto import mock_ses
 
 
+@mock_ses
+def verify_sender_email():
+    ses_client = boto3.client("ses", region_name="us-east-1")
+    ses_client.verify_email_identity(EmailAddress="lemur@example.com")
+
+
 def test_needs_notification(app, certificate, notification):
     from lemur.notifications.messaging import needs_notification
 
@@ -78,6 +85,7 @@ def test_get_eligible_certificates(app, certificate, notification):
 @mock_ses
 def test_send_expiration_notification(certificate, notification, notification_plugin):
     from lemur.notifications.messaging import send_expiration_notifications
+    verify_sender_email()
 
     certificate.notifications.append(notification)
     certificate.notifications[0].options = [
@@ -87,7 +95,9 @@ def test_send_expiration_notification(certificate, notification, notification_pl
 
     delta = certificate.not_after - timedelta(days=10)
     with freeze_time(delta.datetime):
-        assert send_expiration_notifications([]) == (2, 0)
+        # this will only send owner and security emails (no additional recipients),
+        # but it executes 3 successful send attempts
+        assert send_expiration_notifications([]) == (3, 0)
 
 
 @mock_ses
@@ -104,12 +114,14 @@ def test_send_expiration_notification_with_no_notifications(
 @mock_ses
 def test_send_rotation_notification(notification_plugin, certificate):
     from lemur.notifications.messaging import send_rotation_notification
+    verify_sender_email()
 
-    assert send_rotation_notification(certificate, notification_plugin=notification_plugin)
+    assert send_rotation_notification(certificate)
 
 
 @mock_ses
 def test_send_pending_failure_notification(notification_plugin, async_issuer_plugin, pending_certificate):
     from lemur.notifications.messaging import send_pending_failure_notification
+    verify_sender_email()
 
-    assert send_pending_failure_notification(pending_certificate, notification_plugin=notification_plugin)
+    assert send_pending_failure_notification(pending_certificate)

setup.py

@@ -135,6 +135,7 @@ setup(
             'aws_destination = lemur.plugins.lemur_aws.plugin:AWSDestinationPlugin',
             'aws_source = lemur.plugins.lemur_aws.plugin:AWSSourcePlugin',
             'aws_s3 = lemur.plugins.lemur_aws.plugin:S3DestinationPlugin',
+            'aws_sns = lemur.plugins.lemur_aws.plugin:SNSNotificationPlugin',
             'email_notification = lemur.plugins.lemur_email.plugin:EmailNotificationPlugin',
             'slack_notification = lemur.plugins.lemur_slack.plugin:SlackNotificationPlugin',
             'java_truststore_export = lemur.plugins.lemur_jks.plugin:JavaTruststoreExportPlugin',