Merge branch 'master' into master

lemur/acme_providers/cli.py

@@ -1,12 +1,15 @@
 import time
 import json
+import arrow
 
 from flask_script import Manager
 from flask import current_app
 
 from lemur.extensions import sentry
 from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.plugins import plugins
 from lemur.plugins.lemur_acme.plugin import AcmeHandler
+from lemur.plugins.lemur_aws import s3
 
 manager = Manager(
     usage="Handles all ACME related tasks"
@@ -84,3 +87,105 @@ def dnstest(domain, token):
 
     status = SUCCESS_METRIC_STATUS
     print("[+] Done with ACME Tests.")
+
+
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="Value of the Token",
+)
+@manager.option(
+    "-n",
+    "--token_name",
+    dest="token_name",
+    default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="path",
+)
+@manager.option(
+    "-p",
+    "--prefix",
+    dest="prefix",
+    default="test/",
+    required=False,
+    help="S3 bucket prefix",
+)
+@manager.option(
+    "-a",
+    "--account_number",
+    dest="account_number",
+    required=True,
+    help="AWS Account",
+)
+@manager.option(
+    "-b",
+    "--bucket_name",
+    dest="bucket_name",
+    required=True,
+    help="Bucket Name",
+)
+def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
+    """
+    This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
+    It mainly serves for testing purposes.
+    :param token:
+    :param token_name:
+    :param prefix:
+    :param account_number:
+    :param bucket_name:
+    :return:
+    """
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket_name,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account_number,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    p = plugins.get("aws-s3")
+    p.upload_acme_token(token_name, token, additional_options)
+
+    if not prefix.endswith("/"):
+        prefix + "/"
+
+    token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
+    assert(token_res == token)
+    s3.delete(bucket_name, prefix + token_name, account_number=account_number)

lemur/auth/ldap.py

@@ -210,7 +210,8 @@ class LdapPrincipal:
             self.ldap_groups = []
             for group in lgroups:
                 (dn, values) = group
-                self.ldap_groups.append(values["cn"][0].decode("ascii"))
+                if type(values) == dict:
+                    self.ldap_groups.append(values["cn"][0].decode("ascii"))
         else:
             lgroups = self.ldap_client.search_s(
                 self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs

lemur/auth/service.py

@@ -101,7 +101,8 @@ def login_required(f):
             return dict(message="Token is invalid"), 403
 
         try:
-            payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"])
+            header_data = fetch_token_header(token)
+            payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"], algorithms=[header_data["alg"]])
         except jwt.DecodeError:
             return dict(message="Token is invalid"), 403
         except jwt.ExpiredSignatureError:

lemur/authorities/models.py

@@ -6,6 +6,9 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+import json
+
+from flask import current_app
 from sqlalchemy.orm import relationship
 from sqlalchemy import (
     Column,
@@ -15,7 +18,7 @@ from sqlalchemy import (
     func,
     ForeignKey,
     DateTime,
-    PassiveDefault,
+    DefaultClause,
     Boolean,
 )
 from sqlalchemy.dialects.postgresql import JSON
@@ -36,7 +39,7 @@ class Authority(db.Model):
     plugin_name = Column(String(64))
     description = Column(Text)
     options = Column(JSON)
-    date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False)
+    date_created = Column(DateTime, DefaultClause(func.now()), nullable=False)
     roles = relationship(
         "Role",
         secondary=roles_authorities,
@@ -80,5 +83,35 @@ class Authority(db.Model):
     def plugin(self):
         return plugins.get(self.plugin_name)
 
+    @property
+    def is_cab_compliant(self):
+        """
+        Parse the options to find whether authority is CAB Forum Compliant,
+        i.e., adhering to the CA/Browser Forum Baseline Requirements.
+        Returns None if option is not available
+        """
+        if not self.options:
+            return None
+
+        options_array = json.loads(self.options)
+        if isinstance(options_array, list):
+            for option in options_array:
+                if "name" in option and option["name"] == 'cab_compliant':
+                    return option["value"]
+
+        return None
+
+    @property
+    def max_issuance_days(self):
+        if self.is_cab_compliant:
+            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
+
+    @property
+    def default_validity_days(self):
+        if self.is_cab_compliant:
+            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
+
+        return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365)  # 1 year default
+
     def __repr__(self):
         return "Authority(name={name})".format(name=self.name)

lemur/authorities/schemas.py

@@ -111,8 +111,6 @@ class RootAuthorityCertificateOutputSchema(LemurOutputSchema):
     cn = fields.String()
     not_after = fields.DateTime()
     not_before = fields.DateTime()
-    max_issuance_days = fields.Integer()
-    default_validity_days = fields.Integer()
     owner = fields.Email()
     status = fields.Boolean()
     user = fields.Nested(UserNestedOutputSchema)
@@ -127,6 +125,8 @@ class AuthorityOutputSchema(LemurOutputSchema):
     active = fields.Boolean()
     options = fields.Dict()
     roles = fields.List(fields.Nested(AssociatedRoleSchema))
+    max_issuance_days = fields.Integer()
+    default_validity_days = fields.Integer()
     authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema)
 
 
@@ -138,7 +138,10 @@ class AuthorityNestedOutputSchema(LemurOutputSchema):
     owner = fields.Email()
     plugin = fields.Nested(PluginOutputSchema)
     active = fields.Boolean()
-    authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["max_issuance_days", "default_validity_days"])
+    authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["not_after", "not_before"])
+    is_cab_compliant = fields.Boolean()
+    max_issuance_days = fields.Integer()
+    default_validity_days = fields.Integer()
 
 
 authority_update_schema = AuthorityUpdateSchema()

lemur/authorities/service.py

@@ -39,6 +39,22 @@ def update(authority_id, description, owner, active, roles):
     return database.update(authority)
 
 
+def update_options(authority_id, options):
+    """
+    Update an authority with new options.
+
+    :param authority_id:
+    :param options: the new options to be saved into the authority
+    :return:
+    """
+
+    authority = get(authority_id)
+
+    authority.options = options
+
+    return database.update(authority)
+
+
 def mint(**kwargs):
     """
     Creates the authority based on the plugin provided.

lemur/certificates/cli.py

@@ -735,3 +735,45 @@ def automatically_enable_autorotate():
                                   })
         cert.rotation = True
         database.update(cert)
+
+
+@manager.command
+def deactivate_entrust_certificates():
+    """
+    Attempt to deactivate test certificates issued by Entrust
+    """
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Deactivating Entrust certificates"
+    }
+
+    certificates = get_all_valid_certs(['entrust-issuer'])
+    entrust_plugin = plugins.get('entrust-issuer')
+    for cert in certificates:
+        try:
+            response = entrust_plugin.deactivate_certificate(cert)
+            if response == 200:
+                cert.status = "revoked"
+            else:
+                cert.status = "unknown"
+
+            log_data["valid"] = cert.status
+            log_data["certificate_name"] = cert.name
+            log_data["certificate_id"] = cert.id
+            metrics.send(
+                "certificate_deactivate",
+                "counter",
+                1,
+                metric_tags={"status": log_data["valid"],
+                             "certificate_name": log_data["certificate_name"],
+                             "certificate_id": log_data["certificate_id"]},
+            )
+            current_app.logger.info(log_data)
+
+            database.update(cert)
+
+        except Exception as e:
+            current_app.logger.info(log_data)
+            sentry.captureException()
+            current_app.logger.exception(e)

lemur/certificates/models.py

@@ -16,7 +16,7 @@ from sqlalchemy import (
     Integer,
     ForeignKey,
     String,
-    PassiveDefault,
+    DefaultClause,
     func,
     Column,
     Text,
@@ -138,7 +138,7 @@ class Certificate(db.Model):
     not_after = Column(ArrowType)
     not_after_ix = Index("ix_certificates_not_after", not_after.desc())
 
-    date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
+    date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False)
 
     signing_algorithm = Column(String(128))
     status = Column(String(128))
@@ -184,7 +184,6 @@ class Certificate(db.Model):
         "PendingCertificate",
         secondary=pending_cert_replacement_associations,
         backref="pending_replace",
-        viewonly=True,
     )
 
     logs = relationship("Log", backref="certificate")
@@ -317,20 +316,6 @@ class Certificate(db.Model):
     def validity_range(self):
         return self.not_after - self.not_before
 
-    @property
-    def max_issuance_days(self):
-        public_CA = current_app.config.get("PUBLIC_CA_AUTHORITY_NAMES", [])
-        if self.name.lower() in [ca.lower() for ca in public_CA]:
-            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
-
-    @property
-    def default_validity_days(self):
-        public_CA = current_app.config.get("PUBLIC_CA_AUTHORITY_NAMES", [])
-        if self.name.lower() in [ca.lower() for ca in public_CA]:
-            return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
-
-        return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365)   # 1 year default
-
     @property
     def subject(self):
         return self.parsed_cert.subject

lemur/certificates/schemas.py

@@ -8,7 +8,7 @@
 from flask import current_app
 from flask_restful import inputs
 from flask_restful.reqparse import RequestParser
-from marshmallow import fields, validate, validates_schema, post_load, pre_load
+from marshmallow import fields, validate, validates_schema, post_load, pre_load, post_dump
 from marshmallow.exceptions import ValidationError
 
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
@@ -23,6 +23,7 @@ from lemur.domains.schemas import DomainNestedOutputSchema
 from lemur.notifications import service as notification_service
 from lemur.notifications.schemas import NotificationNestedOutputSchema
 from lemur.policies.schemas import RotationPolicyNestedOutputSchema
+from lemur.roles import service as roles_service
 from lemur.roles.schemas import RoleNestedOutputSchema
 from lemur.schemas import (
     AssociatedAuthoritySchema,
@@ -184,25 +185,52 @@ class CertificateEditInputSchema(CertificateSchema):
             data["replaces"] = data[
                 "replacements"
             ]  # TODO remove when field is deprecated
+
+        if data.get("owner"):
+            # Check if role already exists. This avoids adding duplicate role.
+            if data.get("roles") and any(r.get("name") == data["owner"] for r in data["roles"]):
+                return data
+
+            # Add required role
+            owner_role = roles_service.get_or_create(
+                data["owner"],
+                description=f"Auto generated role based on owner: {data['owner']}"
+            )
+
+            # Put  role info in correct format using RoleNestedOutputSchema
+            owner_role_dict = RoleNestedOutputSchema().dump(owner_role).data
+            if data.get("roles"):
+                data["roles"].append(owner_role_dict)
+            else:
+                data["roles"] = [owner_role_dict]
+
         return data
 
     @post_load
     def enforce_notifications(self, data):
         """
-        Ensures that when an owner changes, default notifications are added for the new owner.
-        Old owner notifications are retained unless explicitly removed.
+        Add default notification for current owner if none exist.
+        This ensures that the default notifications are added in the event of owner change.
+        Old owner notifications are retained unless explicitly removed later in the code path.
         :param data:
         :return:
         """
-        if data["owner"]:
+        if data.get("owner"):
             notification_name = "DEFAULT_{0}".format(
                 data["owner"].split("@")[0].upper()
             )
+
+            # Even if one default role exists, return
+            # This allows a User to remove unwanted default notification for current owner
+            if any(n.label.startswith(notification_name) for n in data["notifications"]):
+                return data
+
             data[
                 "notifications"
             ] += notification_service.create_default_expiration_notifications(
                 notification_name, [data["owner"]]
             )
+
         return data
 
 
@@ -304,6 +332,31 @@ class CertificateOutputSchema(LemurOutputSchema):
     )
     rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
 
+    country = fields.String()
+    location = fields.String()
+    state = fields.String()
+    organization = fields.String()
+    organizational_unit = fields.String()
+
+    @post_dump
+    def handle_subject_details(self, data):
+        subject_details = ["country", "state", "location", "organization", "organizational_unit"]
+
+        # Remove subject details if authority is CA/Browser Forum compliant. The code will use default set of values in that case.
+        # If CA/Browser Forum compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below
+        # condition checks for 'not False' ==> 'True or None'
+        if data.get("authority"):
+            is_cab_compliant = data.get("authority").get("isCabCompliant")
+
+            if is_cab_compliant is not False:
+                for field in subject_details:
+                    data.pop(field, None)
+
+        # Removing subject fields if None, else it complains in de-serialization
+        for field in subject_details:
+            if field in data and data[field] is None:
+                data.pop(field)
+
 
 class CertificateShortOutputSchema(LemurOutputSchema):
     id = fields.Integer()

lemur/certificates/service.py

@@ -6,11 +6,13 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+import re
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
 from flask import current_app
 from sqlalchemy import func, or_, not_, cast, Integer
+from sqlalchemy.sql.expression import false, true
 
 from lemur import database
 from lemur.authorities.models import Authority
@@ -105,7 +107,7 @@ def get_all_certs():
 
 def get_all_valid_certs(authority_plugin_name):
     """
-    Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names
+    Retrieves all valid (not expired & not revoked) certificates within Lemur, for the given authority plugin names
     ignored if no authority_plugin_name provided.
 
     Note that depending on the DB size retrieving all certificates might an expensive operation
@@ -116,11 +118,12 @@ def get_all_valid_certs(authority_plugin_name):
         return (
             Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
                 Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
-                Authority.plugin_name.in_(authority_plugin_name)).all()
+                Authority.plugin_name.in_(authority_plugin_name)).filter(Certificate.revoked.is_(False)).all()
         )
     else:
         return (
-            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all()
+            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
+                Certificate.revoked.is_(False)).all()
         )
 
 
@@ -148,7 +151,7 @@ def get_all_certs_attached_to_endpoint_without_autorotate():
         """
     return (
         Certificate.query.filter(Certificate.endpoints.any())
-        .filter(Certificate.rotation == False)
+        .filter(Certificate.rotation == false())
         .filter(Certificate.not_after >= arrow.now())
         .filter(not_(Certificate.replaced.any()))
         .all()  # noqa
@@ -203,9 +206,9 @@ def get_all_pending_reissue():
     :return:
     """
     return (
-        Certificate.query.filter(Certificate.rotation == True)
+        Certificate.query.filter(Certificate.rotation == true())
         .filter(not_(Certificate.replaced.any()))
-        .filter(Certificate.in_rotation_window == True)
+        .filter(Certificate.in_rotation_window == true())
         .all()
     )  # noqa
 
@@ -256,17 +259,29 @@ def update(cert_id, **kwargs):
     return database.update(cert)
 
 
-def create_certificate_roles(**kwargs):
-    # create an role for the owner and assign it
-    owner_role = role_service.get_by_name(kwargs["owner"])
+def cleanup_owner_roles_notification(owner_name, kwargs):
+    kwargs["roles"] = [r for r in kwargs["roles"] if r.name != owner_name]
+    notification_prefix = f"DEFAULT_{owner_name.split('@')[0].upper()}"
+    kwargs["notifications"] = [n for n in kwargs["notifications"] if not n.label.startswith(notification_prefix)]
 
-    if not owner_role:
-        owner_role = role_service.create(
-            kwargs["owner"],
-            description="Auto generated role based on owner: {0}".format(
-                kwargs["owner"]
-            ),
-        )
+
+def update_notify(cert, notify_flag):
+    """
+    Toggle notification value which is a boolean
+    :param notify_flag: new notify value
+    :param cert: Certificate object to be updated
+    :return:
+    """
+    cert.notify = notify_flag
+    return database.update(cert)
+
+
+def create_certificate_roles(**kwargs):
+    # create a role for the owner and assign it
+    owner_role = role_service.get_or_create(
+        kwargs["owner"],
+        description=f"Auto generated role based on owner: {kwargs['owner']}"
+    )
 
     # ensure that the authority's owner is also associated with the certificate
     if kwargs.get("authority"):
@@ -347,7 +362,12 @@ def create(**kwargs):
     try:
         cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs)
     except Exception:
-        current_app.logger.error("Exception minting certificate", exc_info=True)
+        log_data = {
+            "message": "Exception minting certificate",
+            "issuer": kwargs["authority"].name,
+            "cn": kwargs["common_name"],
+        }
+        current_app.logger.error(log_data, exc_info=True)
         sentry.captureException()
         raise
     kwargs["body"] = cert_body
@@ -506,7 +526,7 @@ def render(args):
         )
 
     if current_app.config.get("ALLOW_CERT_DELETION", False):
-        query = query.filter(Certificate.deleted == False)  # noqa
+        query = query.filter(Certificate.deleted == false())
 
     result = database.sort_and_page(query, Certificate, args)
     return result
@@ -542,20 +562,21 @@ def query_common_name(common_name, args):
     :return:
     """
     owner = args.pop("owner")
-    if not owner:
-        owner = "%"
-
     # only not expired certificates
     current_time = arrow.utcnow()
 
-    result = (
-        Certificate.query.filter(Certificate.cn.ilike(common_name))
-        .filter(Certificate.owner.ilike(owner))
-        .filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))
-        .all()
-    )
+    query = Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD"))\
+        .filter(not_(Certificate.revoked))\
+        .filter(not_(Certificate.replaced.any()))  # ignore rotated certificates to avoid duplicates
 
-    return result
+    if owner:
+        query = query.filter(Certificate.owner.ilike(owner))
+
+    if common_name != "%":
+        # if common_name is a wildcard ('%'), no need to include it in the query
+        query = query.filter(Certificate.cn.ilike(common_name))
+
+    return query.all()
 
 
 def create_csr(**csr_config):
@@ -760,6 +781,19 @@ def reissue_certificate(certificate, replace=None, user=None):
     if replace:
         primitives["replaces"] = [certificate]
 
+    # Modify description to include the certificate ID being reissued and mention that this is created by Lemur
+    # as part of reissue
+    reissue_message_prefix = "Reissued by Lemur for cert ID "
+    reissue_message = re.compile(f"{reissue_message_prefix}([0-9]+)")
+    if primitives["description"]:
+        match = reissue_message.search(primitives["description"])
+        if match:
+            primitives["description"] = primitives["description"].replace(match.group(1), str(certificate.id))
+        else:
+            primitives["description"] = f"{reissue_message_prefix}{certificate.id}, {primitives['description']}"
+    else:
+        primitives["description"] = f"{reissue_message_prefix}{certificate.id}"
+
     new_cert = create(**primitives)
 
     return new_cert

lemur/certificates/utils.py

@@ -82,4 +82,4 @@ def get_key_type_from_csr(data):
             raise Exception("Unsupported key type")
 
     except NotImplemented:
-        raise NotImplemented()
+        raise NotImplementedError

lemur/certificates/views.py

@@ -884,10 +884,118 @@ class Certificates(AuthenticatedResource):
                         400,
                     )
 
+        # if owner is changed, remove all notifications and roles associated with old owner
+        if cert.owner != data["owner"]:
+            service.cleanup_owner_roles_notification(cert.owner, data)
+
         cert = service.update(certificate_id, **data)
         log_service.create(g.current_user, "update_cert", certificate=cert)
         return cert
 
+    @validate_schema(certificate_edit_input_schema, certificate_output_schema)
+    def post(self, certificate_id, data=None):
+        """
+        .. http:post:: /certificates/1/update/notify
+
+           Update certificate notification
+
+           **Example request**:
+
+           .. sourcecode:: http
+
+              POST /certificates/1/update/notify HTTP/1.1
+              Host: example.com
+              Accept: application/json, text/javascript
+
+              {
+                 "notify": false
+              }
+
+           **Example response**:
+
+           .. sourcecode:: http
+
+              HTTP/1.1 200 OK
+              Vary: Accept
+              Content-Type: text/javascript
+
+              {
+                "status": null,
+                "cn": "*.test.example.net",
+                "chain": "",
+                "authority": {
+                    "active": true,
+                    "owner": "secure@example.com",
+                    "id": 1,
+                    "description": "verisign test authority",
+                    "name": "verisign"
+                },
+                "owner": "joe@example.com",
+                "serial": "82311058732025924142789179368889309156",
+                "id": 2288,
+                "issuer": "SymantecCorporation",
+                "dateCreated": "2016-06-03T06:09:42.133769+00:00",
+                "notBefore": "2016-06-03T00:00:00+00:00",
+                "notAfter": "2018-01-12T23:59:59+00:00",
+                "destinations": [],
+                "bits": 2048,
+                "body": "-----BEGIN CERTIFICATE-----...",
+                "description": null,
+                "deleted": null,
+                "notify": false,
+                "notifications": [{
+                    "id": 1
+                }]
+                "signingAlgorithm": "sha256",
+                "user": {
+                    "username": "jane",
+                    "active": true,
+                    "email": "jane@example.com",
+                    "id": 2
+                },
+                "active": true,
+                "domains": [{
+                    "sensitive": false,
+                    "id": 1090,
+                    "name": "*.test.example.net"
+                }],
+                "replaces": [],
+                "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
+                "roles": [{
+                    "id": 464,
+                    "description": "This is a google group based role created by Lemur",
+                    "name": "joe@example.com"
+                }],
+                "rotation": true,
+                "rotationPolicy": {"name": "default"},
+                "san": null
+              }
+
+           :reqheader Authorization: OAuth token to authenticate
+           :statuscode 200: no error
+           :statuscode 403: unauthenticated
+
+        """
+        cert = service.get(certificate_id)
+
+        if not cert:
+            return dict(message="Cannot find specified certificate"), 404
+
+        # allow creators
+        if g.current_user != cert.user:
+            owner_role = role_service.get_by_name(cert.owner)
+            permission = CertificatePermission(owner_role, [x.name for x in cert.roles])
+
+            if not permission.can():
+                return (
+                    dict(message="You are not authorized to update this certificate"),
+                    403,
+                )
+
+        cert = service.update_notify(cert, data.get("notify"))
+        log_service.create(g.current_user, "update_cert", certificate=cert)
+        return cert
+
     def delete(self, certificate_id, data=None):
         """
         .. http:delete:: /certificates/1
@@ -1047,6 +1155,7 @@ class NotificationCertificatesList(AuthenticatedResource):
         )
         parser.add_argument("creator", type=str, location="args")
         parser.add_argument("show", type=str, location="args")
+        parser.add_argument("showExpired", type=int, location="args")
 
         args = parser.parse_args()
         args["notification_id"] = notification_id
@@ -1354,6 +1463,9 @@ api.add_resource(
 api.add_resource(
     Certificates, "/certificates/<int:certificate_id>", endpoint="certificate"
 )
+api.add_resource(
+    Certificates, "/certificates/<int:certificate_id>/update/notify", endpoint="certificateUpdateNotify"
+)
 api.add_resource(CertificatesStats, "/certificates/stats", endpoint="certificateStats")
 api.add_resource(
     CertificatesUpload, "/certificates/upload", endpoint="certificateUpload"

lemur/common/celery.py

@@ -759,7 +759,7 @@ def check_revoked():
 
     log_data = {
         "function": function,
-        "message": "check if any certificates are revoked revoked",
+        "message": "check if any valid certificate is revoked",
         "task_id": task_id,
     }
 
@@ -842,3 +842,39 @@ def enable_autorotate_for_certs_attached_to_endpoint():
     cli_certificate.automatically_enable_autorotate()
     metrics.send(f"{function}.success", "counter", 1)
     return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def deactivate_entrust_test_certificates():
+    """
+    This celery task attempts to deactivate all not yet deactivated Entrust certificates, and should only run in TEST
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "message": "deactivate entrust certificates",
+        "task_id": task_id,
+    }
+
+    if task_id and is_task_active(function, task_id, None):
+        log_data["message"] = "Skipping task: Task is already active"
+        current_app.logger.debug(log_data)
+        return
+
+    current_app.logger.debug(log_data)
+    try:
+        cli_certificate.deactivate_entrust_certificates()
+    except SoftTimeLimitExceeded:
+        log_data["message"] = "Time limit exceeded."
+        current_app.logger.error(log_data)
+        sentry.captureException()
+        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+        return
+
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data

lemur/common/defaults.py

@@ -95,9 +95,11 @@ def organization(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[
-            0
-        ].value.strip()
+        o = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
+        if not o:
+            return None
+
+        return o[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get organization! {0}".format(e))
@@ -110,9 +112,11 @@ def organizational_unit(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)[
-            0
-        ].value.strip()
+        ou = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)
+        if not ou:
+            return None
+
+        return ou[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get organizational unit! {0}".format(e))
@@ -125,9 +129,11 @@ def country(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)[
-            0
-        ].value.strip()
+        c = cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)
+        if not c:
+            return None
+
+        return c[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get country! {0}".format(e))
@@ -140,9 +146,11 @@ def state(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)[
-            0
-        ].value.strip()
+        s = cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)
+        if not s:
+            return None
+
+        return s[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get state! {0}".format(e))
@@ -155,9 +163,11 @@ def location(cert):
     :return:
     """
     try:
-        return cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)[
-            0
-        ].value.strip()
+        loc = cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)
+        if not loc:
+            return None
+
+        return loc[0].value.strip()
     except Exception as e:
         sentry.captureException()
         current_app.logger.error("Unable to get location! {0}".format(e))

lemur/common/utils.py

@@ -9,6 +9,7 @@
 import random
 import re
 import string
+import pem
 
 import sqlalchemy
 from cryptography import x509
@@ -16,7 +17,7 @@ from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding
-from cryptography.hazmat.primitives.serialization import load_pem_private_key
+from cryptography.hazmat.primitives.serialization import load_pem_private_key, Encoding, pkcs7
 from flask_restful.reqparse import RequestParser
 from sqlalchemy import and_, func
 
@@ -357,3 +358,19 @@ def find_matching_certificates_by_hash(cert, matching_certs):
         ):
             matching.append(c)
     return matching
+
+
+def convert_pkcs7_bytes_to_pem(certs_pkcs7):
+    """
+    Given a list of certificates in pkcs7 encoding (bytes), covert them into a list of PEM encoded files
+    :raises ValueError or ValidationError
+    :param certs_pkcs7:
+    :return: list of certs in PEM format
+    """
+
+    certificates = pkcs7.load_pem_pkcs7_certificates(certs_pkcs7)
+    certificates_pem = []
+    for cert in certificates:
+        certificates_pem.append(pem.parse(cert.public_bytes(encoding=Encoding.PEM))[0])
+
+    return certificates_pem

lemur/common/validators.py

@@ -22,7 +22,7 @@ def common_name(value):
 
 def sensitive_domain(domain):
     """
-    Checks if user has the admin role, the domain does not match sensitive domains and whitelisted domain patterns.
+    Checks if user has the admin role, the domain does not match sensitive domains and allowed domain patterns.
     :param domain: domain name (str)
     :return:
     """
@@ -30,10 +30,10 @@ def sensitive_domain(domain):
         # User has permission, no need to check anything
         return
 
-    whitelist = current_app.config.get("LEMUR_WHITELISTED_DOMAINS", [])
-    if whitelist and not any(re.match(pattern, domain) for pattern in whitelist):
+    allowlist = current_app.config.get("LEMUR_ALLOWED_DOMAINS", [])
+    if allowlist and not any(re.match(pattern, domain) for pattern in allowlist):
         raise ValidationError(
-            "Domain {0} does not match whitelisted domain patterns. "
+            "Domain {0} does not match allowed domain patterns. "
             "Contact an administrator to issue the certificate.".format(domain)
         )
 

lemur/destinations/schemas.py

@@ -31,6 +31,9 @@ class DestinationOutputSchema(LemurOutputSchema):
     def fill_object(self, data):
         if data:
             data["plugin"]["pluginOptions"] = data["options"]
+            for option in data["plugin"]["pluginOptions"]:
+                if "export-plugin" in option["type"]:
+                    option["value"]["pluginOptions"] = option["value"]["plugin_options"]
         return data
 
 

lemur/destinations/service.py

@@ -41,12 +41,14 @@ def create(label, plugin_name, options, description=None):
     return database.create(destination)
 
 
-def update(destination_id, label, options, description):
+def update(destination_id, label, plugin_name, options, description):
     """
     Updates an existing destination.
 
     :param destination_id:  Lemur assigned ID
     :param label: Destination common name
+    :param plugin_name:
+    :param options:
     :param description:
     :rtype : Destination
     :return:
@@ -54,6 +56,11 @@ def update(destination_id, label, options, description):
     destination = get(destination_id)
 
     destination.label = label
+    destination.plugin_name = plugin_name
+    # remove any sub-plugin objects before try to save the json options
+    for option in options:
+        if "plugin" in option["type"]:
+            del option["value"]["plugin_object"]
     destination.options = options
     destination.description = description
 

lemur/destinations/views.py

@@ -338,6 +338,7 @@ class Destinations(AuthenticatedResource):
         return service.update(
             destination_id,
             data["label"],
+            data["plugin"]["slug"],
             data["plugin"]["plugin_options"],
             data["description"],
         )

lemur/dns_providers/cli.py

@@ -3,9 +3,9 @@ from flask_script import Manager
 import sys
 
 from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.plugins.lemur_acme.acme_handlers import AcmeDnsHandler
 from lemur.dns_providers.service import get_all_dns_providers, set_domains
 from lemur.extensions import metrics, sentry
-from lemur.plugins.base import plugins
 
 manager = Manager(
     usage="Iterates through all DNS providers and sets DNS zones in the database."
@@ -19,7 +19,7 @@ def get_all_zones():
     """
     print("[+] Starting dns provider zone lookup and configuration.")
     dns_providers = get_all_dns_providers()
-    acme_plugin = plugins.get("acme-issuer")
+    acme_dns_handler = AcmeDnsHandler()
 
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
     log_data = {
@@ -29,7 +29,7 @@ def get_all_zones():
 
     for dns_provider in dns_providers:
         try:
-            zones = acme_plugin.get_all_zones(dns_provider)
+            zones = acme_dns_handler.get_all_zones(dns_provider)
             set_domains(dns_provider, zones)
         except Exception as e:
             print("[+] Error with DNS Provider {}: {}".format(dns_provider.name, e))

lemur/factory.py

@@ -10,7 +10,7 @@
 
 """
 import os
-import imp
+import importlib
 import errno
 import pkg_resources
 import socket
@@ -73,8 +73,9 @@ def from_file(file_path, silent=False):
     :param file_path:
     :param silent:
     """
-    d = imp.new_module("config")
-    d.__file__ = file_path
+    module_spec = importlib.util.spec_from_file_location("config", file_path)
+    d = importlib.util.module_from_spec(module_spec)
+
     try:
         with open(file_path) as config_file:
             exec(  # nosec: config file safe

lemur/logs/models.py

@@ -7,7 +7,7 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from sqlalchemy import Column, Integer, ForeignKey, PassiveDefault, func, Enum
+from sqlalchemy import Column, Integer, ForeignKey, DefaultClause, func, Enum
 
 from sqlalchemy_utils.types.arrow import ArrowType
 
@@ -29,5 +29,5 @@ class Log(db.Model):
         ),
         nullable=False,
     )
-    logged_at = Column(ArrowType(), PassiveDefault(func.now()), nullable=False)
+    logged_at = Column(ArrowType(), DefaultClause(func.now()), nullable=False)
     user_id = Column(Integer, ForeignKey("users.id"), nullable=False)

lemur/manage.py

@@ -95,7 +95,7 @@ LEMUR_TOKEN_SECRET = '{secret_token}'
 LEMUR_ENCRYPTION_KEYS = '{encryption_key}'
 
 # List of domain regular expressions that non-admin users can issue
-LEMUR_WHITELISTED_DOMAINS = []
+LEMUR_ALLOWED_DOMAINS = []
 
 # Mail Server
 
@@ -120,6 +120,7 @@ METRIC_PROVIDERS = []
 
 LOG_LEVEL = "DEBUG"
 LOG_FILE = "lemur.log"
+LOG_UPGRADE_FILE = "db_upgrade.log"
 
 
 # Database

lemur/migrations/env.py

@@ -20,8 +20,9 @@ fileConfig(config.config_file_name)
 # target_metadata = mymodel.Base.metadata
 from flask import current_app
 
+db_url_escaped = current_app.config.get('SQLALCHEMY_DATABASE_URI').replace('%', '%%')
 config.set_main_option(
-    "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI")
+    "sqlalchemy.url", db_url_escaped
 )
 target_metadata = current_app.extensions["migrate"].db.metadata
 

lemur/migrations/versions/1db4f82bc780_.py

@@ -10,11 +10,21 @@ Create Date: 2018-08-03 12:56:44.565230
 revision = "1db4f82bc780"
 down_revision = "3adfdd6598df"
 
-import logging
-
 from alembic import op
 
-log = logging.getLogger(__name__)
+from flask import current_app
+from logging import Formatter, FileHandler, getLogger
+
+log = getLogger(__name__)
+handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
+handler.setFormatter(
+    Formatter(
+        "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+    )
+)
+handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.addHandler(handler)
 
 
 def upgrade():

lemur/migrations/versions/c301c59688d2_.py

@@ -7,8 +7,9 @@ the rest of the keys, the certificate body is parsed to determine
 the exact key_type information.
 
 Each individual DB change is explicitly committed, and the respective
-log is added to a file named db_upgrade.log in the current working
-directory. Any error encountered while parsing a certificate will
+log is added to a file configured in LOG_UPGRADE_FILE or, by default,
+to a file named db_upgrade.log in the current working directory.
+Any error encountered while parsing a certificate will
 also be logged along with the certificate ID. If faced with any issue
 while running this upgrade, there is no harm in re-running the upgrade.
 Each run processes only rows for which key_type information is not yet
@@ -31,15 +32,28 @@ down_revision = '434c29e40511'
 
 from alembic import op
 from sqlalchemy.sql import text
-from lemur.common import utils
 import time
 import datetime
+from flask import current_app
 
-log_file = open('db_upgrade.log', 'a')
+from logging import Formatter, FileHandler, getLogger
+
+from lemur.common import utils
+
+log = getLogger(__name__)
+handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
+handler.setFormatter(
+    Formatter(
+        "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+    )
+)
+handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.addHandler(handler)
 
 
 def upgrade():
-    log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
+    log.info("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
     start_time = time.time()
 
     # Update RSA keys using the key length information
@@ -50,8 +64,7 @@ def upgrade():
     # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs.
     update_key_type()
 
-    log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time))
-    log_file.close()
+    log.info("--- Total %s seconds ---\n" % (time.time() - start_time))
 
 
 def downgrade():
@@ -61,6 +74,7 @@ def downgrade():
         "update certificates set key_type=null where not_after > CURRENT_DATE - 32"
     )
     op.execute(stmt)
+    commit()
 
 
 """
@@ -69,18 +83,18 @@ def downgrade():
 
 
 def update_key_type_rsa(bits):
-    log_file.write("Processing certificate with key type RSA %s\n" % bits)
+    log.info("Processing certificate with key type RSA %s\n" % bits)
 
     stmt = text(
         f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null"
     )
-    log_file.write("Query: %s\n" % stmt)
+    log.info("Query: %s\n" % stmt)
 
     start_time = time.time()
     op.execute(stmt)
     commit()
 
-    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- %s seconds ---\n" % (time.time() - start_time))
 
 
 def update_key_type():
@@ -95,9 +109,9 @@ def update_key_type():
         try:
             cert_key_type = utils.get_key_type_from_certificate(body)
         except ValueError as e:
-            log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
+            log.error("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
         else:
-            log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
+            log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
             stmt = text(
                 "update certificates set key_type=:key_type where id=:id"
             )
@@ -106,7 +120,7 @@ def update_key_type():
 
             commit()
 
-    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- %s seconds ---\n" % (time.time() - start_time))
 
 
 def commit():

lemur/notifications/messaging.py

@@ -8,6 +8,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
+import sys
 from collections import defaultdict
 from datetime import timedelta
 from itertools import groupby
@@ -15,6 +16,7 @@ from itertools import groupby
 import arrow
 from flask import current_app
 from sqlalchemy import and_
+from sqlalchemy.sql.expression import false, true
 
 from lemur import database
 from lemur.certificates.models import Certificate
@@ -29,7 +31,7 @@ from lemur.plugins.utils import get_plugin_option
 
 def get_certificates(exclude=None):
     """
-    Finds all certificates that are eligible for notifications.
+    Finds all certificates that are eligible for expiration notifications.
     :param exclude:
     :return:
     """
@@ -39,9 +41,10 @@ def get_certificates(exclude=None):
     q = (
         database.db.session.query(Certificate)
         .filter(Certificate.not_after <= max)
-        .filter(Certificate.notify == True)
-        .filter(Certificate.expired == False)
-    )  # noqa
+        .filter(Certificate.notify == true())
+        .filter(Certificate.expired == false())
+        .filter(Certificate.revoked == false())
+    )
 
     exclude_conditions = []
     if exclude:
@@ -61,7 +64,8 @@ def get_certificates(exclude=None):
 
 def get_eligible_certificates(exclude=None):
     """
-    Finds all certificates that are eligible for certificate expiration.
+    Finds all certificates that are eligible for certificate expiration notification.
+    Returns the set of all eligible certificates, grouped by owner, with a list of applicable notifications.
     :param exclude:
     :return:
     """
@@ -86,21 +90,31 @@ def get_eligible_certificates(exclude=None):
     return certificates
 
 
-def send_notification(event_type, data, targets, notification):
+def send_plugin_notification(event_type, data, recipients, notification):
     """
     Executes the plugin and handles failure.
 
     :param event_type:
     :param data:
-    :param targets:
+    :param recipients:
     :param notification:
     :return:
     """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    log_data = {
+        "function": function,
+        "message": f"Sending expiration notification for to recipients {recipients}",
+        "notification_type": "expiration",
+        "certificate_targets": recipients,
+    }
     status = FAILURE_METRIC_STATUS
     try:
-        notification.plugin.send(event_type, data, targets, notification.options)
+        current_app.logger.debug(log_data)
+        notification.plugin.send(event_type, data, recipients, notification.options)
         status = SUCCESS_METRIC_STATUS
     except Exception as e:
+        log_data["message"] = f"Unable to send {event_type} notification to recipients {recipients}"
+        current_app.logger.error(log_data, exc_info=True)
         sentry.captureException()
 
     metrics.send(
@@ -124,11 +138,11 @@ def send_expiration_notifications(exclude):
     # security team gets all
     security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
 
-    security_data = []
     for owner, notification_group in get_eligible_certificates(exclude=exclude).items():
 
         for notification_label, certificates in notification_group.items():
             notification_data = []
+            security_data = []
 
             notification = certificates[0][0]
 
@@ -140,36 +154,27 @@ def send_expiration_notifications(exclude):
                 notification_data.append(cert_data)
                 security_data.append(cert_data)
 
-            if send_notification(
-                "expiration", notification_data, [owner], notification
+            if send_default_notification(
+                    "expiration", notification_data, [owner], notification.options
             ):
                 success += 1
             else:
                 failure += 1
 
-            notification_recipient = get_plugin_option(
-                "recipients", notification.options
-            )
-            if notification_recipient:
-                notification_recipient = notification_recipient.split(",")
-                # removing owner and security_email from notification_recipient
-                notification_recipient = [i for i in notification_recipient if i not in security_email and i != owner]
+            recipients = notification.plugin.filter_recipients(notification.options, security_email + [owner])
 
-            if (
-                notification_recipient
+            if send_plugin_notification(
+                "expiration",
+                notification_data,
+                recipients,
+                notification,
             ):
-                if send_notification(
-                    "expiration",
-                    notification_data,
-                    notification_recipient,
-                    notification,
-                ):
-                    success += 1
-                else:
-                    failure += 1
+                success += 1
+            else:
+                failure += 1
 
-            if send_notification(
-                "expiration", security_data, security_email, notification
+            if send_default_notification(
+                "expiration", security_data, security_email, notification.options
             ):
                 success += 1
             else:
@@ -178,107 +183,86 @@ def send_expiration_notifications(exclude):
     return success, failure
 
 
-def send_rotation_notification(certificate, notification_plugin=None):
+def send_default_notification(notification_type, data, targets, notification_options=None):
     """
-    Sends a report to certificate owners when their certificate has been
-    rotated.
+    Sends a report to the specified target via the default notification plugin. Applicable for any notification_type.
+    At present, "default" means email, as the other notification plugins do not support dynamically configured targets.
 
-    :param certificate:
-    :param notification_plugin:
+    :param notification_type:
+    :param data:
+    :param targets:
+    :param notification_options:
     :return:
     """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    log_data = {
+        "function": function,
+        "message": f"Sending notification for certificate data {data}",
+        "notification_type": notification_type,
+    }
     status = FAILURE_METRIC_STATUS
-    if not notification_plugin:
-        notification_plugin = plugins.get(
-            current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN")
-        )
-
-    data = certificate_notification_output_schema.dump(certificate).data
+    notification_plugin = plugins.get(
+        current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification")
+    )
 
     try:
-        notification_plugin.send("rotation", data, [data["owner"]])
+        current_app.logger.debug(log_data)
+        # we need the notification.options here because the email templates utilize the interval/unit info
+        notification_plugin.send(notification_type, data, targets, notification_options)
         status = SUCCESS_METRIC_STATUS
     except Exception as e:
-        current_app.logger.error(
-            "Unable to send notification to {}.".format(data["owner"]), exc_info=True
-        )
+        log_data["message"] = f"Unable to send {notification_type} notification for certificate data {data} " \
+                              f"to target {targets}"
+        current_app.logger.error(log_data, exc_info=True)
         sentry.captureException()
 
     metrics.send(
         "notification",
         "counter",
         1,
-        metric_tags={"status": status, "event_type": "rotation"},
+        metric_tags={"status": status, "event_type": notification_type},
     )
 
     if status == SUCCESS_METRIC_STATUS:
         return True
 
 
+def send_rotation_notification(certificate):
+    data = certificate_notification_output_schema.dump(certificate).data
+    return send_default_notification("rotation", data, [data["owner"]])
+
+
 def send_pending_failure_notification(
-    pending_cert, notify_owner=True, notify_security=True, notification_plugin=None
+    pending_cert, notify_owner=True, notify_security=True
 ):
     """
     Sends a report to certificate owners when their pending certificate failed to be created.
 
     :param pending_cert:
-    :param notification_plugin:
+    :param notify_owner:
+    :param notify_security:
     :return:
     """
-    status = FAILURE_METRIC_STATUS
-
-    if not notification_plugin:
-        notification_plugin = plugins.get(
-            current_app.config.get(
-                "LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification"
-            )
-        )
 
     data = pending_certificate_output_schema.dump(pending_cert).data
     data["security_email"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL")
 
+    notify_owner_success = False
     if notify_owner:
-        try:
-            notification_plugin.send("failed", data, [data["owner"]], pending_cert)
-            status = SUCCESS_METRIC_STATUS
-        except Exception as e:
-            current_app.logger.error(
-                "Unable to send pending failure notification to {}.".format(
-                    data["owner"]
-                ),
-                exc_info=True,
-            )
-            sentry.captureException()
+        notify_owner_success = send_default_notification("failed", data, [data["owner"]], pending_cert)
 
+    notify_security_success = False
     if notify_security:
-        try:
-            notification_plugin.send(
-                "failed", data, data["security_email"], pending_cert
-            )
-            status = SUCCESS_METRIC_STATUS
-        except Exception as e:
-            current_app.logger.error(
-                "Unable to send pending failure notification to "
-                "{}.".format(data["security_email"]),
-                exc_info=True,
-            )
-            sentry.captureException()
+        notify_security_success = send_default_notification("failed", data, data["security_email"], pending_cert)
 
-    metrics.send(
-        "notification",
-        "counter",
-        1,
-        metric_tags={"status": status, "event_type": "rotation"},
-    )
-
-    if status == SUCCESS_METRIC_STATUS:
-        return True
+    return notify_owner_success or notify_security_success
 
 
 def needs_notification(certificate):
     """
-    Determine if notifications for a given certificate should
-    currently be sent
+    Determine if notifications for a given certificate should currently be sent.
+    For each notification configured for the cert, verifies it is active, properly configured,
+    and that the configured expiration period is currently met.
 
     :param certificate:
     :return:
@@ -290,7 +274,7 @@ def needs_notification(certificate):
 
     for notification in certificate.notifications:
         if not notification.active or not notification.options:
-            return
+            continue
 
         interval = get_plugin_option("interval", notification.options)
         unit = get_plugin_option("unit", notification.options)
@@ -306,9 +290,8 @@ def needs_notification(certificate):
 
         else:
             raise Exception(
-                "Invalid base unit for expiration interval: {0}".format(unit)
+                f"Invalid base unit for expiration interval: {unit}"
             )
-
         if days == interval:
             notifications.append(notification)
     return notifications

lemur/notifications/service.py

@@ -43,7 +43,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None):
             "name": "recipients",
             "type": "str",
             "required": True,
-            "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$",
+            "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$",
             "helpMessage": "Comma delimited list of email addresses",
             "value": ",".join(recipients),
         },
@@ -63,7 +63,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None):
                     "name": "interval",
                     "type": "int",
                     "required": True,
-                    "validation": "^\d+$",
+                    "validation": r"^\d+$",
                     "helpMessage": "Number of days to be alert before expiration.",
                     "value": i,
                 }
@@ -104,12 +104,13 @@ def create(label, plugin_name, options, description, certificates):
     return database.create(notification)
 
 
-def update(notification_id, label, options, description, active, certificates):
+def update(notification_id, label, plugin_name, options, description, active, certificates):
     """
     Updates an existing notification.
 
     :param notification_id:
     :param label: Notification label
+    :param plugin_name:
     :param options:
     :param description:
     :param active:
@@ -120,6 +121,7 @@ def update(notification_id, label, options, description, active, certificates):
     notification = get(notification_id)
 
     notification.label = label
+    notification.plugin_name = plugin_name
     notification.options = options
     notification.description = description
     notification.active = active

lemur/notifications/views.py

@@ -340,6 +340,7 @@ class Notifications(AuthenticatedResource):
         return service.update(
             notification_id,
             data["label"],
+            data["plugin"]["slug"],
             data["plugin"]["plugin_options"],
             data["description"],
             data["active"],

lemur/pending_certificates/models.py

@@ -9,7 +9,7 @@ from sqlalchemy import (
     Integer,
     ForeignKey,
     String,
-    PassiveDefault,
+    DefaultClause,
     func,
     Column,
     Text,
@@ -76,14 +76,14 @@ class PendingCertificate(db.Model):
     chain = Column(Text())
     private_key = Column(Vault, nullable=True)
 
-    date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
+    date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False)
     dns_provider_id = Column(
         Integer, ForeignKey("dns_providers.id", ondelete="CASCADE")
     )
 
     status = Column(Text(), nullable=True)
     last_updated = Column(
-        ArrowType, PassiveDefault(func.now()), onupdate=func.now(), nullable=False
+        ArrowType, DefaultClause(func.now()), onupdate=func.now(), nullable=False
     )
 
     rotation = Column(Boolean, default=False)

lemur/plugins/bases/notification.py

@@ -20,6 +20,15 @@ class NotificationPlugin(Plugin):
     def send(self, notification_type, message, targets, options, **kwargs):
         raise NotImplementedError
 
+    def filter_recipients(self, options, excluded_recipients):
+        """
+        Given a set of options (which should include configured recipient info), filters out recipients that
+        we do NOT want to notify.
+
+        For any notification types where recipients can't be dynamically modified, this returns an empty list.
+        """
+        return []
+
 
 class ExpirationNotificationPlugin(NotificationPlugin):
     """
@@ -33,7 +42,7 @@ class ExpirationNotificationPlugin(NotificationPlugin):
             "name": "interval",
             "type": "int",
             "required": True,
-            "validation": "^\d+$",
+            "validation": r"^\d+$",
             "helpMessage": "Number of days to be alert before expiration.",
         },
         {
@@ -50,5 +59,5 @@ class ExpirationNotificationPlugin(NotificationPlugin):
     def options(self):
         return self.default_options + self.additional_options
 
-    def send(self, notification_type, message, targets, options, **kwargs):
+    def send(self, notification_type, message, excluded_targets, options, **kwargs):
         raise NotImplementedError

lemur/plugins/lemur_acme/acme_handlers.py

@@ -0,0 +1,521 @@
+"""
+.. module: lemur.plugins.lemur_acme.plugin
+    :platform: Unix
+    :synopsis: This module contains handlers for certain acme related tasks. It needed to be refactored to avoid circular imports
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+    Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
+.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
+.. moduleauthor:: Mathias Petermann <mathias.petermann@projektfokus.ch>
+"""
+import datetime
+import json
+import time
+
+import OpenSSL.crypto
+import josepy as jose
+import dns.resolver
+from acme import challenges, errors, messages
+from acme.client import BackwardsCompatibleClientV2, ClientNetwork
+from acme.errors import TimeoutError
+from acme.messages import Error as AcmeError
+from flask import current_app
+
+from lemur.common.utils import generate_private_key
+from lemur.dns_providers import service as dns_provider_service
+from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration
+from lemur.extensions import metrics, sentry
+
+from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns
+from lemur.authorities import service as authorities_service
+from retrying import retry
+
+
+class AuthorizationRecord(object):
+    def __init__(self, domain, target_domain, authz, dns_challenge, change_id):
+        self.domain = domain
+        self.target_domain = target_domain
+        self.authz = authz
+        self.dns_challenge = dns_challenge
+        self.change_id = change_id
+
+
+class AcmeHandler(object):
+
+    def reuse_account(self, authority):
+        if not authority.options:
+            raise InvalidAuthority("Invalid authority. Options not set")
+        existing_key = False
+        existing_regr = False
+
+        for option in json.loads(authority.options):
+            if option["name"] == "acme_private_key" and option["value"]:
+                existing_key = True
+            if option["name"] == "acme_regr" and option["value"]:
+                existing_regr = True
+
+        if not existing_key and current_app.config.get("ACME_PRIVATE_KEY"):
+            existing_key = True
+
+        if not existing_regr and current_app.config.get("ACME_REGR"):
+            existing_regr = True
+
+        if existing_key and existing_regr:
+            return True
+        else:
+            return False
+
+    def strip_wildcard(self, host):
+        """Removes the leading *. and returns Host and whether it was removed or not (True/False)"""
+        prefix = "*."
+        if host.startswith(prefix):
+            return host[len(prefix):], True
+        return host, False
+
+    def maybe_add_extension(self, host, dns_provider_options):
+        if dns_provider_options and dns_provider_options.get(
+                "acme_challenge_extension"
+        ):
+            host = host + dns_provider_options.get("acme_challenge_extension")
+        return host
+
+    def request_certificate(self, acme_client, authorizations, order):
+        for authorization in authorizations:
+            for authz in authorization.authz:
+                authorization_resource, _ = acme_client.poll(authz)
+
+        deadline = datetime.datetime.now() + datetime.timedelta(seconds=360)
+
+        try:
+            orderr = acme_client.poll_and_finalize(order, deadline)
+
+        except (AcmeError, TimeoutError):
+            sentry.captureException(extra={"order_url": str(order.uri)})
+            metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri})
+            current_app.logger.error(
+                f"Unable to resolve Acme order: {order.uri}", exc_info=True
+            )
+            raise
+        except errors.ValidationError:
+            if order.fullchain_pem:
+                orderr = order
+            else:
+                raise
+
+        metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri})
+        current_app.logger.info(
+            f"Successfully resolved Acme order: {order.uri}", exc_info=True
+        )
+
+        pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem)
+
+        current_app.logger.debug(
+            "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain))
+        )
+        return pem_certificate, pem_certificate_chain
+
+    def extract_cert_and_chain(self, fullchain_pem):
+        pem_certificate = OpenSSL.crypto.dump_certificate(
+            OpenSSL.crypto.FILETYPE_PEM,
+            OpenSSL.crypto.load_certificate(
+                OpenSSL.crypto.FILETYPE_PEM, fullchain_pem
+            ),
+        ).decode()
+
+        if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \
+                and datetime.datetime.now() < datetime.datetime.strptime(
+                current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'):
+            pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA")
+        else:
+            pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip()
+
+        return pem_certificate, pem_certificate_chain
+
+    @retry(stop_max_attempt_number=5, wait_fixed=5000)
+    def setup_acme_client(self, authority):
+        if not authority.options:
+            raise InvalidAuthority("Invalid authority. Options not set")
+        options = {}
+
+        for option in json.loads(authority.options):
+            options[option["name"]] = option.get("value")
+        email = options.get("email", current_app.config.get("ACME_EMAIL"))
+        tel = options.get("telephone", current_app.config.get("ACME_TEL"))
+        directory_url = options.get(
+            "acme_url", current_app.config.get("ACME_DIRECTORY_URL")
+        )
+
+        existing_key = options.get(
+            "acme_private_key", current_app.config.get("ACME_PRIVATE_KEY")
+        )
+        existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR"))
+
+        if existing_key and existing_regr:
+            current_app.logger.debug("Reusing existing ACME account")
+            # Reuse the same account for each certificate issuance
+            key = jose.JWK.json_loads(existing_key)
+            regr = messages.RegistrationResource.json_loads(existing_regr)
+            current_app.logger.debug(
+                "Connecting with directory at {0}".format(directory_url)
+            )
+            net = ClientNetwork(key, account=regr)
+            client = BackwardsCompatibleClientV2(net, key, directory_url)
+            return client, {}
+        else:
+            # Create an account for each certificate issuance
+            key = jose.JWKRSA(key=generate_private_key("RSA2048"))
+
+            current_app.logger.debug("Creating a new ACME account")
+            current_app.logger.debug(
+                "Connecting with directory at {0}".format(directory_url)
+            )
+
+            net = ClientNetwork(key, account=None, timeout=3600)
+            client = BackwardsCompatibleClientV2(net, key, directory_url)
+            registration = client.new_account_and_tos(
+                messages.NewRegistration.from_data(email=email)
+            )
+
+            # if store_account is checked, add the private_key and registration resources to the options
+            if options['store_account']:
+                new_options = json.loads(authority.options)
+                # the key returned by fields_to_partial_json is missing the key type, so we add it manually
+                key_dict = key.fields_to_partial_json()
+                key_dict["kty"] = "RSA"
+                acme_private_key = {
+                    "name": "acme_private_key",
+                    "value": json.dumps(key_dict)
+                }
+                new_options.append(acme_private_key)
+
+                acme_regr = {
+                    "name": "acme_regr",
+                    "value": json.dumps({"body": {}, "uri": registration.uri})
+                }
+                new_options.append(acme_regr)
+
+                authorities_service.update_options(authority.id, options=json.dumps(new_options))
+
+            current_app.logger.debug("Connected: {0}".format(registration.uri))
+
+        return client, registration
+
+    def get_domains(self, options):
+        """
+        Fetches all domains currently requested
+        :param options:
+        :return:
+        """
+        current_app.logger.debug("Fetching domains")
+
+        domains = [options["common_name"]]
+        if options.get("extensions"):
+            for dns_name in options["extensions"]["sub_alt_names"]["names"]:
+                if dns_name.value not in domains:
+                    domains.append(dns_name.value)
+
+        current_app.logger.debug("Got these domains: {0}".format(domains))
+        return domains
+
+    def revoke_certificate(self, certificate):
+        if not self.reuse_account(certificate.authority):
+            raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.")
+        acme_client, _ = self.acme.setup_acme_client(certificate.authority)
+
+        fullchain_com = jose.ComparableX509(
+            OpenSSL.crypto.load_certificate(
+                OpenSSL.crypto.FILETYPE_PEM, certificate.body))
+
+        try:
+            acme_client.revoke(fullchain_com, 0)  # revocation reason = 0
+        except (errors.ConflictError, errors.ClientError, errors.Error) as e:
+            # Certificate already revoked.
+            current_app.logger.error("Certificate revocation failed with message: " + e.detail)
+            metrics.send("acme_revoke_certificate_failure", "counter", 1)
+            return False
+
+        current_app.logger.warning("Certificate succesfully revoked: " + certificate.name)
+        metrics.send("acme_revoke_certificate_success", "counter", 1)
+        return True
+
+
+class AcmeDnsHandler(AcmeHandler):
+
+    def __init__(self):
+        self.dns_providers_for_domain = {}
+        try:
+            self.all_dns_providers = dns_provider_service.get_all_dns_providers()
+        except Exception as e:
+            metrics.send("AcmeHandler_init_error", "counter", 1)
+            sentry.captureException()
+            current_app.logger.error(f"Unable to fetch DNS Providers: {e}")
+            self.all_dns_providers = []
+
+    def get_all_zones(self, dns_provider):
+        dns_provider_options = json.loads(dns_provider.credentials)
+        account_number = dns_provider_options.get("account_id")
+        dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
+        return dns_provider_plugin.get_zones(account_number=account_number)
+
+    def get_dns_challenges(self, host, authorizations):
+        """Get dns challenges for provided domain"""
+
+        domain_to_validate, is_wildcard = self.strip_wildcard(host)
+        dns_challenges = []
+        for authz in authorizations:
+            if not authz.body.identifier.value.lower() == domain_to_validate.lower():
+                continue
+            if is_wildcard and not authz.body.wildcard:
+                continue
+            if not is_wildcard and authz.body.wildcard:
+                continue
+            for combo in authz.body.challenges:
+                if isinstance(combo.chall, challenges.DNS01):
+                    dns_challenges.append(combo)
+
+        return dns_challenges
+
+    def get_dns_provider(self, type):
+        provider_types = {
+            "cloudflare": cloudflare,
+            "dyn": dyn,
+            "route53": route53,
+            "ultradns": ultradns,
+            "powerdns": powerdns
+        }
+        provider = provider_types.get(type)
+        if not provider:
+            raise UnknownProvider("No such DNS provider: {}".format(type))
+        return provider
+
+    def start_dns_challenge(
+            self,
+            acme_client,
+            account_number,
+            domain,
+            target_domain,
+            dns_provider,
+            order,
+            dns_provider_options,
+    ):
+        current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.")
+
+        change_ids = []
+        dns_challenges = self.get_dns_challenges(domain, order.authorizations)
+        host_to_validate, _ = self.strip_wildcard(target_domain)
+        host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
+
+        if not dns_challenges:
+            sentry.captureException()
+            metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1)
+            raise Exception("Unable to determine DNS challenges from authorizations")
+
+        for dns_challenge in dns_challenges:
+
+            # Only prepend '_acme-challenge' if not using CNAME redirection
+            if domain == target_domain:
+                host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
+
+            change_id = dns_provider.create_txt_record(
+                host_to_validate,
+                dns_challenge.validation(acme_client.client.net.key),
+                account_number,
+            )
+            change_ids.append(change_id)
+
+        return AuthorizationRecord(
+            domain, target_domain, order.authorizations, dns_challenges, change_ids
+        )
+
+    def complete_dns_challenge(self, acme_client, authz_record):
+        current_app.logger.debug(
+            "Finalizing DNS challenge for {0}".format(
+                authz_record.authz[0].body.identifier.value
+            )
+        )
+        dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
+        if not dns_providers:
+            metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1)
+            raise Exception(
+                "No DNS providers found for domain: {}".format(authz_record.target_domain)
+            )
+
+        for dns_provider in dns_providers:
+            # Grab account number (For Route53)
+            dns_provider_options = json.loads(dns_provider.credentials)
+            account_number = dns_provider_options.get("account_id")
+            dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
+            for change_id in authz_record.change_id:
+                try:
+                    dns_provider_plugin.wait_for_dns_change(
+                        change_id, account_number=account_number
+                    )
+                except Exception:
+                    metrics.send("complete_dns_challenge_error", "counter", 1)
+                    sentry.captureException()
+                    current_app.logger.debug(
+                        f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: "
+                        f"{account_number}",
+                        exc_info=True,
+                    )
+                    raise
+
+            for dns_challenge in authz_record.dns_challenge:
+                response = dns_challenge.response(acme_client.client.net.key)
+
+                verified = response.simple_verify(
+                    dns_challenge.chall,
+                    authz_record.target_domain,
+                    acme_client.client.net.key.public_key(),
+                )
+
+            if not verified:
+                metrics.send("complete_dns_challenge_verification_error", "counter", 1)
+                raise ValueError("Failed verification")
+
+            time.sleep(5)
+            res = acme_client.answer_challenge(dns_challenge, response)
+            current_app.logger.debug(f"answer_challenge response: {res}")
+
+    def get_authorizations(self, acme_client, order, order_info):
+        authorizations = []
+
+        for domain in order_info.domains:
+
+            # If CNAME exists, set host to the target address
+            target_domain = domain
+            if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False):
+                cname_result, _ = self.strip_wildcard(domain)
+                cname_result = challenges.DNS01().validation_domain_name(cname_result)
+                cname_result = self.get_cname(cname_result)
+                if cname_result:
+                    target_domain = cname_result
+                    self.autodetect_dns_providers(target_domain)
+
+            if not self.dns_providers_for_domain.get(target_domain):
+                metrics.send(
+                    "get_authorizations_no_dns_provider_for_domain", "counter", 1
+                )
+                raise Exception("No DNS providers found for domain: {}".format(target_domain))
+
+            for dns_provider in self.dns_providers_for_domain[target_domain]:
+                dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
+                dns_provider_options = json.loads(dns_provider.credentials)
+                account_number = dns_provider_options.get("account_id")
+                authz_record = self.start_dns_challenge(
+                    acme_client,
+                    account_number,
+                    domain,
+                    target_domain,
+                    dns_provider_plugin,
+                    order,
+                    dns_provider.options,
+                )
+                authorizations.append(authz_record)
+        return authorizations
+
+    def autodetect_dns_providers(self, domain):
+        """
+        Get DNS providers associated with a domain when it has not been provided for certificate creation.
+        :param domain:
+        :return: dns_providers: List of DNS providers that have the correct zone.
+        """
+        self.dns_providers_for_domain[domain] = []
+        match_length = 0
+        for dns_provider in self.all_dns_providers:
+            if not dns_provider.domains:
+                continue
+            for name in dns_provider.domains:
+                if name == domain or domain.endswith("." + name):
+                    if len(name) > match_length:
+                        self.dns_providers_for_domain[domain] = [dns_provider]
+                        match_length = len(name)
+                    elif len(name) == match_length:
+                        self.dns_providers_for_domain[domain].append(dns_provider)
+
+        return self.dns_providers_for_domain
+
+    def finalize_authorizations(self, acme_client, authorizations):
+        for authz_record in authorizations:
+            self.complete_dns_challenge(acme_client, authz_record)
+        for authz_record in authorizations:
+            dns_challenges = authz_record.dns_challenge
+            for dns_challenge in dns_challenges:
+                dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
+                for dns_provider in dns_providers:
+                    # Grab account number (For Route53)
+                    dns_provider_plugin = self.get_dns_provider(
+                        dns_provider.provider_type
+                    )
+                    dns_provider_options = json.loads(dns_provider.credentials)
+                    account_number = dns_provider_options.get("account_id")
+                    host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
+                    host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
+                    if authz_record.domain == authz_record.target_domain:
+                        host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate)
+                    dns_provider_plugin.delete_txt_record(
+                        authz_record.change_id,
+                        account_number,
+                        host_to_validate,
+                        dns_challenge.validation(acme_client.client.net.key),
+                    )
+
+        return authorizations
+
+    def cleanup_dns_challenges(self, acme_client, authorizations):
+        """
+        Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called
+        on an exception
+
+        :param acme_client:
+        :param account_number:
+        :param dns_provider:
+        :param authorizations:
+        :param dns_provider_options:
+        :return:
+        """
+        for authz_record in authorizations:
+            dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
+            for dns_provider in dns_providers:
+                # Grab account number (For Route53)
+                dns_provider_options = json.loads(dns_provider.credentials)
+                account_number = dns_provider_options.get("account_id")
+                dns_challenges = authz_record.dns_challenge
+                host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
+                host_to_validate = self.maybe_add_extension(
+                    host_to_validate, dns_provider_options
+                )
+
+                dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
+                for dns_challenge in dns_challenges:
+                    if authz_record.domain == authz_record.target_domain:
+                        host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
+                    try:
+                        dns_provider_plugin.delete_txt_record(
+                            authz_record.change_id,
+                            account_number,
+                            host_to_validate,
+                            dns_challenge.validation(acme_client.client.net.key),
+                        )
+                    except Exception as e:
+                        # If this fails, it's most likely because the record doesn't exist (It was already cleaned up)
+                        # or we're not authorized to modify it.
+                        metrics.send("cleanup_dns_challenges_error", "counter", 1)
+                        sentry.captureException()
+                        pass
+
+    def get_cname(self, domain):
+        """
+        :param domain: Domain name to look up a CNAME for.
+        :return: First CNAME target or False if no CNAME record exists.
+        """
+        try:
+            result = dns.resolver.query(domain, 'CNAME')
+            if len(result) > 0:
+                return str(result[0].target).rstrip('.')
+        except dns.exception.DNSException:
+            return False

lemur/plugins/lemur_acme/challenge_types.py

@@ -0,0 +1,260 @@
+"""
+.. module: lemur.plugins.lemur_acme.plugin
+    :platform: Unix
+    :synopsis: This module contains the different challenge types for ACME implementations
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Mathias Petermann <mathias.petermann@projektfokus.ch>
+"""
+import datetime
+import json
+
+from acme import challenges
+from acme.messages import errors, STATUS_VALID, ERROR_CODES
+from flask import current_app
+
+from lemur.authorizations import service as authorization_service
+from lemur.exceptions import LemurException, InvalidConfiguration
+from lemur.plugins.base import plugins
+from lemur.destinations import service as destination_service
+from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler
+
+
+class AcmeChallengeMissmatchError(LemurException):
+    pass
+
+
+class AcmeChallenge(object):
+    """
+    This is the base class, all ACME challenges will need to extend, allowing for future extendability
+    """
+
+    def create_certificate(self, csr, issuer_options):
+        """
+        Create the new certificate, using the provided CSR and issuer_options.
+        Right now this is basically a copy of the create_certificate methods in the AcmeHandlers, but should be cleaned
+        and tried to make use of the deploy and cleanup methods
+
+        :param csr:
+        :param issuer_options:
+        :return:
+        """
+        pass
+
+    def deploy(self, challenge, acme_client, validation_target):
+        """
+        In here the challenge validation is fetched and deployed somewhere that it can be validated by the provider
+
+        :param self:
+        :param challenge: the challenge object, must match for the challenge implementation
+        :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on
+        :param validation_target: an identifier for the validation target, e.g. the name of a DNS provider
+        """
+        raise NotImplementedError
+
+    def cleanup(self, challenge, acme_client, validation_target):
+        """
+        Ideally the challenge should be cleaned up, after the validation is done
+        :param challenge: Needed to identify the challenge to be removed
+        :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on
+        :param validation_target: Needed to remove the validation
+        """
+        raise NotImplementedError
+
+
+class AcmeHttpChallenge(AcmeChallenge):
+    challengeType = challenges.HTTP01
+
+    def create_certificate(self, csr, issuer_options):
+        """
+        Creates an ACME certificate using the HTTP-01 challenge.
+
+        :param csr:
+        :param issuer_options:
+        :return: :raise Exception:
+        """
+        self.acme = AcmeHandler()
+        authority = issuer_options.get("authority")
+        acme_client, registration = self.acme.setup_acme_client(authority)
+
+        orderr = acme_client.new_order(csr)
+
+        chall = []
+        deployed_challenges = []
+        all_pre_validated = True
+        for authz in orderr.authorizations:
+            # Choosing challenge.
+            # check if authorizations is already in a valid state
+            if authz.body.status != STATUS_VALID:
+                all_pre_validated = False
+                # authz.body.challenges is a set of ChallengeBody objects.
+                for i in authz.body.challenges:
+                    # Find the supported challenge.
+                    if isinstance(i.chall, challenges.HTTP01):
+                        chall.append(i)
+            else:
+                current_app.logger.info("{} already validated, skipping".format(authz.body.identifier.value))
+
+        if len(chall) == 0 and not all_pre_validated:
+            raise Exception('HTTP-01 challenge was not offered by the CA server at {}'.format(orderr.uri))
+        elif not all_pre_validated:
+            validation_target = None
+            for option in json.loads(issuer_options["authority"].options):
+                if option["name"] == "tokenDestination":
+                    validation_target = option["value"]
+
+            if validation_target is None:
+                raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge')
+
+            for challenge in chall:
+                try:
+                    response = self.deploy(challenge, acme_client, validation_target)
+                    deployed_challenges.append(challenge.chall.path)
+                    acme_client.answer_challenge(challenge, response)
+                except Exception as e:
+                    current_app.logger.error(e)
+                    raise Exception('Failure while trying to deploy token to configure destination. See logs for more information')
+
+            current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order")
+
+        try:
+            finalized_orderr = acme_client.poll_and_finalize(orderr,
+                                                             datetime.datetime.now() + datetime.timedelta(seconds=90))
+        except errors.ValidationError as validationError:
+            for authz in validationError.failed_authzrs:
+                for chall in authz.body.challenges:
+                    if chall.error:
+                        current_app.logger.error(
+                            "ValidationError occured of type {}, with message {}".format(chall.error.typ,
+                                                                                         ERROR_CODES[chall.error.code]))
+            raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.')
+
+        pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem)
+
+        if len(deployed_challenges) != 0:
+            for token_path in deployed_challenges:
+                self.cleanup(token_path, validation_target)
+
+        # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate
+        return pem_certificate, pem_certificate_chain, None
+
+    def deploy(self, challenge, acme_client, validation_target):
+
+        if not isinstance(challenge.chall, challenges.HTTP01):
+            raise AcmeChallengeMissmatchError(
+                'The provided challenge is not of type HTTP01, but instead of type {}'.format(
+                    challenge.__class__.__name__))
+
+        destination = destination_service.get(validation_target)
+
+        if destination is None:
+            raise Exception(
+                'Couldn\'t find the destination with name {}. Cant complete HTTP01 challenge'.format(validation_target))
+
+        destination_plugin = plugins.get(destination.plugin_name)
+
+        response, validation = challenge.response_and_validation(acme_client.net.key)
+
+        destination_plugin.upload_acme_token(challenge.chall.path, validation, destination.options)
+        current_app.logger.info("Uploaded HTTP-01 challenge token.")
+
+        return response
+
+    def cleanup(self, token_path, validation_target):
+        destination = destination_service.get(validation_target)
+
+        if destination is None:
+            current_app.logger.info(
+                'Couldn\'t find the destination with name {}, won\'t cleanup the challenge'.format(validation_target))
+
+        destination_plugin = plugins.get(destination.plugin_name)
+
+        destination_plugin.delete_acme_token(token_path, destination.options)
+        current_app.logger.info("Cleaned up HTTP-01 challenge token.")
+
+
+class AcmeDnsChallenge(AcmeChallenge):
+    challengeType = challenges.DNS01
+
+    def create_certificate(self, csr, issuer_options):
+        """
+        Creates an ACME certificate.
+
+        :param csr:
+        :param issuer_options:
+        :return: :raise Exception:
+        """
+        self.acme = AcmeDnsHandler()
+        authority = issuer_options.get("authority")
+        create_immediately = issuer_options.get("create_immediately", False)
+        acme_client, registration = self.acme.setup_acme_client(authority)
+        dns_provider = issuer_options.get("dns_provider", {})
+
+        if dns_provider:
+            dns_provider_options = dns_provider.options
+            credentials = json.loads(dns_provider.credentials)
+            current_app.logger.debug(
+                "Using DNS provider: {0}".format(dns_provider.provider_type)
+            )
+            dns_provider_plugin = __import__(
+                dns_provider.provider_type, globals(), locals(), [], 1
+            )
+            account_number = credentials.get("account_id")
+            provider_type = dns_provider.provider_type
+            if provider_type == "route53" and not account_number:
+                error = "Route53 DNS Provider {} does not have an account number configured.".format(
+                    dns_provider.name
+                )
+                current_app.logger.error(error)
+                raise InvalidConfiguration(error)
+        else:
+            dns_provider = {}
+            dns_provider_options = None
+            account_number = None
+            provider_type = None
+
+        domains = self.acme.get_domains(issuer_options)
+        if not create_immediately:
+            # Create pending authorizations that we'll need to do the creation
+            dns_authorization = authorization_service.create(
+                account_number, domains, provider_type
+            )
+            # Return id of the DNS Authorization
+            return None, None, dns_authorization.id
+
+        authorizations = self.acme.get_authorizations(
+            acme_client,
+            account_number,
+            domains,
+            dns_provider_plugin,
+            dns_provider_options,
+        )
+        self.acme.finalize_authorizations(
+            acme_client,
+            account_number,
+            dns_provider_plugin,
+            authorizations,
+            dns_provider_options,
+        )
+        pem_certificate, pem_certificate_chain = self.acme.request_certificate(
+            acme_client, authorizations, csr
+        )
+        # TODO add external ID (if possible)
+        return pem_certificate, pem_certificate_chain, None
+
+    def deploy(self, challenge, acme_client, validation_target):
+        pass
+
+    def cleanup(self, authorizations, acme_client, validation_target):
+        """
+        Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called
+        on an exception
+
+        :param authorizations: all the authorizations to be cleaned up
+        :param acme_client: an already bootstrapped acme_client, to avoid passing all issuer_options and so on
+        :param validation_target: Unused right now
+        :return:
+        """
+        acme = AcmeDnsHandler()
+        acme.cleanup_dns_challenges(acme_client, authorizations)

lemur/plugins/lemur_acme/plugin.py

@@ -11,408 +11,27 @@
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 .. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
 """
-import datetime
-import json
-import time
 
-import OpenSSL.crypto
-import josepy as jose
-from acme import challenges, errors, messages
-from acme.client import BackwardsCompatibleClientV2, ClientNetwork
-from acme.errors import PollError, TimeoutError, WildcardUnsupportedError
+from acme.errors import PollError, WildcardUnsupportedError
 from acme.messages import Error as AcmeError
 from botocore.exceptions import ClientError
 from flask import current_app
-
 from lemur.authorizations import service as authorization_service
-from lemur.common.utils import generate_private_key
 from lemur.dns_providers import service as dns_provider_service
-from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider
+from lemur.exceptions import InvalidConfiguration
 from lemur.extensions import metrics, sentry
+
 from lemur.plugins import lemur_acme as acme
 from lemur.plugins.bases import IssuerPlugin
-from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns
-from retrying import retry
-
-
-class AuthorizationRecord(object):
-    def __init__(self, host, authz, dns_challenge, change_id):
-        self.host = host
-        self.authz = authz
-        self.dns_challenge = dns_challenge
-        self.change_id = change_id
-
-
-class AcmeHandler(object):
-    def __init__(self):
-        self.dns_providers_for_domain = {}
-        try:
-            self.all_dns_providers = dns_provider_service.get_all_dns_providers()
-        except Exception as e:
-            metrics.send("AcmeHandler_init_error", "counter", 1)
-            sentry.captureException()
-            current_app.logger.error(f"Unable to fetch DNS Providers: {e}")
-            self.all_dns_providers = []
-
-    def get_dns_challenges(self, host, authorizations):
-        """Get dns challenges for provided domain"""
-
-        domain_to_validate, is_wildcard = self.strip_wildcard(host)
-        dns_challenges = []
-        for authz in authorizations:
-            if not authz.body.identifier.value.lower() == domain_to_validate.lower():
-                continue
-            if is_wildcard and not authz.body.wildcard:
-                continue
-            if not is_wildcard and authz.body.wildcard:
-                continue
-            for combo in authz.body.challenges:
-                if isinstance(combo.chall, challenges.DNS01):
-                    dns_challenges.append(combo)
-
-        return dns_challenges
-
-    def strip_wildcard(self, host):
-        """Removes the leading *. and returns Host and whether it was removed or not (True/False)"""
-        prefix = "*."
-        if host.startswith(prefix):
-            return host[len(prefix):], True
-        return host, False
-
-    def maybe_add_extension(self, host, dns_provider_options):
-        if dns_provider_options and dns_provider_options.get(
-            "acme_challenge_extension"
-        ):
-            host = host + dns_provider_options.get("acme_challenge_extension")
-        return host
-
-    def start_dns_challenge(
-        self,
-        acme_client,
-        account_number,
-        host,
-        dns_provider,
-        order,
-        dns_provider_options,
-    ):
-        current_app.logger.debug("Starting DNS challenge for {0}".format(host))
-
-        change_ids = []
-        dns_challenges = self.get_dns_challenges(host, order.authorizations)
-        host_to_validate, _ = self.strip_wildcard(host)
-        host_to_validate = self.maybe_add_extension(
-            host_to_validate, dns_provider_options
-        )
-
-        if not dns_challenges:
-            sentry.captureException()
-            metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1)
-            raise Exception("Unable to determine DNS challenges from authorizations")
-
-        for dns_challenge in dns_challenges:
-            change_id = dns_provider.create_txt_record(
-                dns_challenge.validation_domain_name(host_to_validate),
-                dns_challenge.validation(acme_client.client.net.key),
-                account_number,
-            )
-            change_ids.append(change_id)
-
-        return AuthorizationRecord(
-            host, order.authorizations, dns_challenges, change_ids
-        )
-
-    def complete_dns_challenge(self, acme_client, authz_record):
-        current_app.logger.debug(
-            "Finalizing DNS challenge for {0}".format(
-                authz_record.authz[0].body.identifier.value
-            )
-        )
-        dns_providers = self.dns_providers_for_domain.get(authz_record.host)
-        if not dns_providers:
-            metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1)
-            raise Exception(
-                "No DNS providers found for domain: {}".format(authz_record.host)
-            )
-
-        for dns_provider in dns_providers:
-            # Grab account number (For Route53)
-            dns_provider_options = json.loads(dns_provider.credentials)
-            account_number = dns_provider_options.get("account_id")
-            dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
-            for change_id in authz_record.change_id:
-                try:
-                    dns_provider_plugin.wait_for_dns_change(
-                        change_id, account_number=account_number
-                    )
-                except Exception:
-                    metrics.send("complete_dns_challenge_error", "counter", 1)
-                    sentry.captureException()
-                    current_app.logger.debug(
-                        f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: "
-                        f"{account_number}",
-                        exc_info=True,
-                    )
-                    raise
-
-            for dns_challenge in authz_record.dns_challenge:
-                response = dns_challenge.response(acme_client.client.net.key)
-
-                verified = response.simple_verify(
-                    dns_challenge.chall,
-                    authz_record.host,
-                    acme_client.client.net.key.public_key(),
-                )
-
-            if not verified:
-                metrics.send("complete_dns_challenge_verification_error", "counter", 1)
-                raise ValueError("Failed verification")
-
-            time.sleep(5)
-            res = acme_client.answer_challenge(dns_challenge, response)
-            current_app.logger.debug(f"answer_challenge response: {res}")
-
-    def request_certificate(self, acme_client, authorizations, order):
-        for authorization in authorizations:
-            for authz in authorization.authz:
-                authorization_resource, _ = acme_client.poll(authz)
-
-        deadline = datetime.datetime.now() + datetime.timedelta(seconds=360)
-
-        try:
-            orderr = acme_client.poll_and_finalize(order, deadline)
-
-        except (AcmeError, TimeoutError):
-            sentry.captureException(extra={"order_url": str(order.uri)})
-            metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri})
-            current_app.logger.error(
-                f"Unable to resolve Acme order: {order.uri}", exc_info=True
-            )
-            raise
-        except errors.ValidationError:
-            if order.fullchain_pem:
-                orderr = order
-            else:
-                raise
-
-        metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri})
-        current_app.logger.info(
-            f"Successfully resolved Acme order: {order.uri}", exc_info=True
-        )
-
-        pem_certificate = OpenSSL.crypto.dump_certificate(
-            OpenSSL.crypto.FILETYPE_PEM,
-            OpenSSL.crypto.load_certificate(
-                OpenSSL.crypto.FILETYPE_PEM, orderr.fullchain_pem
-            ),
-        ).decode()
-
-        if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \
-                and datetime.datetime.now() < datetime.datetime.strptime(
-                    current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'):
-            pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA")
-        else:
-            pem_certificate_chain = orderr.fullchain_pem[
-                len(pem_certificate) :  # noqa
-            ].lstrip()
-
-        current_app.logger.debug(
-            "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain))
-        )
-        return pem_certificate, pem_certificate_chain
-
-    @retry(stop_max_attempt_number=5, wait_fixed=5000)
-    def setup_acme_client(self, authority):
-        if not authority.options:
-            raise InvalidAuthority("Invalid authority. Options not set")
-        options = {}
-
-        for option in json.loads(authority.options):
-            options[option["name"]] = option.get("value")
-        email = options.get("email", current_app.config.get("ACME_EMAIL"))
-        tel = options.get("telephone", current_app.config.get("ACME_TEL"))
-        directory_url = options.get(
-            "acme_url", current_app.config.get("ACME_DIRECTORY_URL")
-        )
-
-        existing_key = options.get(
-            "acme_private_key", current_app.config.get("ACME_PRIVATE_KEY")
-        )
-        existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR"))
-
-        if existing_key and existing_regr:
-            # Reuse the same account for each certificate issuance
-            key = jose.JWK.json_loads(existing_key)
-            regr = messages.RegistrationResource.json_loads(existing_regr)
-            current_app.logger.debug(
-                "Connecting with directory at {0}".format(directory_url)
-            )
-            net = ClientNetwork(key, account=regr)
-            client = BackwardsCompatibleClientV2(net, key, directory_url)
-            return client, {}
-        else:
-            # Create an account for each certificate issuance
-            key = jose.JWKRSA(key=generate_private_key("RSA2048"))
-
-            current_app.logger.debug(
-                "Connecting with directory at {0}".format(directory_url)
-            )
-
-            net = ClientNetwork(key, account=None, timeout=3600)
-            client = BackwardsCompatibleClientV2(net, key, directory_url)
-            registration = client.new_account_and_tos(
-                messages.NewRegistration.from_data(email=email)
-            )
-            current_app.logger.debug("Connected: {0}".format(registration.uri))
-
-        return client, registration
-
-    def get_domains(self, options):
-        """
-        Fetches all domains currently requested
-        :param options:
-        :return:
-        """
-        current_app.logger.debug("Fetching domains")
-
-        domains = [options["common_name"]]
-        if options.get("extensions"):
-            for dns_name in options["extensions"]["sub_alt_names"]["names"]:
-                if dns_name.value not in domains:
-                    domains.append(dns_name.value)
-
-        current_app.logger.debug("Got these domains: {0}".format(domains))
-        return domains
-
-    def get_authorizations(self, acme_client, order, order_info):
-        authorizations = []
-
-        for domain in order_info.domains:
-            if not self.dns_providers_for_domain.get(domain):
-                metrics.send(
-                    "get_authorizations_no_dns_provider_for_domain", "counter", 1
-                )
-                raise Exception("No DNS providers found for domain: {}".format(domain))
-            for dns_provider in self.dns_providers_for_domain[domain]:
-                dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
-                dns_provider_options = json.loads(dns_provider.credentials)
-                account_number = dns_provider_options.get("account_id")
-                authz_record = self.start_dns_challenge(
-                    acme_client,
-                    account_number,
-                    domain,
-                    dns_provider_plugin,
-                    order,
-                    dns_provider.options,
-                )
-                authorizations.append(authz_record)
-        return authorizations
-
-    def autodetect_dns_providers(self, domain):
-        """
-        Get DNS providers associated with a domain when it has not been provided for certificate creation.
-        :param domain:
-        :return: dns_providers: List of DNS providers that have the correct zone.
-        """
-        self.dns_providers_for_domain[domain] = []
-        match_length = 0
-        for dns_provider in self.all_dns_providers:
-            if not dns_provider.domains:
-                continue
-            for name in dns_provider.domains:
-                if name == domain or domain.endswith("." + name):
-                    if len(name) > match_length:
-                        self.dns_providers_for_domain[domain] = [dns_provider]
-                        match_length = len(name)
-                    elif len(name) == match_length:
-                        self.dns_providers_for_domain[domain].append(dns_provider)
-
-        return self.dns_providers_for_domain
-
-    def finalize_authorizations(self, acme_client, authorizations):
-        for authz_record in authorizations:
-            self.complete_dns_challenge(acme_client, authz_record)
-        for authz_record in authorizations:
-            dns_challenges = authz_record.dns_challenge
-            for dns_challenge in dns_challenges:
-                dns_providers = self.dns_providers_for_domain.get(authz_record.host)
-                for dns_provider in dns_providers:
-                    # Grab account number (For Route53)
-                    dns_provider_plugin = self.get_dns_provider(
-                        dns_provider.provider_type
-                    )
-                    dns_provider_options = json.loads(dns_provider.credentials)
-                    account_number = dns_provider_options.get("account_id")
-                    host_to_validate, _ = self.strip_wildcard(authz_record.host)
-                    host_to_validate = self.maybe_add_extension(
-                        host_to_validate, dns_provider_options
-                    )
-                    dns_provider_plugin.delete_txt_record(
-                        authz_record.change_id,
-                        account_number,
-                        dns_challenge.validation_domain_name(host_to_validate),
-                        dns_challenge.validation(acme_client.client.net.key),
-                    )
-
-        return authorizations
-
-    def cleanup_dns_challenges(self, acme_client, authorizations):
-        """
-        Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called
-        on an exception
-
-        :param acme_client:
-        :param account_number:
-        :param dns_provider:
-        :param authorizations:
-        :param dns_provider_options:
-        :return:
-        """
-        for authz_record in authorizations:
-            dns_providers = self.dns_providers_for_domain.get(authz_record.host)
-            for dns_provider in dns_providers:
-                # Grab account number (For Route53)
-                dns_provider_options = json.loads(dns_provider.credentials)
-                account_number = dns_provider_options.get("account_id")
-                dns_challenges = authz_record.dns_challenge
-                host_to_validate, _ = self.strip_wildcard(authz_record.host)
-                host_to_validate = self.maybe_add_extension(
-                    host_to_validate, dns_provider_options
-                )
-                dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
-                for dns_challenge in dns_challenges:
-                    try:
-                        dns_provider_plugin.delete_txt_record(
-                            authz_record.change_id,
-                            account_number,
-                            dns_challenge.validation_domain_name(host_to_validate),
-                            dns_challenge.validation(acme_client.client.net.key),
-                        )
-                    except Exception as e:
-                        # If this fails, it's most likely because the record doesn't exist (It was already cleaned up)
-                        # or we're not authorized to modify it.
-                        metrics.send("cleanup_dns_challenges_error", "counter", 1)
-                        sentry.captureException()
-                        pass
-
-    def get_dns_provider(self, type):
-        provider_types = {
-            "cloudflare": cloudflare,
-            "dyn": dyn,
-            "route53": route53,
-            "ultradns": ultradns,
-            "powerdns": powerdns
-        }
-        provider = provider_types.get(type)
-        if not provider:
-            raise UnknownProvider("No such DNS provider: {}".format(type))
-        return provider
+from lemur.plugins.lemur_acme.acme_handlers import AcmeHandler, AcmeDnsHandler
+from lemur.plugins.lemur_acme.challenge_types import AcmeHttpChallenge, AcmeDnsChallenge
 
 
 class ACMEIssuerPlugin(IssuerPlugin):
     title = "Acme"
     slug = "acme-issuer"
     description = (
-        "Enables the creation of certificates via ACME CAs (including Let's Encrypt)"
+        "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the DNS-01 challenge"
     )
     version = acme.VERSION
 
@@ -424,7 +43,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
             "name": "acme_url",
             "type": "str",
             "required": True,
-            "validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/",
+            "validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/",
             "helpMessage": "Must be a valid web url starting with http[s]://",
         },
         {
@@ -437,7 +56,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
             "name": "email",
             "type": "str",
             "default": "",
-            "validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/",
+            "validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/",
             "helpMessage": "Email to use",
         },
         {
@@ -447,35 +66,20 @@ class ACMEIssuerPlugin(IssuerPlugin):
             "validation": "/^-----BEGIN CERTIFICATE-----/",
             "helpMessage": "Certificate to use",
         },
+        {
+            "name": "store_account",
+            "type": "bool",
+            "required": False,
+            "helpMessage": "Disable to create a new account for each ACME request",
+            "default": False,
+        }
     ]
 
     def __init__(self, *args, **kwargs):
         super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
 
-    def get_dns_provider(self, type):
-        self.acme = AcmeHandler()
-
-        provider_types = {
-            "cloudflare": cloudflare,
-            "dyn": dyn,
-            "route53": route53,
-            "ultradns": ultradns,
-            "powerdns": powerdns
-        }
-        provider = provider_types.get(type)
-        if not provider:
-            raise UnknownProvider("No such DNS provider: {}".format(type))
-        return provider
-
-    def get_all_zones(self, dns_provider):
-        self.acme = AcmeHandler()
-        dns_provider_options = json.loads(dns_provider.credentials)
-        account_number = dns_provider_options.get("account_id")
-        dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
-        return dns_provider_plugin.get_zones(account_number=account_number)
-
     def get_ordered_certificate(self, pending_cert):
-        self.acme = AcmeHandler()
+        self.acme = AcmeDnsHandler()
         acme_client, registration = self.acme.setup_acme_client(pending_cert.authority)
         order_info = authorization_service.get(pending_cert.external_id)
         if pending_cert.dns_provider_id:
@@ -521,7 +125,8 @@ class ACMEIssuerPlugin(IssuerPlugin):
         return cert
 
     def get_ordered_certificates(self, pending_certs):
-        self.acme = AcmeHandler()
+        self.acme = AcmeDnsHandler()
+        self.acme_dns_challenge = AcmeDnsChallenge()
         pending = []
         certs = []
         for pending_cert in pending_certs:
@@ -618,76 +223,22 @@ class ACMEIssuerPlugin(IssuerPlugin):
                     }
                 )
                 # Ensure DNS records get deleted
-                self.acme.cleanup_dns_challenges(
-                    entry["acme_client"], entry["authorizations"]
+                self.acme_dns_challenge.cleanup(
+                    entry["authorizations"], entry["acme_client"]
                 )
         return certs
 
     def create_certificate(self, csr, issuer_options):
         """
-        Creates an ACME certificate.
+        Creates an ACME certificate using the DNS-01 challenge.
 
         :param csr:
         :param issuer_options:
         :return: :raise Exception:
         """
-        self.acme = AcmeHandler()
-        authority = issuer_options.get("authority")
-        create_immediately = issuer_options.get("create_immediately", False)
-        acme_client, registration = self.acme.setup_acme_client(authority)
-        dns_provider = issuer_options.get("dns_provider", {})
+        acme_dns_challenge = AcmeDnsChallenge()
 
-        if dns_provider:
-            dns_provider_options = dns_provider.options
-            credentials = json.loads(dns_provider.credentials)
-            current_app.logger.debug(
-                "Using DNS provider: {0}".format(dns_provider.provider_type)
-            )
-            dns_provider_plugin = __import__(
-                dns_provider.provider_type, globals(), locals(), [], 1
-            )
-            account_number = credentials.get("account_id")
-            provider_type = dns_provider.provider_type
-            if provider_type == "route53" and not account_number:
-                error = "Route53 DNS Provider {} does not have an account number configured.".format(
-                    dns_provider.name
-                )
-                current_app.logger.error(error)
-                raise InvalidConfiguration(error)
-        else:
-            dns_provider = {}
-            dns_provider_options = None
-            account_number = None
-            provider_type = None
-
-        domains = self.acme.get_domains(issuer_options)
-        if not create_immediately:
-            # Create pending authorizations that we'll need to do the creation
-            dns_authorization = authorization_service.create(
-                account_number, domains, provider_type
-            )
-            # Return id of the DNS Authorization
-            return None, None, dns_authorization.id
-
-        authorizations = self.acme.get_authorizations(
-            acme_client,
-            account_number,
-            domains,
-            dns_provider_plugin,
-            dns_provider_options,
-        )
-        self.acme.finalize_authorizations(
-            acme_client,
-            account_number,
-            dns_provider_plugin,
-            authorizations,
-            dns_provider_options,
-        )
-        pem_certificate, pem_certificate_chain = self.acme.request_certificate(
-            acme_client, authorizations, csr
-        )
-        # TODO add external ID (if possible)
-        return pem_certificate, pem_certificate_chain, None
+        return acme_dns_challenge.create_certificate(csr, issuer_options)
 
     @staticmethod
     def create_authority(options):
@@ -715,3 +266,108 @@ class ACMEIssuerPlugin(IssuerPlugin):
     def cancel_ordered_certificate(self, pending_cert, **kwargs):
         # Needed to override issuer function.
         pass
+
+    def revoke_certificate(self, certificate, comments):
+        self.acme = AcmeDnsHandler()
+        return self.acme.revoke_certificate(certificate)
+
+
+class ACMEHttpIssuerPlugin(IssuerPlugin):
+    title = "Acme HTTP-01"
+    slug = "acme-http-issuer"
+    description = (
+        "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the HTTP-01 challenge"
+    )
+    version = acme.VERSION
+
+    author = "Netflix"
+    author_url = "https://github.com/netflix/lemur.git"
+
+    options = [
+        {
+            "name": "acme_url",
+            "type": "str",
+            "required": True,
+            "validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/",
+            "helpMessage": "Must be a valid web url starting with http[s]://",
+        },
+        {
+            "name": "telephone",
+            "type": "str",
+            "default": "",
+            "helpMessage": "Telephone to use",
+        },
+        {
+            "name": "email",
+            "type": "str",
+            "default": "",
+            "validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/",
+            "helpMessage": "Email to use",
+        },
+        {
+            "name": "certificate",
+            "type": "textarea",
+            "default": "",
+            "validation": "/^-----BEGIN CERTIFICATE-----/",
+            "helpMessage": "Certificate to use",
+        },
+        {
+            "name": "store_account",
+            "type": "bool",
+            "required": False,
+            "helpMessage": "Disable to create a new account for each ACME request",
+            "default": False,
+        },
+        {
+            "name": "tokenDestination",
+            "type": "destinationSelect",
+            "required": True,
+            "helpMessage": "The destination to use to deploy the token.",
+        },
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super(ACMEHttpIssuerPlugin, self).__init__(*args, **kwargs)
+
+    def create_certificate(self, csr, issuer_options):
+        """
+        Creates an ACME certificate using the HTTP-01 challenge.
+
+        :param csr:
+        :param issuer_options:
+        :return: :raise Exception:
+        """
+        acme_http_challenge = AcmeHttpChallenge()
+
+        return acme_http_challenge.create_certificate(csr, issuer_options)
+
+    @staticmethod
+    def create_authority(options):
+        """
+        Creates an authority, this authority is then used by Lemur to allow a user
+        to specify which Certificate Authority they want to sign their certificate.
+
+        :param options:
+        :return:
+        """
+        role = {"username": "", "password": "", "name": "acme"}
+        plugin_options = options.get("plugin", {}).get("plugin_options")
+        if not plugin_options:
+            error = "Invalid options for lemur_acme plugin: {}".format(options)
+            current_app.logger.error(error)
+            raise InvalidConfiguration(error)
+        # Define static acme_root based off configuration variable by default. However, if user has passed a
+        # certificate, use this certificate as the root.
+        acme_root = current_app.config.get("ACME_ROOT")
+        for option in plugin_options:
+            if option.get("name") == "certificate":
+                acme_root = option.get("value")
+        return acme_root, "", [role]
+
+    def cancel_ordered_certificate(self, pending_cert, **kwargs):
+        # Needed to override issuer function.
+        pass
+
+    def revoke_certificate(self, certificate, comments):
+        self.acme = AcmeHandler()
+        return self.acme.revoke_certificate(certificate)

lemur/plugins/lemur_acme/tests/test_acme_dns.py

@@ -1,16 +1,20 @@
 import unittest
 from unittest.mock import patch, Mock
 
+import josepy as jose
 from cryptography.x509 import DNSName
+from flask import Flask
 from lemur.plugins.lemur_acme import plugin
+from lemur.plugins.lemur_acme.acme_handlers import AuthorizationRecord
+from lemur.common.utils import generate_private_key
 from mock import MagicMock
 
 
-class TestAcme(unittest.TestCase):
-    @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
+class TestAcmeDns(unittest.TestCase):
+    @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
     def setUp(self, mock_dns_provider_service):
         self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin()
-        self.acme = plugin.AcmeHandler()
+        self.acme = plugin.AcmeDnsHandler()
         mock_dns_provider = Mock()
         mock_dns_provider.name = "cloudflare"
         mock_dns_provider.credentials = "{}"
@@ -20,6 +24,16 @@ class TestAcme(unittest.TestCase):
             "test.fakedomain.net": [mock_dns_provider],
         }
 
+        # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
+        # needed to run tests in dev environment without getting error 'Working outside of application context'.
+        _app = Flask('lemur_test_acme')
+        self.ctx = _app.app_context()
+        assert self.ctx
+        self.ctx.push()
+
+    def tearDown(self):
+        self.ctx.pop()
+
     @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1)
     def test_get_dns_challenges(self, mock_len):
         assert mock_len
@@ -37,36 +51,19 @@ class TestAcme(unittest.TestCase):
         result = yield self.acme.get_dns_challenges(host, mock_authz)
         self.assertEqual(result, mock_entry)
 
-    def test_strip_wildcard(self):
-        expected = ("example.com", False)
-        result = self.acme.strip_wildcard("example.com")
-        self.assertEqual(expected, result)
-
-        expected = ("example.com", True)
-        result = self.acme.strip_wildcard("*.example.com")
-        self.assertEqual(expected, result)
-
-    def test_authz_record(self):
-        a = plugin.AuthorizationRecord("host", "authz", "challenge", "id")
-        self.assertEqual(type(a), plugin.AuthorizationRecord)
-
     @patch("acme.client.Client")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
     @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1)
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges")
     def test_start_dns_challenge(
-            self, mock_get_dns_challenges, mock_len, mock_app, mock_acme
+            self, mock_get_dns_challenges, mock_len, mock_acme
     ):
         assert mock_len
         mock_order = Mock()
-        mock_app.logger.debug = Mock()
         mock_authz = Mock()
         mock_authz.body.resolved_combinations = []
         mock_entry = MagicMock()
-        from acme import challenges
 
-        c = challenges.DNS01()
-        mock_entry.chall = TestAcme.test_complete_dns_challenge_fail
+        mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail
         mock_authz.body.resolved_combinations.append(mock_entry)
         mock_acme.request_domain_challenges = Mock(return_value=mock_authz)
         mock_dns_provider = Mock()
@@ -77,16 +74,15 @@ class TestAcme(unittest.TestCase):
         iterator = iter(values)
         iterable.__iter__.return_value = iterator
         result = self.acme.start_dns_challenge(
-            mock_acme, "accountid", "host", mock_dns_provider, mock_order, {}
+            mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {}
         )
-        self.assertEqual(type(result), plugin.AuthorizationRecord)
+        self.assertEqual(type(result), AuthorizationRecord)
 
     @patch("acme.client.Client")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
     @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change")
     @patch("time.sleep")
     def test_complete_dns_challenge_success(
-            self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme
+            self, mock_sleep, mock_wait_for_dns_change, mock_acme
     ):
         mock_dns_provider = Mock()
         mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
@@ -95,7 +91,7 @@ class TestAcme(unittest.TestCase):
         mock_authz.dns_challenge.response = Mock()
         mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True)
         mock_authz.authz = []
-        mock_authz.host = "www.test.com"
+        mock_authz.target_domain = "www.test.com"
         mock_authz_record = Mock()
         mock_authz_record.body.identifier.value = "test"
         mock_authz.authz.append(mock_authz_record)
@@ -107,39 +103,38 @@ class TestAcme(unittest.TestCase):
         self.acme.complete_dns_challenge(mock_acme, mock_authz)
 
     @patch("acme.client.Client")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
     @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change")
     def test_complete_dns_challenge_fail(
-            self, mock_wait_for_dns_change, mock_current_app, mock_acme
+            self, mock_wait_for_dns_change, mock_acme
     ):
         mock_dns_provider = Mock()
         mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
 
+        mock_dns_challenge = Mock()
+        response = Mock()
+        response.simple_verify = Mock(return_value=False)
+        mock_dns_challenge.response = Mock(return_value=response)
+
         mock_authz = Mock()
-        mock_authz.dns_challenge.response = Mock()
-        mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False)
-        mock_authz.authz = []
-        mock_authz.host = "www.test.com"
+        mock_authz.dns_challenge = []
+        mock_authz.dns_challenge.append(mock_dns_challenge)
+
+        mock_authz.target_domain = "www.test.com"
         mock_authz_record = Mock()
         mock_authz_record.body.identifier.value = "test"
+        mock_authz.authz = []
         mock_authz.authz.append(mock_authz_record)
         mock_authz.change_id = []
         mock_authz.change_id.append("123")
-        mock_authz.dns_challenge = []
-        dns_challenge = Mock()
-        mock_authz.dns_challenge.append(dns_challenge)
-        self.assertRaises(
-            ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz)
-        )
+        with self.assertRaises(ValueError):
+            self.acme.complete_dns_challenge(mock_acme, mock_authz)
 
     @patch("acme.client.Client")
     @patch("OpenSSL.crypto", return_value="mock_cert")
     @patch("josepy.util.ComparableX509")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges")
     def test_request_certificate(
             self,
-            mock_current_app,
             mock_get_dns_challenges,
             mock_jose,
             mock_crypto,
@@ -156,7 +151,6 @@ class TestAcme(unittest.TestCase):
         mock_acme.fetch_chain = Mock(return_value="mock_chain")
         mock_crypto.dump_certificate = Mock(return_value=b"chain")
         mock_order = Mock()
-        mock_current_app.config = {}
         self.acme.request_certificate(mock_acme, [], mock_order)
 
     def test_setup_acme_client_fail(self):
@@ -165,30 +159,77 @@ class TestAcme(unittest.TestCase):
         with self.assertRaises(Exception):
             self.acme.setup_acme_client(mock_authority)
 
-    @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
-    def test_setup_acme_client_success(self, mock_current_app, mock_acme):
+    @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
+    def test_setup_acme_client_success_load_account_from_authority(self, mock_acme, mock_key_json_load):
         mock_authority = Mock()
-        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
+        mock_authority.id = 2
+        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
+                                 '{"name": "store_account", "value": true},' \
+                                 '{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, ' \
+                                 '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]'
+        mock_client = Mock()
+        mock_acme.return_value = mock_client
+
+        mock_key_json_load.return_value = jose.JWKRSA(key=generate_private_key("RSA2048"))
+
+        result_client, result_registration = self.acme.setup_acme_client(mock_authority)
+
+        mock_acme.new_account_and_tos.assert_not_called()
+        assert result_client
+        assert not result_registration
+
+    @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWKRSA.fields_to_partial_json")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
+    def test_setup_acme_client_success_store_new_account(self, mock_acme, mock_authorities_service,
+                                                         mock_key_generation):
+        mock_authority = Mock()
+        mock_authority.id = 2
+        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
+                                 '{"name": "store_account", "value": true}]'
+        mock_client = Mock()
+        mock_registration = Mock()
+        mock_registration.uri = "http://test.com"
+        mock_client.register = mock_registration
+        mock_client.agree_to_tos = Mock(return_value=True)
+        mock_client.new_account_and_tos.return_value = mock_registration
+        mock_acme.return_value = mock_client
+
+        mock_key_generation.return_value = {"n": "PwIOkViO"}
+
+        mock_authorities_service.update_options = Mock(return_value=True)
+
+        self.acme.setup_acme_client(mock_authority)
+
+        mock_authorities_service.update_options.assert_called_with(2, options='[{"name": "mock_name", "value": "mock_value"}, '
+        '{"name": "store_account", "value": true}, '
+        '{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, '
+        '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]')
+
+    @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
+    def test_setup_acme_client_success(self, mock_acme, mock_authorities_service):
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
+                                 '{"name": "store_account", "value": false}]'
         mock_client = Mock()
         mock_registration = Mock()
         mock_registration.uri = "http://test.com"
         mock_client.register = mock_registration
         mock_client.agree_to_tos = Mock(return_value=True)
         mock_acme.return_value = mock_client
-        mock_current_app.config = {}
         result_client, result_registration = self.acme.setup_acme_client(mock_authority)
+        mock_authorities_service.update_options.assert_not_called()
         assert result_client
         assert result_registration
 
-    @patch('lemur.plugins.lemur_acme.plugin.current_app')
-    def test_get_domains_single(self, mock_current_app):
+    def test_get_domains_single(self):
         options = {"common_name": "test.netflix.net"}
         result = self.acme.get_domains(options)
         self.assertEqual(result, [options["common_name"]])
 
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
-    def test_get_domains_multiple(self, mock_current_app):
+    def test_get_domains_multiple(self):
         options = {
             "common_name": "test.netflix.net",
             "extensions": {
@@ -200,8 +241,7 @@ class TestAcme(unittest.TestCase):
             result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"]
         )
 
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
-    def test_get_domains_san(self, mock_current_app):
+    def test_get_domains_san(self):
         options = {
             "common_name": "test.netflix.net",
             "extensions": {
@@ -213,10 +253,62 @@ class TestAcme(unittest.TestCase):
             result, [options["common_name"], "test2.netflix.net"]
         )
 
-    @patch(
-        "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge",
-        return_value="test",
-    )
+    def test_create_authority(self):
+        options = {
+            "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]}
+        }
+        acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options)
+        self.assertEqual(acme_root, "123")
+        self.assertEqual(b, "")
+        self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}])
+
+    @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
+    def test_get_dns_provider(self, mock_dns_provider_service):
+        provider = plugin.AcmeDnsHandler()
+        route53 = provider.get_dns_provider("route53")
+        assert route53
+        cloudflare = provider.get_dns_provider("cloudflare")
+        assert cloudflare
+        dyn = provider.get_dns_provider("dyn")
+        assert dyn
+
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate")
+    @patch("lemur.plugins.lemur_acme.challenge_types.authorization_service")
+    def test_create_certificate(
+            self,
+            mock_authorization_service,
+            mock_request_certificate,
+            mock_finalize_authorizations,
+            mock_get_authorizations,
+            mock_dns_provider_service,
+            mock_acme,
+    ):
+        provider = plugin.ACMEIssuerPlugin()
+        mock_authority = Mock()
+
+        mock_client = Mock()
+        mock_acme.return_value = (mock_client, "")
+
+        mock_dns_provider = Mock()
+        mock_dns_provider.credentials = '{"account_id": 1}'
+        mock_dns_provider.provider_type = "route53"
+        mock_dns_provider_service.get.return_value = mock_dns_provider
+
+        issuer_options = {
+            "authority": mock_authority,
+            "dns_provider": mock_dns_provider,
+            "common_name": "test.netflix.net",
+        }
+        csr = "123"
+        mock_request_certificate.return_value = ("pem_certificate", "chain")
+        result = provider.create_certificate(csr, issuer_options)
+        assert result
+
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.start_dns_challenge", return_value="test")
     def test_get_authorizations(self, mock_start_dns_challenge):
         mock_order = Mock()
         mock_order.body.identifiers = []
@@ -231,7 +323,7 @@ class TestAcme(unittest.TestCase):
         self.assertEqual(result, ["test"])
 
     @patch(
-        "lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge",
+        "lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.complete_dns_challenge",
         return_value="test",
     )
     def test_finalize_authorizations(self, mock_complete_dns_challenge):
@@ -249,51 +341,21 @@ class TestAcme(unittest.TestCase):
         result = self.acme.finalize_authorizations(mock_acme_client, mock_authz)
         self.assertEqual(result, mock_authz)
 
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
-    def test_create_authority(self, mock_current_app):
-        mock_current_app.config = Mock()
-        options = {
-            "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]}
-        }
-        acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options)
-        self.assertEqual(acme_root, "123")
-        self.assertEqual(b, "")
-        self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}])
-
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
-    @patch("lemur.plugins.lemur_acme.dyn.current_app")
-    @patch("lemur.plugins.lemur_acme.cloudflare.current_app")
-    @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
-    def test_get_dns_provider(
-            self,
-            mock_dns_provider_service,
-            mock_current_app_cloudflare,
-            mock_current_app_dyn,
-            mock_current_app,
-    ):
-        provider = plugin.ACMEIssuerPlugin()
-        route53 = provider.get_dns_provider("route53")
-        assert route53
-        cloudflare = provider.get_dns_provider("cloudflare")
-        assert cloudflare
-        dyn = provider.get_dns_provider("dyn")
-        assert dyn
-
     @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
     @patch("lemur.plugins.lemur_acme.plugin.authorization_service")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
     @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate")
     def test_get_ordered_certificate(
             self,
             mock_request_certificate,
             mock_finalize_authorizations,
             mock_get_authorizations,
+            mock_dns_provider_service_p,
             mock_dns_provider_service,
             mock_authorization_service,
-            mock_current_app,
             mock_acme,
     ):
         mock_client = Mock()
@@ -311,20 +373,20 @@ class TestAcme(unittest.TestCase):
         )
 
     @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
     @patch("lemur.plugins.lemur_acme.plugin.authorization_service")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service")
     @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate")
     def test_get_ordered_certificates(
             self,
             mock_request_certificate,
             mock_finalize_authorizations,
             mock_get_authorizations,
             mock_dns_provider_service,
+            mock_dns_provider_service_p,
             mock_authorization_service,
-            mock_current_app,
             mock_acme,
     ):
         mock_client = Mock()
@@ -349,41 +411,3 @@ class TestAcme(unittest.TestCase):
             result[1]["cert"],
             {"body": "pem_certificate", "chain": "chain", "external_id": "2"},
         )
-
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
-    @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
-    @patch("lemur.plugins.lemur_acme.plugin.authorization_service")
-    def test_create_certificate(
-            self,
-            mock_authorization_service,
-            mock_request_certificate,
-            mock_finalize_authorizations,
-            mock_get_authorizations,
-            mock_current_app,
-            mock_dns_provider_service,
-            mock_acme,
-    ):
-        provider = plugin.ACMEIssuerPlugin()
-        mock_authority = Mock()
-
-        mock_client = Mock()
-        mock_acme.return_value = (mock_client, "")
-
-        mock_dns_provider = Mock()
-        mock_dns_provider.credentials = '{"account_id": 1}'
-        mock_dns_provider.provider_type = "route53"
-        mock_dns_provider_service.get.return_value = mock_dns_provider
-
-        issuer_options = {
-            "authority": mock_authority,
-            "dns_provider": mock_dns_provider,
-            "common_name": "test.netflix.net",
-        }
-        csr = "123"
-        mock_request_certificate.return_value = ("pem_certificate", "chain")
-        result = provider.create_certificate(csr, issuer_options)
-        assert result

lemur/plugins/lemur_acme/tests/test_acme_handler.py

@@ -0,0 +1,112 @@
+import unittest
+from unittest.mock import patch, Mock
+
+from flask import Flask
+from cryptography.x509 import DNSName
+from lemur.plugins.lemur_acme import acme_handlers
+
+
+class TestAcmeHandler(unittest.TestCase):
+    def setUp(self):
+        self.acme = acme_handlers.AcmeHandler()
+
+        # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
+        # needed to run tests in dev environment without getting error 'Working outside of application context'.
+        _app = Flask('lemur_test_acme')
+        self.ctx = _app.app_context()
+        assert self.ctx
+        self.ctx.push()
+
+    def tearDown(self):
+        self.ctx.pop()
+
+    def test_strip_wildcard(self):
+        expected = ("example.com", False)
+        result = self.acme.strip_wildcard("example.com")
+        self.assertEqual(expected, result)
+
+        expected = ("example.com", True)
+        result = self.acme.strip_wildcard("*.example.com")
+        self.assertEqual(expected, result)
+
+    def test_authz_record(self):
+        a = acme_handlers.AuthorizationRecord("domain", "host", "authz", "challenge", "id")
+        self.assertEqual(type(a), acme_handlers.AuthorizationRecord)
+
+    def test_setup_acme_client_fail(self):
+        mock_authority = Mock()
+        mock_authority.options = []
+        with self.assertRaises(Exception):
+            self.acme.setup_acme_client(mock_authority)
+
+    def test_reuse_account_not_defined(self):
+        mock_authority = Mock()
+        mock_authority.options = []
+        with self.assertRaises(Exception):
+            self.acme.reuse_account(mock_authority)
+
+    def test_reuse_account_from_authority(self):
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "acme_private_key", "value": "PRIVATE_KEY"}, {"name": "acme_regr", "value": "ACME_REGR"}]'
+
+        self.assertTrue(self.acme.reuse_account(mock_authority))
+
+    @patch("lemur.plugins.lemur_acme.acme_handlers.current_app")
+    def test_reuse_account_from_config(self, mock_current_app):
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
+        mock_current_app.config = {"ACME_PRIVATE_KEY": "PRIVATE_KEY", "ACME_REGR": "ACME_REGR"}
+
+        self.assertTrue(self.acme.reuse_account(mock_authority))
+
+    def test_reuse_account_no_configuration(self):
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
+
+        self.assertFalse(self.acme.reuse_account(mock_authority))
+
+    @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service")
+    @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2")
+    def test_setup_acme_client_success(self, mock_acme, mock_authorities_service):
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \
+                                 '{"name": "store_account", "value": false}]'
+        mock_client = Mock()
+        mock_registration = Mock()
+        mock_registration.uri = "http://test.com"
+        mock_client.register = mock_registration
+        mock_client.agree_to_tos = Mock(return_value=True)
+        mock_acme.return_value = mock_client
+        result_client, result_registration = self.acme.setup_acme_client(mock_authority)
+        mock_authorities_service.update_options.assert_not_called()
+        assert result_client
+        assert result_registration
+
+    def test_get_domains_single(self):
+        options = {"common_name": "test.netflix.net"}
+        result = self.acme.get_domains(options)
+        self.assertEqual(result, [options["common_name"]])
+
+    def test_get_domains_multiple(self):
+        options = {
+            "common_name": "test.netflix.net",
+            "extensions": {
+                "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]}
+            },
+        }
+        result = self.acme.get_domains(options)
+        self.assertEqual(
+            result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"]
+        )
+
+    def test_get_domains_san(self):
+        options = {
+            "common_name": "test.netflix.net",
+            "extensions": {
+                "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]}
+            },
+        }
+        result = self.acme.get_domains(options)
+        self.assertEqual(
+            result, [options["common_name"], "test2.netflix.net"]
+        )

lemur/plugins/lemur_acme/tests/test_acme_http.py

@@ -0,0 +1,171 @@
+import unittest
+from unittest.mock import patch, Mock
+
+from flask import Flask
+from acme import challenges
+from lemur.plugins.lemur_acme import plugin
+
+
+class TestAcmeHttp(unittest.TestCase):
+
+    def setUp(self):
+        self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin()
+        self.acme = plugin.AcmeHandler()
+
+        # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
+        # needed to run tests in dev environment without getting error 'Working outside of application context'.
+        _app = Flask('lemur_test_acme')
+        self.ctx = _app.app_context()
+        assert self.ctx
+        self.ctx.push()
+
+    def tearDown(self):
+        self.ctx.pop()
+
+    def test_create_authority(self):
+        options = {
+            "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]}
+        }
+        acme_root, b, role = self.ACMEHttpIssuerPlugin.create_authority(options)
+        self.assertEqual(acme_root, "123")
+        self.assertEqual(b, "")
+        self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}])
+
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
+    @patch("lemur.plugins.base.manager.PluginManager.get")
+    @patch("lemur.plugins.lemur_acme.challenge_types.destination_service")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
+    @patch("lemur.plugins.lemur_acme.plugin.authorization_service")
+    def test_create_certificate(
+            self,
+            mock_authorization_service,
+            mock_request_certificate,
+            mock_destination_service,
+            mock_plugin_manager_get,
+            mock_acme,
+    ):
+        provider = plugin.ACMEHttpIssuerPlugin()
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]'
+
+        mock_order_resource = Mock()
+        mock_order_resource.authorizations = [Mock()]
+        mock_order_resource.authorizations[0].body.challenges = [Mock()]
+        mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = (Mock(), "Anything-goes")
+        mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(
+            token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f')
+
+        mock_client = Mock()
+        mock_client.new_order.return_value = mock_order_resource
+        mock_client.answer_challenge.return_value = True
+
+        mock_finalized_order = Mock()
+        mock_finalized_order.fullchain_pem = "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n"
+        mock_client.poll_and_finalize.return_value = mock_finalized_order
+
+        mock_acme.return_value = (mock_client, "")
+
+        mock_destination = Mock()
+        mock_destination.label = "mock-sftp-destination"
+        mock_destination.plugin_name = "SFTPDestinationPlugin"
+        mock_destination_service.get.return_value = mock_destination
+
+        mock_destination_plugin = Mock()
+        mock_destination_plugin.upload_acme_token.return_value = True
+        mock_plugin_manager_get.return_value = mock_destination_plugin
+
+        issuer_options = {
+            "authority": mock_authority,
+            "tokenDestination": "mock-sftp-destination",
+            "common_name": "test.netflix.net",
+        }
+        csr = "123"
+        mock_request_certificate.return_value = ("pem_certificate", "chain")
+        pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options)
+
+        self.assertEqual(pem_certificate, "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n")
+        self.assertEqual(pem_certificate_chain,
+                         "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n")
+
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
+    @patch("lemur.plugins.base.manager.PluginManager.get")
+    @patch("lemur.plugins.lemur_acme.challenge_types.destination_service")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
+    @patch("lemur.plugins.lemur_acme.plugin.authorization_service")
+    def test_create_certificate_missing_destination_token(
+            self,
+            mock_authorization_service,
+            mock_request_certificate,
+            mock_destination_service,
+            mock_plugin_manager_get,
+            mock_acme,
+    ):
+        provider = plugin.ACMEHttpIssuerPlugin()
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
+
+        mock_order_resource = Mock()
+        mock_order_resource.authorizations = [Mock()]
+        mock_order_resource.authorizations[0].body.challenges = [Mock()]
+        mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(
+            token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f')
+
+        mock_client = Mock()
+        mock_client.new_order.return_value = mock_order_resource
+        mock_acme.return_value = (mock_client, "")
+
+        mock_destination = Mock()
+        mock_destination.label = "mock-sftp-destination"
+        mock_destination.plugin_name = "SFTPDestinationPlugin"
+        mock_destination_service.get_by_label.return_value = mock_destination
+
+        mock_destination_plugin = Mock()
+        mock_destination_plugin.upload_acme_token.return_value = True
+        mock_plugin_manager_get.return_value = mock_destination_plugin
+
+        issuer_options = {
+            "authority": mock_authority,
+            "tokenDestination": "mock-sftp-destination",
+            "common_name": "test.netflix.net",
+        }
+        csr = "123"
+        mock_request_certificate.return_value = ("pem_certificate", "chain")
+        with self.assertRaisesRegex(Exception, "No token_destination configured"):
+            provider.create_certificate(csr, issuer_options)
+
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client")
+    @patch("lemur.plugins.base.manager.PluginManager.get")
+    @patch("lemur.plugins.lemur_acme.challenge_types.destination_service")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate")
+    @patch("lemur.plugins.lemur_acme.plugin.authorization_service")
+    def test_create_certificate_missing_http_challenge(
+            self,
+            mock_authorization_service,
+            mock_request_certificate,
+            mock_destination_service,
+            mock_plugin_manager_get,
+            mock_acme,
+    ):
+        provider = plugin.ACMEHttpIssuerPlugin()
+        mock_authority = Mock()
+        mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]'
+
+        mock_order_resource = Mock()
+        mock_order_resource.authorizations = [Mock()]
+        mock_order_resource.authorizations[0].body.challenges = [Mock()]
+        mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.DNS01(
+            token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f')
+
+        mock_client = Mock()
+        mock_client.new_order.return_value = mock_order_resource
+        mock_acme.return_value = (mock_client, "")
+
+        issuer_options = {
+            "authority": mock_authority,
+            "tokenDestination": "mock-sftp-destination",
+            "common_name": "test.netflix.net",
+        }
+        csr = "123"
+        mock_request_certificate.return_value = ("pem_certificate", "chain")
+        with self.assertRaisesRegex(Exception, "HTTP-01 challenge was not offered"):
+            provider.create_certificate(csr, issuer_options)

lemur/plugins/lemur_acme/tests/test_powerdns.py

@@ -1,5 +1,7 @@
 import unittest
 from unittest.mock import patch, Mock
+
+from flask import Flask
 from lemur.plugins.lemur_acme import plugin, powerdns
 
 
@@ -17,6 +19,16 @@ class TestPowerdns(unittest.TestCase):
             "test.fakedomain.net": [mock_dns_provider],
         }
 
+        # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
+        # needed to run tests in dev environment without getting error 'Working outside of application context'.
+        _app = Flask('lemur_test_acme')
+        self.ctx = _app.app_context()
+        assert self.ctx
+        self.ctx.push()
+
+    def tearDown(self):
+        self.ctx.pop()
+
     @patch("lemur.plugins.lemur_acme.powerdns.current_app")
     def test_get_zones(self, mock_current_app):
         account_number = "1234567890"

lemur/plugins/lemur_acme/tests/test_ultradns.py

@@ -1,6 +1,7 @@
 import unittest
 from unittest.mock import patch, Mock
 
+from flask import Flask
 from lemur.plugins.lemur_acme import plugin, ultradns
 from requests.models import Response
 
@@ -19,6 +20,16 @@ class TestUltradns(unittest.TestCase):
             "test.fakedomain.net": [mock_dns_provider],
         }
 
+        # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
+        # needed to run tests in dev environment without getting error 'Working outside of application context'.
+        _app = Flask('lemur_test_acme')
+        self.ctx = _app.app_context()
+        assert self.ctx
+        self.ctx.push()
+
+    def tearDown(self):
+        self.ctx.pop()
+
     @patch("lemur.plugins.lemur_acme.ultradns.requests")
     @patch("lemur.plugins.lemur_acme.ultradns.current_app")
     def test_ultradns_get_token(self, mock_current_app, mock_requests):

lemur/plugins/lemur_aws/plugin.py

@@ -32,13 +32,15 @@
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 .. moduleauthor:: Harm Weites <harm@weites.com>
 """
+
+import sys
 from acme.errors import ClientError
 from flask import current_app
-from lemur.extensions import sentry, metrics
 
-from lemur.plugins import lemur_aws as aws
+from lemur.extensions import sentry, metrics
+from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin
 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
-from lemur.plugins.lemur_aws import iam, s3, elb, ec2
+from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns
 
 
 def get_region_from_dns(dns):
@@ -406,3 +408,92 @@ class S3DestinationPlugin(ExportDestinationPlugin):
                 self.get_option("encrypt", options),
                 account_number=self.get_option("accountNumber", options),
             )
+
+    def upload_acme_token(self, token_path, token, options, **kwargs):
+        """
+         This is called from the acme http challenge
+        :param self:
+        :param token_path:
+        :param token:
+        :param options:
+        :param kwargs:
+        :return:
+        """
+        current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
+
+        function = f"{__name__}.{sys._getframe().f_code.co_name}"
+
+        account_number = self.get_option("accountNumber", options)
+        bucket_name = self.get_option("bucket", options)
+        prefix = self.get_option("prefix", options)
+        region = self.get_option("region", options)
+        filename = token_path.split("/")[-1]
+        if not prefix.endswith("/"):
+            prefix + "/"
+
+        res = s3.put(bucket_name=bucket_name,
+                     region_name=region,
+                     prefix=prefix + filename,
+                     data=token,
+                     encrypt=False,
+                     account_number=account_number)
+        res = "Success" if res else "Failure"
+        log_data = {
+            "function": function,
+            "message": "check if any valid certificate is revoked",
+            "result": res,
+            "bucket_name": bucket_name,
+            "filename": filename
+        }
+        current_app.logger.info(log_data)
+        metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
+                                                               "bucket_name": bucket_name,
+                                                               "filename": filename})
+
+
+class SNSNotificationPlugin(ExpirationNotificationPlugin):
+    title = "AWS SNS"
+    slug = "aws-sns"
+    description = "Sends notifications to AWS SNS"
+    version = aws.VERSION
+
+    author = "Jasmine Schladen <jschladen@netflix.com>"
+    author_url = "https://github.com/Netflix/lemur"
+
+    additional_options = [
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "required": True,
+            "validation": "[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access the SNS topic",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "required": True,
+            "validation": "[0-9a-z\\-]{1,25}",
+            "helpMessage": "Region in which the SNS topic is located, e.g. \"us-east-1\"",
+        },
+        {
+            "name": "topicName",
+            "type": "str",
+            "required": True,
+            # base topic name is 1-256 characters (alphanumeric plus underscore and hyphen)
+            "validation": "^[a-zA-Z0-9_\\-]{1,256}$",
+            "helpMessage": "The name of the topic to use for expiration notifications",
+        }
+    ]
+
+    def send(self, notification_type, message, excluded_targets, options, **kwargs):
+        """
+        While we receive a `targets` parameter here, it is unused, as the SNS topic is pre-configured in the
+        plugin configuration, and can't reasonably be changed dynamically.
+        """
+
+        topic_arn = f"arn:aws:sns:{self.get_option('region', options)}:" \
+                    f"{self.get_option('accountNumber', options)}:" \
+                    f"{self.get_option('topicName', options)}"
+
+        current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}")
+        sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options))

lemur/plugins/lemur_aws/s3.py

@@ -6,12 +6,15 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+from botocore.exceptions import ClientError
 from flask import current_app
+from lemur.extensions import sentry
+
 from .sts import sts_client
 
 
 @sts_client("s3", service_type="resource")
-def put(bucket_name, region, prefix, data, encrypt, **kwargs):
+def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
     """
     Use STS to write to an S3 bucket
     """
@@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs):
             ServerSideEncryption="AES256",
         )
     else:
-        bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
+        try:
+            bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
+            return True
+        except ClientError:
+            sentry.captureException()
+            return False
+
+
+@sts_client("s3", service_type="client")
+def delete(bucket_name, prefixed_object_name, **kwargs):
+    """
+    Use STS to delete an object
+    """
+    try:
+        response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name)
+        current_app.logger.debug(f"Delete data from S3."
+                                 f"Bucket: {bucket_name},"
+                                 f"Prefix: {prefixed_object_name},"
+                                 f"Status_code: {response}")
+        return response['ResponseMetadata']['HTTPStatusCode'] < 300
+    except ClientError:
+        sentry.captureException()
+        return False
+
+
+@sts_client("s3", service_type="client")
+def get(bucket_name, prefixed_object_name, **kwargs):
+    """
+    Use STS to get an object
+    """
+    try:
+        response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name)
+        current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
+                                 f"object_name: {prefixed_object_name}")
+        return response['Body'].read().decode("utf-8")
+    except ClientError:
+        sentry.captureException()
+        return None

lemur/plugins/lemur_aws/sns.py

@@ -0,0 +1,58 @@
+"""
+.. module: lemur.plugins.lemur_aws.sts
+    :platform: Unix
+    :copyright: (c) 2020 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Jasmine Schladen <jschladen@netflix.com>
+"""
+import json
+
+import arrow
+import boto3
+from flask import current_app
+
+
+def publish(topic_arn, certificates, notification_type, **kwargs):
+    sns_client = boto3.client("sns", **kwargs)
+    message_ids = {}
+    subject = "Lemur: {0} Notification".format(notification_type.capitalize())
+    for certificate in certificates:
+        message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject)
+
+    return message_ids
+
+
+def publish_single(sns_client, topic_arn, certificate, notification_type, subject):
+    response = sns_client.publish(
+        TopicArn=topic_arn,
+        Message=format_message(certificate, notification_type),
+        Subject=subject,
+    )
+
+    response_code = response["ResponseMetadata"]["HTTPStatusCode"]
+    if response_code != 200:
+        raise Exception(f"Failed to publish {notification_type} notification to SNS topic {topic_arn}. "
+                        f"SNS response: {response_code} {response}")
+
+    current_app.logger.info(f"AWS SNS message published to topic [{topic_arn}] with message ID {response['MessageId']}")
+    current_app.logger.debug(f"AWS SNS message published to topic [{topic_arn}]: [{response}]")
+
+    return response["MessageId"]
+
+
+def create_certificate_url(name):
+    return "https://{hostname}/#/certificates/{name}".format(
+        hostname=current_app.config.get("LEMUR_HOSTNAME"), name=name
+    )
+
+
+def format_message(certificate, notification_type):
+    json_message = {
+        "notification_type": notification_type,
+        "certificate_name": certificate["name"],
+        "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),  # 2047-12-31T22:00:00
+        "endpoints_detected": len(certificate["endpoints"]),
+        "owner": certificate["owner"],
+        "details": create_certificate_url(certificate["name"])
+    }
+    return json.dumps(json_message)

lemur/plugins/lemur_aws/tests/test_plugin.py

@@ -1,5 +1,82 @@
+import boto3
+from moto import mock_sts, mock_s3
+
+
 def test_get_certificates(app):
     from lemur.plugins.base import plugins
 
     p = plugins.get("aws-s3")
     assert p
+
+
+@mock_sts()
+@mock_s3()
+def test_upload_acme_token(app):
+    from lemur.plugins.base import plugins
+    from lemur.plugins.lemur_aws.s3 import get
+
+    bucket = "public-bucket"
+    account = "123456789012"
+    prefix = "some-path/more-path/"
+    token_content = "Challenge"
+    token_name = "TOKEN"
+    token_path = ".well-known/acme-challenge/" + token_name
+
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    s3_client = boto3.client('s3')
+    s3_client.create_bucket(Bucket=bucket)
+    p = plugins.get("aws-s3")
+
+    p.upload_acme_token(token_path=token_path,
+                        token_content=token_content,
+                        token=token_content,
+                        options=additional_options)
+
+    response = get(bucket_name=bucket,
+                   prefixed_object_name=prefix + token_name,
+                   encrypt=False,
+                   account_number=account)
+
+    # put data, and getting the same data
+    assert (response == token_content)

lemur/plugins/lemur_aws/tests/test_s3.py

@@ -0,0 +1,41 @@
+import boto3
+from moto import mock_sts, mock_s3
+
+
+@mock_sts()
+@mock_s3()
+def test_put_delete_s3_object(app):
+    from lemur.plugins.lemur_aws.s3 import put, delete, get
+
+    bucket = "public-bucket"
+    region = "us-east-1"
+    account = "123456789012"
+    path = "some-path/foo"
+    data = "dummy data"
+
+    s3_client = boto3.client('s3')
+    s3_client.create_bucket(Bucket=bucket)
+
+    put(bucket_name=bucket,
+        region_name=region,
+        prefix=path,
+        data=data,
+        encrypt=False,
+        account_number=account,
+        region=region)
+
+    response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+
+    # put data, and getting the same data
+    assert (response == data)
+
+    response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account)
+
+    # attempting to get thccle wrong data
+    assert (response is None)
+
+    delete(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+    response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+
+    # delete data, and getting the same data
+    assert (response is None)

lemur/plugins/lemur_aws/tests/test_sns.py

@@ -0,0 +1,123 @@
+import json
+from datetime import timedelta
+
+import arrow
+import boto3
+from moto import mock_sns, mock_sqs, mock_ses
+
+from lemur.certificates.schemas import certificate_notification_output_schema
+from lemur.plugins.lemur_aws.sns import format_message
+from lemur.plugins.lemur_aws.sns import publish
+from lemur.tests.factories import NotificationFactory, CertificateFactory
+from lemur.tests.test_messaging import verify_sender_email
+
+
+@mock_sns()
+def test_format(certificate, endpoint):
+    data = [certificate_notification_output_schema.dump(certificate).data]
+
+    for certificate in data:
+        expected_message = {
+            "notification_type": "expiration",
+            "certificate_name": certificate["name"],
+            "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
+            "endpoints_detected": 0,
+            "owner": certificate["owner"],
+            "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
+        }
+        assert expected_message == json.loads(format_message(certificate, "expiration"))
+
+
+@mock_sns()
+@mock_sqs()
+def create_and_subscribe_to_topic():
+    sns_client = boto3.client("sns", region_name="us-east-1")
+    topic_arn = sns_client.create_topic(Name='lemursnstest')["TopicArn"]
+
+    sqs_client = boto3.client("sqs", region_name="us-east-1")
+    queue = sqs_client.create_queue(QueueName="lemursnstestqueue")
+    queue_url = queue["QueueUrl"]
+    queue_arn = sqs_client.get_queue_attributes(QueueUrl=queue_url)["Attributes"]["QueueArn"]
+    sns_client.subscribe(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_arn)
+
+    return [topic_arn, sqs_client, queue_url]
+
+
+@mock_sns()
+@mock_sqs()
+def test_publish(certificate, endpoint):
+    data = [certificate_notification_output_schema.dump(certificate).data]
+
+    topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic()
+
+    message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1")
+    assert len(message_ids) == len(data)
+    received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
+
+    for certificate in data:
+        expected_message_id = message_ids[certificate["name"]]
+        actual_message = next(
+            (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None)
+        actual_json = json.loads(actual_message["Body"])
+        assert actual_json["Message"] == format_message(certificate, "expiration")
+        assert actual_json["Subject"] == "Lemur: Expiration Notification"
+
+
+def get_options():
+    return [
+        {"name": "interval", "value": 10},
+        {"name": "unit", "value": "days"},
+        {"name": "region", "value": "us-east-1"},
+        {"name": "accountNumber", "value": "123456789012"},
+        {"name": "topicName", "value": "lemursnstest"},
+    ]
+
+
+@mock_sns()
+@mock_sqs()
+@mock_ses()  # because email notifications are also sent
+def test_send_expiration_notification():
+    from lemur.notifications.messaging import send_expiration_notifications
+
+    verify_sender_email()  # emails are sent to owner and security; SNS only used for configured notification
+    topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic()
+
+    notification = NotificationFactory(plugin_name="aws-sns")
+    notification.options = get_options()
+
+    now = arrow.utcnow()
+    in_ten_days = now + timedelta(days=10, hours=1)  # a bit more than 10 days since we'll check in the future
+
+    certificate = CertificateFactory()
+    certificate.not_after = in_ten_days
+    certificate.notifications.append(notification)
+
+    assert send_expiration_notifications([]) == (3, 0)  # owner, SNS, and security
+
+    received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"]
+    assert len(received_messages) == 1
+    expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration")
+    actual_message = json.loads(received_messages[0]["Body"])["Message"]
+    assert actual_message == expected_message
+
+
+# Currently disabled as the SNS plugin doesn't support this type of notification
+# def test_send_rotation_notification(endpoint, source_plugin):
+#     from lemur.notifications.messaging import send_rotation_notification
+#     from lemur.deployment.service import rotate_certificate
+#
+#     notification = NotificationFactory(plugin_name="aws-sns")
+#     notification.options = get_options()
+#
+#     new_certificate = CertificateFactory()
+#     rotate_certificate(endpoint, new_certificate)
+#     assert endpoint.certificate == new_certificate
+#
+#     assert send_rotation_notification(new_certificate)
+
+
+# Currently disabled as the SNS plugin doesn't support this type of notification
+# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin):
+#     from lemur.notifications.messaging import send_pending_failure_notification
+#
+#     assert send_pending_failure_notification(pending_certificate)

lemur/plugins/lemur_digicert/plugin.py

@@ -21,7 +21,7 @@ import requests
 import sys
 from cryptography import x509
 from flask import current_app, g
-from lemur.common.utils import validate_conf
+from lemur.common.utils import validate_conf, convert_pkcs7_bytes_to_pem
 from lemur.extensions import metrics
 from lemur.plugins import lemur_digicert as digicert
 from lemur.plugins.bases import IssuerPlugin, SourcePlugin
@@ -37,7 +37,13 @@ def log_status_code(r, *args, **kwargs):
     :param kwargs:
     :return:
     """
+    log_data = {
+        "reason": (r.reason if r.reason else ""),
+        "status_code": r.status_code,
+        "url": (r.url if r.url else ""),
+    }
     metrics.send("digicert_status_code_{}".format(r.status_code), "counter", 1)
+    current_app.logger.info(log_data)
 
 
 def signature_hash(signing_algorithm):
@@ -171,11 +177,10 @@ def map_cis_fields(options, csr):
         "csr": csr,
         "signature_hash": signature_hash(options.get("signing_algorithm")),
         "validity": {
-            "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z"
+            "valid_to": validity_end.format("YYYY-MM-DDTHH:mm:ss") + "Z"
         },
         "organization": {
             "name": options["organization"],
-            "units": [options["organizational_unit"]],
         },
     }
     #  possibility to default to a SIGNING_ALGORITHM for a given profile
@@ -205,7 +210,7 @@ def handle_response(response):
     :return:
     """
     if response.status_code > 399:
-        raise Exception(response.json()["errors"][0]["message"])
+        raise Exception("DigiCert rejected request with the error:" + response.json()["errors"][0]["message"])
 
     return response.json()
 
@@ -216,13 +221,20 @@ def handle_cis_response(response):
     :param response:
     :return:
     """
-    if response.status_code > 399:
-        raise Exception(response.text)
+    if response.status_code == 404:
+        raise Exception("DigiCert: order not in issued state")
+    elif response.status_code == 406:
+        raise Exception("DigiCert: wrong header request format")
+    elif response.status_code > 399:
+        raise Exception("DigiCert rejected request with the error:" + response.text)
 
-    return response.json()
+    if response.url.endswith("download"):
+        return response.content
+    else:
+        return response.json()
 
 
-@retry(stop_max_attempt_number=10, wait_fixed=10000)
+@retry(stop_max_attempt_number=10, wait_fixed=1000)
 def get_certificate_id(session, base_url, order_id):
     """Retrieve certificate order id from Digicert API."""
     order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id)
@@ -233,17 +245,18 @@ def get_certificate_id(session, base_url, order_id):
     return response_data["certificate"]["id"]
 
 
-@retry(stop_max_attempt_number=10, wait_fixed=10000)
+@retry(stop_max_attempt_number=10, wait_fixed=1000)
 def get_cis_certificate(session, base_url, order_id):
-    """Retrieve certificate order id from Digicert API."""
-    certificate_url = "{0}/platform/cis/certificate/{1}".format(base_url, order_id)
-    session.headers.update({"Accept": "application/x-pem-file"})
+    """Retrieve certificate order id from Digicert API, including the chain"""
+    certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id)
+    session.headers.update({"Accept": "application/x-pkcs7-certificates"})
     response = session.get(certificate_url)
+    response_content = handle_cis_response(response)
 
-    if response.status_code == 404:
-        raise Exception("Order not in issued state.")
-
-    return response.content
+    cert_chain_pem = convert_pkcs7_bytes_to_pem(response_content)
+    if len(cert_chain_pem) < 3:
+        raise Exception("Missing the certificate chain")
+    return cert_chain_pem
 
 
 class DigiCertSourcePlugin(SourcePlugin):
@@ -447,7 +460,6 @@ class DigiCertCISSourcePlugin(SourcePlugin):
             "DIGICERT_CIS_API_KEY",
             "DIGICERT_CIS_URL",
             "DIGICERT_CIS_ROOTS",
-            "DIGICERT_CIS_INTERMEDIATES",
             "DIGICERT_CIS_PROFILE_NAMES",
         ]
         validate_conf(current_app, required_vars)
@@ -522,7 +534,6 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
             "DIGICERT_CIS_API_KEY",
             "DIGICERT_CIS_URL",
             "DIGICERT_CIS_ROOTS",
-            "DIGICERT_CIS_INTERMEDIATES",
             "DIGICERT_CIS_PROFILE_NAMES",
         ]
 
@@ -552,22 +563,15 @@ class DigiCertCISIssuerPlugin(IssuerPlugin):
         data = handle_cis_response(response)
 
         # retrieve certificate
-        certificate_pem = get_cis_certificate(self.session, base_url, data["id"])
+        certificate_chain_pem = get_cis_certificate(self.session, base_url, data["id"])
 
         self.session.headers.pop("Accept")
-        end_entity = pem.parse(certificate_pem)[0]
+        end_entity = certificate_chain_pem[0]
+        intermediate = certificate_chain_pem[1]
 
-        if "ECC" in issuer_options["key_type"]:
-            return (
-                "\n".join(str(end_entity).splitlines()),
-                current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name),
-                data["id"],
-            )
-
-        # By default return RSA
         return (
             "\n".join(str(end_entity).splitlines()),
-            current_app.config.get("DIGICERT_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name),
+            "\n".join(str(intermediate).splitlines()),
             data["id"],
         )
 

lemur/plugins/lemur_digicert/tests/test_digicert.py

@@ -121,9 +121,9 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority):
             "csr": CSR_STR,
             "additional_dns_names": names,
             "signature_hash": "sha256",
-            "organization": {"name": "Example, Inc.", "units": ["Example Org"]},
+            "organization": {"name": "Example, Inc."},
             "validity": {
-                "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM") + "Z"
+                "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:mm:ss") + "Z"
             },
             "profile_name": None,
         }
@@ -157,9 +157,9 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho
             "csr": CSR_STR,
             "additional_dns_names": names,
             "signature_hash": "sha256",
-            "organization": {"name": "Example, Inc.", "units": ["Example Org"]},
+            "organization": {"name": "Example, Inc."},
             "validity": {
-                "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z"
+                "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:mm:ss") + "Z"
             },
             "profile_name": None,
         }

lemur/plugins/lemur_email/plugin.py

@@ -17,16 +17,19 @@ from lemur.plugins.bases import ExpirationNotificationPlugin
 from lemur.plugins import lemur_email as email
 
 from lemur.plugins.lemur_email.templates.config import env
+from lemur.plugins.utils import get_plugin_option
 
 
-def render_html(template_name, message):
+def render_html(template_name, options, certificates):
     """
     Renders the html for our email notification.
 
     :param template_name:
-    :param message:
+    :param options:
+    :param certificates:
     :return:
     """
+    message = {"options": options, "certificates": certificates}
     template = env.get_template("{}.html".format(template_name))
     return template.render(
         dict(message=message, hostname=current_app.config.get("LEMUR_HOSTNAME"))
@@ -35,7 +38,7 @@ def render_html(template_name, message):
 
 def send_via_smtp(subject, body, targets):
     """
-    Attempts to deliver email notification via SES service.
+    Attempts to deliver email notification via SMTP.
 
     :param subject:
     :param body:
@@ -52,21 +55,26 @@ def send_via_smtp(subject, body, targets):
 
 def send_via_ses(subject, body, targets):
     """
-    Attempts to deliver email notification via SMTP.
+    Attempts to deliver email notification via SES service.
     :param subject:
     :param body:
     :param targets:
     :return:
     """
-    client = boto3.client("ses", region_name="us-east-1")
-    client.send_email(
-        Source=current_app.config.get("LEMUR_EMAIL"),
-        Destination={"ToAddresses": targets},
-        Message={
+    ses_region = current_app.config.get("LEMUR_SES_REGION", "us-east-1")
+    client = boto3.client("ses", region_name=ses_region)
+    source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN")
+    args = {
+        "Source": current_app.config.get("LEMUR_EMAIL"),
+        "Destination": {"ToAddresses": targets},
+        "Message": {
             "Subject": {"Data": subject, "Charset": "UTF-8"},
             "Body": {"Html": {"Data": body, "Charset": "UTF-8"}},
         },
-    )
+    }
+    if source_arn:
+        args["SourceArn"] = source_arn
+    client.send_email(**args)
 
 
 class EmailNotificationPlugin(ExpirationNotificationPlugin):
@@ -83,7 +91,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
             "name": "recipients",
             "type": "str",
             "required": True,
-            "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$",
+            "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$",
             "helpMessage": "Comma delimited list of email addresses",
         }
     ]
@@ -100,8 +108,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
 
         subject = "Lemur: {0} Notification".format(notification_type.capitalize())
 
-        data = {"options": options, "certificates": message}
-        body = render_html(notification_type, data)
+        body = render_html(notification_type, options, message)
 
         s_type = current_app.config.get("LEMUR_EMAIL_SENDER", "ses").lower()
 
@@ -110,3 +117,13 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
 
         elif s_type == "smtp":
             send_via_smtp(subject, body, targets)
+
+    @staticmethod
+    def filter_recipients(options, excluded_recipients, **kwargs):
+        notification_recipients = get_plugin_option("recipients", options)
+        if notification_recipients:
+            notification_recipients = notification_recipients.split(",")
+            # removing owner and security_email from notification_recipient
+            notification_recipients = [i for i in notification_recipients if i not in excluded_recipients]
+
+        return notification_recipients

lemur/plugins/lemur_email/templates/rotation.html

@@ -83,12 +83,12 @@
                                                                 <td width="32px"></td>
                                                                 <td width="16px"></td>
                                                                 <td style="line-height:1.2">
-                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.name }}</span>
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.certificates.name }}</span>
                                                                     <br>
                                                                     <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
-                                                                        <br>{{ certificate.owner }}
-                                                                        <br>{{ certificate.validityEnd | time }}
-                                                                        <a href="https://{{ hostname }}/#/certificates/{{ certificate.name }}" target="_blank">Details</a>
+                                                                        <br>{{ message.certificates.owner }}
+                                                                        <br>{{ message.certificates.validityEnd | time }}
+                                                                        <a href="https://{{ hostname }}/#/certificates/{{ message.certificates.name }}" target="_blank">Details</a>
                                                                     </span>
                                                                 </td>
                                                             </tr>
@@ -110,12 +110,12 @@
                                                             <td width="32px"></td>
                                                             <td width="16px"></td>
                                                             <td style="line-height:1.2">
-                                                                <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ certificate.replacedBy[0].name }}</span>
+                                                                <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.certificates.name }}</span>
                                                                 <br>
                                                                 <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
-                                                                        <br>{{ certificate.replacedBy[0].owner }}
-                                                                        <br>{{ certificate.replacedBy[0].validityEnd | time }}
-                                                                        <a href="https://{{ hostname }}/#/certificates/{{ certificate.replacedBy[0].name }}" target="_blank">Details</a>
+                                                                        <br>{{ message.certificates.owner }}
+                                                                        <br>{{ message.certificates.validityEnd | time }}
+                                                                        <a href="https://{{ hostname }}/#/certificates/{{ message.certificates.name }}" target="_blank">Details</a>
                                                                     </span>
                                                             </td>
                                                         </tr>
@@ -133,7 +133,7 @@
                                                     <table border="0" cellspacing="0" cellpadding="0"
                                                            style="margin-top:48px;margin-bottom:48px">
                                                         <tbody>
-                                                        {%  for endpoint in certificate.endpoints %}
+                                                        {%  for endpoint in message.certificates.endpoints %}
                                                             <tr valign="middle">
                                                                 <td width="32px"></td>
                                                                 <td width="16px"></td>

lemur/plugins/lemur_email/tests/test_email.py

@@ -1,36 +1,90 @@
 import os
-from lemur.plugins.lemur_email.templates.config import env
+from datetime import timedelta
 
+import arrow
+from moto import mock_ses
+
+from lemur.certificates.schemas import certificate_notification_output_schema
+from lemur.plugins.lemur_email.plugin import render_html
 from lemur.tests.factories import CertificateFactory
+from lemur.tests.test_messaging import verify_sender_email
 
 dir_path = os.path.dirname(os.path.realpath(__file__))
 
 
-def test_render(certificate, endpoint):
-    from lemur.certificates.schemas import certificate_notification_output_schema
+def get_options():
+    return [
+        {"name": "interval", "value": 10},
+        {"name": "unit", "value": "days"},
+        {"name": "recipients", "value": "person1@example.com,person2@example.com"},
+    ]
+
+
+def test_render_expiration(certificate, endpoint):
 
     new_cert = CertificateFactory()
     new_cert.replaces.append(certificate)
 
-    data = {
-        "certificates": [certificate_notification_output_schema.dump(certificate).data],
-        "options": [
-            {"name": "interval", "value": 10},
-            {"name": "unit", "value": "days"},
-        ],
-    }
+    assert render_html("expiration", get_options(), [certificate_notification_output_schema.dump(certificate).data])
 
-    template = env.get_template("{}.html".format("expiration"))
-
-    body = template.render(dict(message=data, hostname="lemur.test.example.com"))
-
-    template = env.get_template("{}.html".format("rotation"))
 
+def test_render_rotation(certificate, endpoint):
     certificate.endpoints.append(endpoint)
 
-    body = template.render(
-        dict(
-            certificate=certificate_notification_output_schema.dump(certificate).data,
-            hostname="lemur.test.example.com",
-        )
-    )
+    assert render_html("rotation", get_options(), certificate_notification_output_schema.dump(certificate).data)
+
+
+def test_render_rotation_failure(pending_certificate):
+    assert render_html("failed", get_options(), certificate_notification_output_schema.dump(pending_certificate).data)
+
+
+@mock_ses
+def test_send_expiration_notification():
+    from lemur.notifications.messaging import send_expiration_notifications
+    from lemur.tests.factories import CertificateFactory
+    from lemur.tests.factories import NotificationFactory
+
+    now = arrow.utcnow()
+    in_ten_days = now + timedelta(days=10, hours=1)  # a bit more than 10 days since we'll check in the future
+    certificate = CertificateFactory()
+    notification = NotificationFactory(plugin_name="email-notification")
+
+    certificate.not_after = in_ten_days
+    certificate.notifications.append(notification)
+    certificate.notifications[0].options = get_options()
+
+    verify_sender_email()
+    assert send_expiration_notifications([]) == (3, 0)  # owner, recipients (only counted as 1), and security
+
+
+@mock_ses
+def test_send_rotation_notification(endpoint, source_plugin):
+    from lemur.notifications.messaging import send_rotation_notification
+    from lemur.deployment.service import rotate_certificate
+
+    new_certificate = CertificateFactory()
+    rotate_certificate(endpoint, new_certificate)
+    assert endpoint.certificate == new_certificate
+
+    verify_sender_email()
+    assert send_rotation_notification(new_certificate)
+
+
+@mock_ses
+def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin):
+    from lemur.notifications.messaging import send_pending_failure_notification
+
+    verify_sender_email()
+    assert send_pending_failure_notification(pending_certificate)
+
+
+def test_filter_recipients(certificate, endpoint):
+    from lemur.plugins.lemur_email.plugin import EmailNotificationPlugin
+
+    options = [{"name": "recipients", "value": "security@example.com,bob@example.com,joe@example.com"}]
+    assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@example.com", "bob@example.com",
+                                                                      "joe@example.com"]
+    assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com"]) == ["bob@example.com",
+                                                                                            "joe@example.com"]
+    assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com", "bob@example.com",
+                                                               "joe@example.com"]) == []

lemur/plugins/lemur_entrust/plugin.py

@@ -1,9 +1,9 @@
-
 import arrow
 import requests
 import json
 import sys
 from flask import current_app
+from retrying import retry
 
 from lemur.plugins import lemur_entrust as entrust
 from lemur.plugins.bases import IssuerPlugin, SourcePlugin
@@ -20,7 +20,13 @@ def log_status_code(r, *args, **kwargs):
     :param kwargs:
     :return:
     """
+    log_data = {
+        "reason": (r.reason if r.reason else ""),
+        "status_code": r.status_code,
+        "url": (r.url if r.url else ""),
+    }
     metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1)
+    current_app.logger.info(log_data)
 
 
 def determine_end_date(end_date):
@@ -34,8 +40,7 @@ def determine_end_date(end_date):
 
     if not end_date:
         end_date = max_validity_end
-
-    if end_date > max_validity_end:
+    elif end_date > max_validity_end:
         end_date = max_validity_end
     return end_date.format('YYYY-MM-DD')
 
@@ -74,9 +79,7 @@ def process_options(options, client_id):
         "certType": product_type,
         "certExpiryDate": validity_end,
         # "keyType": "RSA", Entrust complaining about this parameter
-        "tracking": tracking_data,
-        "org": options.get("organization"),
-        "clientId": client_id
+        "tracking": tracking_data
     }
     return data
 
@@ -108,7 +111,7 @@ def get_client_id(my_response, organization):
 def handle_response(my_response):
     """
     Helper function for parsing responses from the Entrust API.
-    :param content:
+    :param my_response:
     :return: :raise Exception:
     """
     msg = {
@@ -121,22 +124,47 @@ def handle_response(my_response):
     }
 
     try:
-        d = json.loads(my_response.content)
+        data = json.loads(my_response.content)
     except ValueError:
         # catch an empty jason object here
-        d = {'response': 'No detailed message'}
-    s = my_response.status_code
-    if s > 399:
-        raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}")
+        data = {'response': 'No detailed message'}
+    status_code = my_response.status_code
+    if status_code > 399:
+        raise Exception(f"ENTRUST error: {msg.get(status_code, status_code)}\n{data['errors']}")
 
     log_data = {
         "function": f"{__name__}.{sys._getframe().f_code.co_name}",
         "message": "Response",
-        "status": s,
-        "response": d
+        "status": status_code,
+        "response": data
     }
     current_app.logger.info(log_data)
-    return d
+    if data == {'response': 'No detailed message'}:
+        # status if no data
+        return status_code
+    else:
+        #  return data from the response
+        return data
+
+
+@retry(stop_max_attempt_number=3, wait_fixed=5000)
+def order_and_download_certificate(session, url, data):
+    """
+    Helper function to place a certificacte order and download it
+    :param session:
+    :param url: Entrust endpoint url
+    :param data: CSR, and the required order details, such as validity length
+    :return: the cert chain
+    :raise Exception:
+    """
+    try:
+        response = session.post(url, json=data, timeout=(15, 40))
+    except requests.exceptions.Timeout:
+        raise Exception("Timeout for POST")
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Error for POST {e}")
+
+    return handle_response(response)
 
 
 class EntrustIssuerPlugin(IssuerPlugin):
@@ -210,14 +238,8 @@ class EntrustIssuerPlugin(IssuerPlugin):
         data = process_options(issuer_options, client_id)
         data["csr"] = csr
 
-        try:
-            response = self.session.post(url, json=data, timeout=(15, 40))
-        except requests.exceptions.Timeout:
-            raise Exception("Timeout for POST")
-        except requests.exceptions.RequestException as e:
-            raise Exception(f"Error for POST {e}")
+        response_dict = order_and_download_certificate(self.session, url, data)
 
-        response_dict = handle_response(response)
         external_id = response_dict['trackingId']
         cert = response_dict['endEntityCert']
         if len(response_dict['chainCerts']) < 2:
@@ -232,6 +254,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
 
         return cert, chain, external_id
 
+    @retry(stop_max_attempt_number=3, wait_fixed=1000)
     def revoke_certificate(self, certificate, comments):
         """Revoke an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
@@ -248,6 +271,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
         metrics.send("entrust_revoke_certificate", "counter", 1)
         return handle_response(response)
 
+    @retry(stop_max_attempt_number=3, wait_fixed=1000)
     def deactivate_certificate(self, certificate):
         """Deactivates an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
@@ -276,7 +300,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
     def get_ordered_certificate(self, order_id):
         raise NotImplementedError("Not implemented\n", self, order_id)
 
-    def canceled_ordered_certificate(self, pending_cert, **kwargs):
+    def cancel_ordered_certificate(self, pending_cert, **kwargs):
         raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs)
 
 

lemur/plugins/lemur_entrust/tests/test_entrust.py

@@ -3,6 +3,7 @@ from unittest.mock import patch, Mock
 import arrow
 from cryptography import x509
 from lemur.plugins.lemur_entrust import plugin
+from freezegun import freeze_time
 
 
 def config_mock(*args):
@@ -21,11 +22,18 @@ def config_mock(*args):
     return values[args[0]]
 
 
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_determine_end_date(mock_current_app):
+    with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):
+        assert arrow.get(2017, 12, 3).format('YYYY-MM-DD') == plugin.determine_end_date(0)  # 1 year + 1 month
+        assert arrow.get(2017, 3, 5).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2017, 3, 5))
+        assert arrow.get(2017, 12, 3).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2020, 5, 7))
+
+
 @patch("lemur.plugins.lemur_entrust.plugin.current_app")
 def test_process_options(mock_current_app, authority):
     mock_current_app.config.get = Mock(side_effect=config_mock)
-    plugin.determine_end_date = Mock(return_value=arrow.get(2020, 10, 7).format('YYYY-MM-DD'))
-
+    plugin.determine_end_date = Mock(return_value=arrow.get(2017, 11, 5).format('YYYY-MM-DD'))
     authority.name = "Entrust"
     names = [u"one.example.com", u"two.example.com", u"three.example.com"]
     options = {
@@ -35,7 +43,7 @@ def test_process_options(mock_current_app, authority):
         "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
         "organization": "Example, Inc.",
         "organizational_unit": "Example Org",
-        "validity_end": arrow.get(2020, 10, 7),
+        "validity_end": arrow.utcnow().shift(years=1, months=+1),
         "authority": authority,
     }
 
@@ -43,7 +51,7 @@ def test_process_options(mock_current_app, authority):
         "signingAlg": "SHA-2",
         "eku": "SERVER_AND_CLIENT_AUTH",
         "certType": "ADVANTAGE_SSL",
-        "certExpiryDate": arrow.get(2020, 10, 7).format('YYYY-MM-DD'),
+        "certExpiryDate": arrow.get(2017, 11, 5).format('YYYY-MM-DD'),
         "tracking": {
             "requesterName": mock_current_app.config.get("ENTRUST_NAME"),
             "requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"),

lemur/plugins/lemur_sftp/plugin.py

@@ -16,8 +16,10 @@
 
 .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov
 """
+from os import path
 
 import paramiko
+from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError
 
 from flask import current_app
 from lemur.plugins import lemur_sftp
@@ -47,7 +49,7 @@ class SFTPDestinationPlugin(DestinationPlugin):
             "type": "int",
             "required": True,
             "helpMessage": "The SFTP port, default is 22.",
-            "validation": "^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})",
+            "validation": r"^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})",
             "default": "22",
         },
         {
@@ -95,33 +97,15 @@ class SFTPDestinationPlugin(DestinationPlugin):
         },
     ]
 
-    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
-
-        current_app.logger.debug("SFTP destination plugin is started")
-
-        cn = common_name(parse_certificate(body))
+    def open_sftp_connection(self, options):
         host = self.get_option("host", options)
         port = self.get_option("port", options)
         user = self.get_option("user", options)
         password = self.get_option("password", options)
         ssh_priv_key = self.get_option("privateKeyPath", options)
         ssh_priv_key_pass = self.get_option("privateKeyPass", options)
-        dst_path = self.get_option("destinationPath", options)
-        export_format = self.get_option("exportFormat", options)
 
-        # prepare files for upload
-        files = {cn + ".key": private_key, cn + ".pem": body}
-
-        if cert_chain:
-            if export_format == "NGINX":
-                # assemble body + chain in the single file
-                files[cn + ".pem"] += "\n" + cert_chain
-
-            elif export_format == "Apache":
-                # store chain in the separate file
-                files[cn + ".ca.bundle.pem"] = cert_chain
-
-        # upload files
+        # delete files
         try:
             current_app.logger.debug(
                 "Connecting to {0}@{1}:{2}".format(user, host, port)
@@ -145,50 +129,170 @@ class SFTPDestinationPlugin(DestinationPlugin):
                 current_app.logger.error(
                     "No password or private key provided. Can't proceed"
                 )
-                raise paramiko.ssh_exception.AuthenticationException
+                raise AuthenticationException
 
             # open the sftp session inside the ssh connection
-            sftp = ssh.open_sftp()
+            return ssh.open_sftp(), ssh
 
-            # make sure that the destination path exist
-            try:
-                current_app.logger.debug("Creating {0}".format(dst_path))
-                sftp.mkdir(dst_path)
-            except IOError:
-                current_app.logger.debug("{0} already exist, resuming".format(dst_path))
-            try:
-                dst_path_cn = dst_path + "/" + cn
-                current_app.logger.debug("Creating {0}".format(dst_path_cn))
-                sftp.mkdir(dst_path_cn)
-            except IOError:
-                current_app.logger.debug(
-                    "{0} already exist, resuming".format(dst_path_cn)
-                )
+        except AuthenticationException as e:
+            current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
+            raise AuthenticationException("Couldn't connect to {0}, due to an Authentication exception.")
+        except NoValidConnectionsError as e:
+            current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
+            raise NoValidConnectionsError("Couldn't connect to {0}, possible timeout or invalid hostname")
 
-            # upload certificate files to the sftp destination
-            for filename, data in files.items():
+    # this is called when using this as a default destination plugin
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+
+        current_app.logger.debug("SFTP destination plugin is started")
+
+        cn = common_name(parse_certificate(body))
+        dst_path = self.get_option("destinationPath", options)
+        dst_path_cn = dst_path + "/" + cn
+        export_format = self.get_option("exportFormat", options)
+
+        # prepare files for upload
+        files = {cn + ".key": private_key, cn + ".pem": body}
+
+        if cert_chain:
+            if export_format == "NGINX":
+                # assemble body + chain in the single file
+                files[cn + ".pem"] += "\n" + cert_chain
+
+            elif export_format == "Apache":
+                # store chain in the separate file
+                files[cn + ".ca.bundle.pem"] = cert_chain
+
+        self.upload_file(dst_path_cn, files, options)
+
+    # this is called from the acme http challenge
+    def upload_acme_token(self, token_path, token, options, **kwargs):
+
+        current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge")
+
+        dst_path = self.get_option("destinationPath", options)
+
+        _, filename = path.split(token_path)
+
+        # prepare files for upload
+        files = {filename: token}
+
+        self.upload_file(dst_path, files, options)
+
+    # this is called from the acme http challenge
+    def delete_acme_token(self, token_path, options, **kwargs):
+        dst_path = self.get_option("destinationPath", options)
+
+        _, filename = path.split(token_path)
+
+        # prepare files for upload
+        files = {filename: None}
+
+        self.delete_file(dst_path, files, options)
+
+    # here the file is deleted
+    def delete_file(self, dst_path, files, options):
+
+        try:
+            # open the ssh and sftp sessions
+            sftp, ssh = self.open_sftp_connection(options)
+
+            # delete files
+            for filename, _ in files.items():
                 current_app.logger.debug(
-                    "Uploading {0} to {1}".format(filename, dst_path_cn)
+                    "Deleting {0} from {1}".format(filename, dst_path)
                 )
                 try:
-                    with sftp.open(dst_path_cn + "/" + filename, "w") as f:
-                        f.write(data)
-                except (PermissionError) as permerror:
+                    sftp.remove(path.join(dst_path, filename))
+                except PermissionError as permerror:
                     if permerror.errno == 13:
                         current_app.logger.debug(
-                            "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(filename, dst_path_cn)
+                            "Deleting {0} from {1} returned Permission Denied Error, making file writable and retrying".format(
+                                filename, dst_path)
                         )
-                        sftp.chmod(dst_path_cn + "/" + filename, 0o600)
-                        with sftp.open(dst_path_cn + "/" + filename, "w") as f:
-                            f.write(data)
-                # read only for owner, -r--------
-                sftp.chmod(dst_path_cn + "/" + filename, 0o400)
+                        sftp.chmod(path.join(dst_path, filename), 0o600)
+                        sftp.remove(path.join(dst_path, filename))
 
             ssh.close()
-
+        except (AuthenticationException, NoValidConnectionsError) as e:
+            raise e
         except Exception as e:
             current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
             try:
                 ssh.close()
             except BaseException:
                 pass
+
+    # here the file is uploaded for real, this helps to keep this class DRY
+    def upload_file(self, dst_path, files, options):
+
+        try:
+            # open the ssh and sftp sessions
+            sftp, ssh = self.open_sftp_connection(options)
+
+            # split the path into it's segments, so we can create it recursively
+            allparts = []
+            path_copy = dst_path
+            while True:
+                parts = path.split(path_copy)
+                if parts[0] == path_copy:  # sentinel for absolute paths
+                    allparts.insert(0, parts[0])
+                    break
+                elif parts[1] == path_copy:  # sentinel for relative paths
+                    allparts.insert(0, parts[1])
+                    break
+                else:
+                    path_copy = parts[0]
+                    allparts.insert(0, parts[1])
+
+            # make sure that the destination path exists, recursively
+            remote_path = allparts[0]
+            for part in allparts:
+                try:
+                    if part != "/" and part != "":
+                        remote_path = path.join(remote_path, part)
+                    sftp.stat(remote_path)
+                except IOError:
+                    current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path))
+                    try:
+                        sftp.mkdir(remote_path)
+                    except IOError as ioerror:
+                        current_app.logger.debug(
+                            "Couldn't create {0}, error message: {1}".format(remote_path, ioerror))
+
+            # upload certificate files to the sftp destination
+            for filename, data in files.items():
+                current_app.logger.debug(
+                    "Uploading {0} to {1}".format(filename, dst_path)
+                )
+                try:
+                    with sftp.open(path.join(dst_path, filename), "w") as f:
+                        f.write(data)
+                except PermissionError as permerror:
+                    if permerror.errno == 13:
+                        current_app.logger.debug(
+                            "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(
+                                filename, dst_path)
+                        )
+                        sftp.chmod(path.join(dst_path, filename), 0o600)
+                        with sftp.open(path.join(dst_path, filename), "w") as f:
+                            f.write(data)
+                # most likely the upload user isn't the webuser, -rw-r--r--
+                sftp.chmod(path.join(dst_path, filename), 0o644)
+
+            ssh.close()
+
+        except (AuthenticationException, NoValidConnectionsError) as e:
+            raise e
+        except Exception as e:
+            current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e))
+            try:
+                ssh.close()
+            except BaseException:
+                pass
+            message = ''
+            if hasattr(e, 'errors'):
+                for _, error in e.errors.items():
+                    message = error.strerror
+                raise Exception(
+                    'Couldn\'t upload file to {}, error message: {}'.format(self.get_option("host", options), message))

lemur/plugins/lemur_sftp/tests/test_sftp.py

@@ -0,0 +1,144 @@
+import unittest
+from unittest.mock import patch, Mock, MagicMock, mock_open
+
+from flask import Flask
+from lemur.plugins.lemur_sftp import plugin
+from paramiko.ssh_exception import AuthenticationException
+
+
+class TestSftp(unittest.TestCase):
+    def setUp(self):
+        self.sftp_destination = plugin.SFTPDestinationPlugin()
+        # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
+        # needed to run tests in dev environment without getting error 'Working outside of application context'.
+        _app = Flask('lemur_test_sftp')
+        self.ctx = _app.app_context()
+        assert self.ctx
+        self.ctx.push()
+
+    def tearDown(self):
+        self.ctx.pop()
+
+    def test_failing_ssh_connection(self):
+        dst_path = '/var/non-existent'
+        files = {'first-file': 'data'}
+        options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
+                   {'name': 'user', 'value': 'test_acme'}]
+
+        with self.assertRaises(AuthenticationException):
+            self.sftp_destination.upload_file(dst_path, files, options)
+
+    @patch("lemur.plugins.lemur_sftp.plugin.paramiko")
+    def test_upload_file_single_with_password(self, mock_paramiko):
+        dst_path = '/var/non-existent'
+        files = {'first-file': 'data'}
+        options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
+                   {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}]
+
+        mock_sftp = Mock()
+        mock_sftp.open = mock_open()
+
+        mock_ssh = mock_paramiko.SSHClient.return_value
+        mock_ssh.connect = MagicMock()
+        mock_ssh.open_sftp.return_value = mock_sftp
+
+        self.sftp_destination.upload_file(dst_path, files, options)
+
+        mock_sftp.open.assert_called_once_with('/var/non-existent/first-file', 'w')
+        handle = mock_sftp.open()
+        handle.write.assert_called_once_with('data')
+        mock_ssh.close.assert_called_once()
+        mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
+                                            password='test_password')
+
+    @patch("lemur.plugins.lemur_sftp.plugin.paramiko")
+    def test_upload_file_multiple_with_key(self, mock_paramiko):
+        dst_path = '/var/non-existent'
+        files = {'first-file': 'data', 'second-file': 'data2'}
+        options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
+                   {'name': 'user', 'value': 'test_acme'}, {'name': 'privateKeyPath', 'value': '/var/id_rsa'},
+                   {'name': 'privateKeyPass', 'value': 'ssh-key-password'}]
+
+        mock_sftp = Mock()
+        mock_sftp.open = mock_open()
+
+        mock_paramiko.RSAKey.from_private_key_file.return_value = 'ssh-rsa test-key'
+
+        mock_ssh = mock_paramiko.SSHClient.return_value
+        mock_ssh.connect = MagicMock()
+        mock_ssh.open_sftp.return_value = mock_sftp
+
+        self.sftp_destination.upload_file(dst_path, files, options)
+
+        mock_sftp.open.assert_called_with('/var/non-existent/second-file', 'w')
+        handle = mock_sftp.open()
+        handle.write.assert_called_with('data2')
+        mock_ssh.close.assert_called_once()
+
+        mock_paramiko.RSAKey.from_private_key_file.assert_called_with('/var/id_rsa', 'ssh-key-password')
+        mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
+                                            pkey='ssh-rsa test-key')
+
+    @patch("lemur.plugins.lemur_sftp.plugin.paramiko")
+    def test_upload_acme_token(self, mock_paramiko):
+        token_path = './well-known/acme-challenge/some-token-path'
+        token = 'token-data'
+        options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
+                   {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'},
+                   {'name': 'destinationPath', 'value': '/var/destination-path'}]
+
+        mock_sftp = Mock()
+        mock_sftp.open = mock_open()
+
+        mock_ssh = mock_paramiko.SSHClient.return_value
+        mock_ssh.connect = MagicMock()
+        mock_ssh.open_sftp.return_value = mock_sftp
+
+        self.sftp_destination.upload_acme_token(token_path, token, options)
+
+        mock_sftp.open.assert_called_once_with('/var/destination-path/some-token-path', 'w')
+        handle = mock_sftp.open()
+        handle.write.assert_called_once_with('token-data')
+        mock_ssh.close.assert_called_once()
+        mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
+                                            password='test_password')
+
+    @patch("lemur.plugins.lemur_sftp.plugin.paramiko")
+    def test_delete_file_with_password(self, mock_paramiko):
+        dst_path = '/var/non-existent'
+        files = {'first-file': None}
+        options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
+                   {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}]
+
+        mock_sftp = Mock()
+
+        mock_ssh = mock_paramiko.SSHClient.return_value
+        mock_ssh.connect = MagicMock()
+        mock_ssh.open_sftp.return_value = mock_sftp
+
+        self.sftp_destination.delete_file(dst_path, files, options)
+
+        mock_sftp.remove.assert_called_once_with('/var/non-existent/first-file')
+        mock_ssh.close.assert_called_once()
+        mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
+                                            password='test_password')
+
+    @patch("lemur.plugins.lemur_sftp.plugin.paramiko")
+    def test_delete_acme_token(self, mock_paramiko):
+        token_path = './well-known/acme-challenge/some-token-path'
+        options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'},
+                   {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'},
+                   {'name': 'destinationPath', 'value': '/var/destination-path'}]
+
+        mock_sftp = Mock()
+
+        mock_ssh = mock_paramiko.SSHClient.return_value
+        mock_ssh.connect = MagicMock()
+        mock_ssh.open_sftp.return_value = mock_sftp
+
+        self.sftp_destination.delete_acme_token(token_path, options)
+
+        mock_sftp.remove.assert_called_once_with('/var/destination-path/some-token-path')
+        mock_ssh.close.assert_called_once()
+        mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22',
+                                            password='test_password')

lemur/plugins/lemur_slack/plugin.py

@@ -58,26 +58,19 @@ def create_rotation_attachments(certificate):
         "title": certificate["name"],
         "title_link": create_certificate_url(certificate["name"]),
         "fields": [
+            {"title": "Owner", "value": certificate["owner"], "short": True},
             {
-                {"title": "Owner", "value": certificate["owner"], "short": True},
-                {
-                    "title": "Expires",
-                    "value": arrow.get(certificate["validityEnd"]).format(
-                        "dddd, MMMM D, YYYY"
-                    ),
-                    "short": True,
-                },
-                {
-                    "title": "Replaced By",
-                    "value": len(certificate["replaced"][0]["name"]),
-                    "short": True,
-                },
-                {
-                    "title": "Endpoints Rotated",
-                    "value": len(certificate["endpoints"]),
-                    "short": True,
-                },
-            }
+                "title": "Expires",
+                "value": arrow.get(certificate["validityEnd"]).format(
+                    "dddd, MMMM D, YYYY"
+                ),
+                "short": True,
+            },
+            {
+                "title": "Endpoints Rotated",
+                "value": len(certificate["endpoints"]),
+                "short": True,
+            },
         ],
     }
 
@@ -96,7 +89,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
             "name": "webhook",
             "type": "str",
             "required": True,
-            "validation": "^https:\/\/hooks\.slack\.com\/services\/.+$",
+            "validation": r"^https:\/\/hooks\.slack\.com\/services\/.+$",
             "helpMessage": "The url Slack told you to use for this integration",
         },
         {
@@ -119,6 +112,9 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
         """
         A typical check can be performed using the notify command:
         `lemur notify`
+
+        While we receive a `targets` parameter here, it is unused, as Slack webhooks do not allow
+        dynamic re-targeting of messages. The webhook itself specifies a channel.
         """
         attachments = None
         if notification_type == "expiration":
@@ -131,7 +127,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
             raise Exception("Unable to create message attachments")
 
         body = {
-            "text": "Lemur {0} Notification".format(notification_type.capitalize()),
+            "text": f"Lemur {notification_type.capitalize()} Notification",
             "attachments": attachments,
             "channel": self.get_option("recipients", options),
             "username": self.get_option("username", options),
@@ -140,8 +136,8 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
         r = requests.post(self.get_option("webhook", options), json.dumps(body))
 
         if r.status_code not in [200]:
-            raise Exception("Failed to send message")
+            raise Exception(f"Failed to send message. Slack response: {r.status_code} {body}")
 
-        current_app.logger.error(
-            "Slack response: {0} Message Body: {1}".format(r.status_code, body)
+        current_app.logger.info(
+            f"Slack response: {r.status_code} Message Body: {body}"
         )

lemur/plugins/lemur_slack/tests/test_slack.py

@@ -1,3 +1,12 @@
+from datetime import timedelta
+
+import arrow
+from moto import mock_ses
+
+from lemur.tests.factories import NotificationFactory, CertificateFactory
+from lemur.tests.test_messaging import verify_sender_email
+
+
 def test_formatting(certificate):
     from lemur.plugins.lemur_slack.plugin import create_expiration_attachments
     from lemur.certificates.schemas import certificate_notification_output_schema
@@ -21,3 +30,52 @@ def test_formatting(certificate):
     }
 
     assert attachment == create_expiration_attachments(data)[0]
+
+
+def get_options():
+    return [
+        {"name": "interval", "value": 10},
+        {"name": "unit", "value": "days"},
+        {"name": "webhook", "value": "https://slack.com/api/api.test"},
+    ]
+
+
+@mock_ses()  # because email notifications are also sent
+def test_send_expiration_notification():
+    from lemur.notifications.messaging import send_expiration_notifications
+
+    verify_sender_email()  # emails are sent to owner and security; Slack only used for configured notification
+
+    notification = NotificationFactory(plugin_name="slack-notification")
+    notification.options = get_options()
+
+    now = arrow.utcnow()
+    in_ten_days = now + timedelta(days=10, hours=1)  # a bit more than 10 days since we'll check in the future
+
+    certificate = CertificateFactory()
+    certificate.not_after = in_ten_days
+    certificate.notifications.append(notification)
+
+    assert send_expiration_notifications([]) == (3, 0)  # owner, Slack, and security
+
+
+# Currently disabled as the Slack plugin doesn't support this type of notification
+# def test_send_rotation_notification(endpoint, source_plugin):
+#     from lemur.notifications.messaging import send_rotation_notification
+#     from lemur.deployment.service import rotate_certificate
+#
+#     notification = NotificationFactory(plugin_name="slack-notification")
+#     notification.options = get_options()
+#
+#     new_certificate = CertificateFactory()
+#     rotate_certificate(endpoint, new_certificate)
+#     assert endpoint.certificate == new_certificate
+#
+#     assert send_rotation_notification(new_certificate, notification_plugin=notification.plugin)
+
+
+# Currently disabled as the Slack plugin doesn't support this type of notification
+# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin):
+#     from lemur.notifications.messaging import send_pending_failure_notification
+#
+#     assert send_pending_failure_notification(pending_certificate, notification_plugin=plugins.get("slack-notification"))

lemur/roles/service.py

@@ -128,3 +128,11 @@ def render(args):
         query = database.filter(query, Role, terms)
 
     return database.sort_and_page(query, Role, args)
+
+
+def get_or_create(role_name, description):
+    role = get_by_name(role_name)
+    if not role:
+        role = create(name=role_name, description=description)
+
+    return role

lemur/sources/service.py

@@ -264,13 +264,14 @@ def create(label, plugin_name, options, description=None):
     return database.create(source)
 
 
-def update(source_id, label, options, description):
+def update(source_id, label, plugin_name, options, description):
     """
     Updates an existing source.
 
     :param source_id:  Lemur assigned ID
     :param label: Source common name
     :param options:
+    :param plugin_name:
     :param description:
     :rtype : Source
     :return:
@@ -278,6 +279,7 @@ def update(source_id, label, options, description):
     source = get(source_id)
 
     source.label = label
+    source.plugin_name = plugin_name
     source.options = options
     source.description = description
 

lemur/sources/views.py

@@ -284,6 +284,7 @@ class Sources(AuthenticatedResource):
         return service.update(
             source_id,
             data["label"],
+            data["plugin"]["slug"],
             data["plugin"]["plugin_options"],
             data["description"],
         )

lemur/static/app/angular/authorities/authority/authority.js

@@ -34,7 +34,7 @@ angular.module('lemur')
     };
   })
 
-  .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster)  {
+  .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster, DestinationService)  {
     $scope.authority = LemurRestangular.restangularizeElement(null, {}, 'authorities');
     // set the defaults
     AuthorityService.getDefaults($scope.authority).then(function () {
@@ -52,6 +52,12 @@ angular.module('lemur')
       });
     });
 
+    $scope.getDestinations = function() {
+      return DestinationService.findDestinationsByName('').then(function(destinations) {
+        $scope.destinations = destinations;
+      });
+    };
+
     $scope.getAuthoritiesByName = function (value) {
       return AuthorityService.findAuthorityByName(value).then(function (authorities) {
         $scope.authorities = authorities;

lemur/static/app/angular/authorities/authority/options.tpl.html

@@ -20,8 +20,10 @@
       Key Type
     </label>
     <div class="col-sm-10">
-      <select class="form-control" ng-model="authority.keyType" ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
-      'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2']" ng-init="authority.keyType = 'RSA2048'"></select>
+      <select class="form-control" ng-model="authority.keyType"
+              ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME256V1', 'ECCSECP384R1', 'ECCSECP521R1']"
+              ng-init="authority.keyType = 'RSA2048'">
+      </select>
     </div>
   </div>
   <div ng-show="authority.sensitivity == 'high'" class="form-group">
@@ -64,11 +66,28 @@
           <div class="col-sm-10">
             <input name="sub" ng-if="item.type == 'int'" type="number" ng-pattern="item.validation?item.validation:'^[0-9]+$'"
                    class="form-control" ng-model="item.value"/>
+
             <select name="sub" ng-if="item.type == 'select'" class="form-control" ng-options="i for i in item.available"
                     ng-model="item.value"></select>
+
+            <!-- DestSelect options -->
+            <ui-select class="input-md" ng-model="item.value" theme="bootstrap" title="choose a destination" ng-if="item.type == 'destinationSelect'">
+                <ui-select-match placeholder="select an destination...">{{$select.selected.label}}</ui-select-match>
+                <ui-select-choices class="form-control"
+                             refresh="getDestinations()"
+                             refresh-delay="300"
+                             repeat="destination.id as destination in destinations | filter: $select.search">
+                    <div ng-bind-html="destination.label | highlight: $select.search"></div>
+                    <small>
+                        <span ng-bind-html="''+destination.description | highlight: $select.search"></span>
+                    </small>
+                </ui-select-choices>
+            </ui-select>
+
             <input name="sub" ng-if="item.type == 'bool'" class="form-control" type="checkbox" ng-model="item.value">
             <input name="sub" ng-if="item.type == 'str'" type="text" class="form-control" ng-model="item.value"/>
               <textarea name="sub" ng-if="item.type == 'textarea'" class="form-control" ng-model="item.value"></textarea>
+
             <div ng-if="item.type == 'export-plugin'">
               <form name="exportForm" class="form-horizontal" role="form" novalidate>
                 <select class="form-control" ng-model="item.value"

lemur/static/app/angular/certificates/certificate/certificate.js

@@ -190,7 +190,7 @@ angular.module('lemur')
     function populateValidityDateAsPerDefault(certificate) {
       // calculate start and end date as per default validity
       let startDate = new Date(), endDate = new Date();
-      endDate.setDate(startDate.getDate() + certificate.authority.authorityCertificate.defaultValidityDays);
+      endDate.setDate(startDate.getDate() + certificate.authority.defaultValidityDays);
       certificate.validityStart = startDate;
       certificate.validityEnd = endDate;
     }
@@ -255,9 +255,6 @@ angular.module('lemur')
     $scope.certificate.replacedBy = []; // should not clone 'replaced by' info
     $scope.certificate.removeReplaces(); // should not clone 'replacement cert' info
 
-    if(!$scope.certificate.keyType) {
-      $scope.certificate.keyType = 'RSA2048'; // default algo to select during clone if backend did not return algo
-    }
     CertificateService.getDefaults($scope.certificate);
   });
 
@@ -362,7 +359,7 @@ angular.module('lemur')
     function populateValidityDateAsPerDefault(certificate) {
       // calculate start and end date as per default validity
       let startDate = new Date(), endDate = new Date();
-      endDate.setDate(startDate.getDate() + certificate.authority.authorityCertificate.defaultValidityDays);
+      endDate.setDate(startDate.getDate() + certificate.authority.defaultValidityDays);
       certificate.validityStart = startDate;
       certificate.validityEnd = endDate;
     }

lemur/static/app/angular/certificates/certificate/options.tpl.html

@@ -32,10 +32,7 @@
         </label>
         <div class="col-sm-10">
           <select class="form-control" ng-model="certificate.keyType"
-                  ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1',
-                                                    'ECCSECP224R1', 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1',
-                                                    'ECCSECT163K1', 'ECCSECT233K1', 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1',
-                                                    'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', 'ECCSECT409R1', 'ECCSECT571R2']"
+                  ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME256V1', 'ECCSECP384R1']"
                   ng-init="certificate.keyType = 'RSA2048'"></select>
         </div>
       </div>

lemur/static/app/angular/certificates/certificate/tracking.tpl.html

@@ -139,7 +139,7 @@
       <div class="col-sm-4">
         <div class="btn-group btn-group-toggle" data-toggle="buttons">
           <label class="btn btn-info" ng-model="certificate.validityType" uib-btn-radio="'defaultDays'" ng-click="clearDates()">
-              Default ({{certificate.authority.authorityCertificate.defaultValidityDays}} days)</label>
+              Default ({{certificate.authority.defaultValidityDays}} days)</label>
           <label class="btn btn-info" ng-model="certificate.validityType" uib-btn-radio="'customDates'" ng-change="clearDates()">Custom</label>
         </div>
       </div>

lemur/static/app/angular/certificates/services.js

@@ -172,12 +172,12 @@ angular.module('lemur')
           // Minimum end date will be same as selected start date
           this.authority.authorityCertificate.minValidityEnd = value;
 
-          if(!this.authority.authorityCertificate || !this.authority.authorityCertificate.maxIssuanceDays) {
+          if(!this.authority.maxIssuanceDays) {
             this.authority.authorityCertificate.maxValidityEnd = this.authority.authorityCertificate.notAfter;
           } else {
             // Move max end date by maxIssuanceDays
             let endDate = new Date(value);
-            endDate.setDate(endDate.getDate() + this.authority.authorityCertificate.maxIssuanceDays);
+            endDate.setDate(endDate.getDate() + this.authority.maxIssuanceDays);
             this.authority.authorityCertificate.maxValidityEnd = endDate;
           }
         }
@@ -289,6 +289,11 @@ angular.module('lemur')
         if (certificate.dnsProviderId) {
           certificate.dnsProvider = {id: certificate.dnsProviderId};
         }
+
+        if(!certificate.keyType) {
+          certificate.keyType = 'RSA2048'; // default algo to select during clone if backend did not return algo
+        }
+
       });
     };
 
@@ -301,7 +306,7 @@ angular.module('lemur')
     };
 
     CertificateService.updateNotify = function (certificate) {
-      return certificate.put();
+      return certificate.post();
     };
 
     CertificateService.export = function (certificate) {

lemur/static/app/angular/destinations/destination/destination.js

@@ -52,19 +52,19 @@ angular.module('lemur')
           if (plugin.slug === $scope.destination.plugin.slug) {
             plugin.pluginOptions = $scope.destination.plugin.pluginOptions;
             $scope.destination.plugin = plugin;
-            _.each($scope.destination.plugin.pluginOptions, function (option) {
-              if (option.type === 'export-plugin') {
-                PluginService.getByType('export').then(function (plugins) {
-                  $scope.exportPlugins = plugins;
+            PluginService.getByType('export').then(function (plugins) {
+              $scope.exportPlugins = plugins;
 
+              _.each($scope.destination.plugin.pluginOptions, function (option) {
+                if (option.type === 'export-plugin') {
                   _.each($scope.exportPlugins, function (plugin) {
                     if (plugin.slug === option.value.slug) {
                       plugin.pluginOptions = option.value.pluginOptions;
                       option.value = plugin;
                     }
                   });
-                });
-              }
+                }
+              });
             });
           }
         });

lemur/static/app/angular/notifications/notification/notification.js

@@ -42,8 +42,8 @@ angular.module('lemur')
       PluginService.getByType('notification').then(function (plugins) {
         $scope.plugins = plugins;
         _.each($scope.plugins, function (plugin) {
-          if (plugin.slug === $scope.notification.pluginName) {
-            plugin.pluginOptions = $scope.notification.notificationOptions;
+          if (plugin.slug === $scope.notification.plugin.slug) {
+            plugin.pluginOptions = $scope.notification.plugin.pluginOptions;
             $scope.notification.plugin = plugin;
           }
         });
@@ -51,16 +51,6 @@ angular.module('lemur')
       NotificationService.getCertificates(notification);
     });
 
-    PluginService.getByType('notification').then(function (plugins) {
-      $scope.plugins = plugins;
-      _.each($scope.plugins, function (plugin) {
-        if (plugin.slug === $scope.notification.pluginName) {
-          plugin.pluginOptions = $scope.notification.notificationOptions;
-          $scope.notification.plugin = plugin;
-        }
-      });
-    });
-
     $scope.save = function (notification) {
       NotificationService.update(notification).then(
         function () {

lemur/static/app/angular/notifications/services.js

@@ -27,7 +27,7 @@ angular.module('lemur')
     };
 
     NotificationService.getCertificates = function (notification) {
-      notification.getList('certificates').then(function (certificates) {
+      notification.getList('certificates', {showExpired: 0}).then(function (certificates) {
         notification.certificates = certificates;
       });
     };
@@ -40,7 +40,7 @@ angular.module('lemur')
 
 
     NotificationService.loadMoreCertificates = function (notification, page) {
-      notification.getList('certificates', {page: page}).then(function (certificates) {
+      notification.getList('certificates', {page: page, showExpired: 0}).then(function (certificates) {
         _.each(certificates, function (certificate) {
           notification.roles.push(certificate);
         });

lemur/static/app/angular/pending_certificates/services.js

@@ -152,12 +152,12 @@ angular.module('lemur')
           // Minimum end date will be same as selected start date
           this.authority.authorityCertificate.minValidityEnd = value;
 
-          if(!this.authority.authorityCertificate || !this.authority.authorityCertificate.maxIssuanceDays) {
+          if(!this.authority.maxIssuanceDays) {
             this.authority.authorityCertificate.maxValidityEnd = this.authority.authorityCertificate.notAfter;
           } else {
             // Move max end date by maxIssuanceDays
             let endDate = new Date(value);
-            endDate.setDate(endDate.getDate() + this.authority.authorityCertificate.maxIssuanceDays);
+            endDate.setDate(endDate.getDate() + this.authority.maxIssuanceDays);
             this.authority.authorityCertificate.maxValidityEnd = endDate;
           }
         }

lemur/static/app/angular/sources/source/source.js

@@ -41,22 +41,14 @@ angular.module('lemur')
       PluginService.getByType('source').then(function (plugins) {
         $scope.plugins = plugins;
         _.each($scope.plugins, function (plugin) {
-          if (plugin.slug === $scope.source.pluginName) {
+          if (plugin.slug === $scope.source.plugin.slug) {
+            plugin.pluginOptions = $scope.source.plugin.pluginOptions;
             $scope.source.plugin = plugin;
           }
         });
       });
     });
 
-    PluginService.getByType('source').then(function (plugins) {
-      $scope.plugins = plugins;
-      _.each($scope.plugins, function (plugin) {
-        if (plugin.slug === $scope.source.pluginName) {
-          $scope.source.plugin = plugin;
-        }
-      });
-    });
-
     $scope.save = function (source) {
       SourceService.update(source).then(
         function () {

lemur/tests/conf.py

@@ -36,17 +36,17 @@ LEMUR_ENCRYPTION_KEYS = base64.urlsafe_b64encode(get_random_secret(length=32).en
 
 
 # List of domain regular expressions that non-admin users can issue
-LEMUR_WHITELISTED_DOMAINS = [
-    "^[a-zA-Z0-9-]+\.example\.com$",
-    "^[a-zA-Z0-9-]+\.example\.org$",
-    "^example\d+\.long\.com$",
+LEMUR_ALLOWED_DOMAINS = [
+    r"^[a-zA-Z0-9-]+\.example\.com$",
+    r"^[a-zA-Z0-9-]+\.example\.org$",
+    r"^example\d+\.long\.com$",
 ]
 
 # Mail Server
 
 # Lemur currently only supports SES for sending email, this address
 # needs to be verified
-LEMUR_EMAIL = ""
+LEMUR_EMAIL = "lemur@example.com"
 LEMUR_SECURITY_TEAM_EMAIL = ["security@example.com"]
 
 LEMUR_HOSTNAME = "lemur.example.com"
@@ -99,7 +99,6 @@ DIGICERT_CIS_URL = "mock://www.digicert.com"
 DIGICERT_CIS_PROFILE_NAMES = {"sha2-rsa-ecc-root": "ssl_plus"}
 DIGICERT_CIS_API_KEY = "api-key"
 DIGICERT_CIS_ROOTS = {"root": "ROOT"}
-DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"}
 
 VERISIGN_URL = "http://example.com"
 VERISIGN_PEM_PATH = "~/"

lemur/tests/test_certificates.py

@@ -9,7 +9,6 @@ from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from marshmallow import ValidationError
 from freezegun import freeze_time
-# from mock import patch
 from unittest.mock import patch
 
 from lemur.certificates.service import create_csr
@@ -171,16 +170,52 @@ def test_certificate_output_schema(session, certificate, issuer_plugin):
     ) as wrapper:
         data, errors = CertificateOutputSchema().dump(certificate)
         assert data["issuer"] == "LemurTrustUnittestsClass1CA2018"
+        assert data["distinguishedName"] == "L=Earth,ST=N/A,C=EE,OU=Karate Lessons,O=Daniel San & co,CN=san.example.org"
+        # Authority does not have 'cab_compliant', thus subject details should not be returned
+        assert "organization" not in data
 
     assert wrapper.call_count == 1
 
 
+def test_certificate_output_schema_subject_details(session, certificate, issuer_plugin):
+    from lemur.certificates.schemas import CertificateOutputSchema
+    from lemur.authorities.service import update_options
+
+    # Mark authority as non-cab-compliant
+    update_options(certificate.authority.id, '[{"name": "cab_compliant","value":false}]')
+
+    data, errors = CertificateOutputSchema().dump(certificate)
+    assert not errors
+    assert data["issuer"] == "LemurTrustUnittestsClass1CA2018"
+    assert data["distinguishedName"] == "L=Earth,ST=N/A,C=EE,OU=Karate Lessons,O=Daniel San & co,CN=san.example.org"
+
+    # Original subject details should be returned because of cab_compliant option update above
+    assert data["country"] == "EE"
+    assert data["state"] == "N/A"
+    assert data["location"] == "Earth"
+    assert data["organization"] == "Daniel San & co"
+    assert data["organizationalUnit"] == "Karate Lessons"
+
+    # Mark authority as cab-compliant
+    update_options(certificate.authority.id, '[{"name": "cab_compliant","value":true}]')
+    data, errors = CertificateOutputSchema().dump(certificate)
+    assert not errors
+    assert "country" not in data
+    assert "state" not in data
+    assert "location" not in data
+    assert "organization" not in data
+    assert "organizationalUnit" not in data
+
+
 def test_certificate_edit_schema(session):
     from lemur.certificates.schemas import CertificateEditInputSchema
 
     input_data = {"owner": "bob@example.com"}
     data, errors = CertificateEditInputSchema().load(input_data)
+
+    assert not errors
     assert len(data["notifications"]) == 3
+    assert data["roles"][0].name == input_data["owner"]
 
 
 def test_authority_key_identifier_schema():
@@ -397,7 +432,7 @@ def test_certificate_cn_admin(client, authority, logged_in_admin):
     from lemur.certificates.schemas import CertificateInputSchema
 
     input_data = {
-        "commonName": "*.admin-overrides-whitelist.com",
+        "commonName": "*.admin-overrides-allowlist.com",
         "owner": "jim@example.com",
         "authority": {"id": authority.id},
         "description": "testtestest",
@@ -458,7 +493,7 @@ def test_certificate_incative_authority(client, authority, session, logged_in_us
 
 
 def test_certificate_disallowed_names(client, authority, session, logged_in_user):
-    """The CN and SAN are disallowed by LEMUR_WHITELISTED_DOMAINS."""
+    """The CN and SAN are disallowed by LEMUR_ALLOWED_DOMAINS."""
     from lemur.certificates.schemas import CertificateInputSchema
 
     input_data = {
@@ -481,10 +516,10 @@ def test_certificate_disallowed_names(client, authority, session, logged_in_user
 
     data, errors = CertificateInputSchema().load(input_data)
     assert errors["common_name"][0].startswith(
-        "Domain *.example.com does not match whitelisted domain patterns"
+        "Domain *.example.com does not match allowed domain patterns"
     )
     assert errors["extensions"]["sub_alt_names"]["names"][0].startswith(
-        "Domain evilhacker.org does not match whitelisted domain patterns"
+        "Domain evilhacker.org does not match allowed domain patterns"
     )
 
 
@@ -671,7 +706,7 @@ def test_csr_empty_san(client):
 
 
 def test_csr_disallowed_cn(client, logged_in_user):
-    """Domain name CN is disallowed via LEMUR_WHITELISTED_DOMAINS."""
+    """Domain name CN is disallowed via LEMUR_ALLOWED_DOMAINS."""
     from lemur.common import validators
 
     request, pkey = create_csr(
@@ -680,12 +715,12 @@ def test_csr_disallowed_cn(client, logged_in_user):
     with pytest.raises(ValidationError) as err:
         validators.csr(request)
     assert str(err.value).startswith(
-        "Domain evilhacker.org does not match whitelisted domain patterns"
+        "Domain evilhacker.org does not match allowed domain patterns"
     )
 
 
 def test_csr_disallowed_san(client, logged_in_user):
-    """SAN name is disallowed by LEMUR_WHITELISTED_DOMAINS."""
+    """SAN name is disallowed by LEMUR_ALLOWED_DOMAINS."""
     from lemur.common import validators
 
     request, pkey = create_csr(
@@ -701,7 +736,7 @@ def test_csr_disallowed_san(client, logged_in_user):
     with pytest.raises(ValidationError) as err:
         validators.csr(request)
     assert str(err.value).startswith(
-        "Domain evilhacker.org does not match whitelisted domain patterns"
+        "Domain evilhacker.org does not match allowed domain patterns"
     )
 
 
@@ -756,12 +791,23 @@ def test_reissue_certificate(
     issuer_plugin, crypto_authority, certificate, logged_in_user
 ):
     from lemur.certificates.service import reissue_certificate
+    from lemur.authorities.service import update_options
+    from lemur.tests.conf import LEMUR_DEFAULT_ORGANIZATION
 
     # test-authority would return a mismatching private key, so use 'cryptography-issuer' plugin instead.
     certificate.authority = crypto_authority
     new_cert = reissue_certificate(certificate)
     assert new_cert
-    assert (new_cert.key_type == "RSA2048")
+    assert new_cert.key_type == "RSA2048"
+    assert new_cert.organization != certificate.organization
+    # Check for default value since authority does not have cab_compliant option set
+    assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION
+    assert new_cert.description.startswith(f"Reissued by Lemur for cert ID {certificate.id}")
+
+    # update cab_compliant option to false for crypto_authority to maintain subject details
+    update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]')
+    new_cert = reissue_certificate(certificate)
+    assert new_cert.organization == certificate.organization
 
 
 def test_create_csr():
@@ -918,24 +964,38 @@ def test_certificate_get_body(client):
         "CN=LemurTrust Unittests Class 1 CA 2018"
     )
 
+    # No authority details are provided in this test, no information about being cab_compliant is available.
+    # Thus original subject details should be returned.
+    assert response_body["country"] == "EE"
+    assert response_body["state"] == "N/A"
+    assert response_body["location"] == "Earth"
+    assert response_body["organization"] == "LemurTrust Enterprises Ltd"
+    assert response_body["organizationalUnit"] == "Unittesting Operations Center"
+
 
 @pytest.mark.parametrize(
     "token,status",
     [
-        (VALID_USER_HEADER_TOKEN, 405),
-        (VALID_ADMIN_HEADER_TOKEN, 405),
-        (VALID_ADMIN_API_TOKEN, 405),
-        ("", 405),
+        (VALID_USER_HEADER_TOKEN, 403),
+        (VALID_ADMIN_HEADER_TOKEN, 200),
+        (VALID_ADMIN_API_TOKEN, 200),
+        ("", 401),
     ],
 )
-def test_certificate_post(client, token, status):
-    assert (
-        client.post(
-            api.url_for(Certificates, certificate_id=1), data={}, headers=token
-        ).status_code
-        == status
+def test_certificate_post_update_notify(client, certificate, token, status):
+    # negate the current notify flag and pass it to update POST call to flip the notify
+    toggled_notify = not certificate.notify
+
+    response = client.post(
+        api.url_for(Certificates, certificate_id=certificate.id),
+        data=json.dumps({"notify": toggled_notify}),
+        headers=token
     )
 
+    assert response.status_code == status
+    if status == 200:
+        assert response.json.get("notify") == toggled_notify
+
 
 @pytest.mark.parametrize(
     "token,status",
@@ -964,6 +1024,9 @@ def test_certificate_put_with_data(client, certificate, issuer_plugin):
         headers=VALID_ADMIN_HEADER_TOKEN,
     )
     assert resp.status_code == 200
+    assert len(certificate.notifications) == 3
+    assert certificate.roles[0].name == "bob@example.com"
+    assert certificate.notify
 
 
 @pytest.mark.parametrize(

lemur/tests/test_dns_providers.py

@@ -13,7 +13,7 @@ class TestDNSProvider(unittest.TestCase):
         self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com'))
         self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com'))
         self.assertFalse(dnsutil.is_valid_domain('e/xample.com'))
-        self.assertFalse(dnsutil.is_valid_domain('exam\ple.com'))
+        self.assertFalse(dnsutil.is_valid_domain('exam\\ple.com'))
         self.assertFalse(dnsutil.is_valid_domain('<example.com'))
         self.assertFalse(dnsutil.is_valid_domain('*.example.com'))
         self.assertFalse(dnsutil.is_valid_domain('-example.io'))

lemur/tests/test_messaging.py

@@ -1,11 +1,18 @@
+from datetime import timedelta
+
+import arrow
+import boto3
 import pytest
 from freezegun import freeze_time
-
-from datetime import timedelta
-import arrow
 from moto import mock_ses
 
 
+@mock_ses
+def verify_sender_email():
+    ses_client = boto3.client("ses", region_name="us-east-1")
+    ses_client.verify_email_identity(EmailAddress="lemur@example.com")
+
+
 def test_needs_notification(app, certificate, notification):
     from lemur.notifications.messaging import needs_notification
 
@@ -78,6 +85,7 @@ def test_get_eligible_certificates(app, certificate, notification):
 @mock_ses
 def test_send_expiration_notification(certificate, notification, notification_plugin):
     from lemur.notifications.messaging import send_expiration_notifications
+    verify_sender_email()
 
     certificate.notifications.append(notification)
     certificate.notifications[0].options = [
@@ -87,7 +95,9 @@ def test_send_expiration_notification(certificate, notification, notification_pl
 
     delta = certificate.not_after - timedelta(days=10)
     with freeze_time(delta.datetime):
-        assert send_expiration_notifications([]) == (2, 0)
+        # this will only send owner and security emails (no additional recipients),
+        # but it executes 3 successful send attempts
+        assert send_expiration_notifications([]) == (3, 0)
 
 
 @mock_ses
@@ -104,5 +114,14 @@ def test_send_expiration_notification_with_no_notifications(
 @mock_ses
 def test_send_rotation_notification(notification_plugin, certificate):
     from lemur.notifications.messaging import send_rotation_notification
+    verify_sender_email()
 
-    send_rotation_notification(certificate, notification_plugin=notification_plugin)
+    assert send_rotation_notification(certificate)
+
+
+@mock_ses
+def test_send_pending_failure_notification(notification_plugin, async_issuer_plugin, pending_certificate):
+    from lemur.notifications.messaging import send_pending_failure_notification
+    verify_sender_email()
+
+    assert send_pending_failure_notification(pending_certificate)

lemur/tests/test_utils.py

@@ -10,6 +10,7 @@ from lemur.tests.vectors import (
     ECDSA_SECP384r1_CERT,
     ECDSA_SECP384r1_CERT_STR,
     DSA_CERT,
+    CERT_CHAIN_PKCS7_PEM
 )
 
 
@@ -114,3 +115,16 @@ def test_get_key_type_from_certificate():
     from lemur.common.utils import get_key_type_from_certificate
     assert (get_key_type_from_certificate(SAN_CERT_STR) == "RSA2048")
     assert (get_key_type_from_certificate(ECDSA_SECP384r1_CERT_STR) == "ECCSECP384R1")
+
+
+def test_convert_pkcs7_bytes_to_pem():
+    from lemur.common.utils import convert_pkcs7_bytes_to_pem
+    from lemur.common.utils import parse_certificate
+    cert_chain = convert_pkcs7_bytes_to_pem(CERT_CHAIN_PKCS7_PEM)
+    assert(len(cert_chain) == 3)
+
+    leaf = cert_chain[1]
+    root = cert_chain[2]
+
+    assert(parse_certificate("\n".join(str(root).splitlines())) == ROOTCA_CERT)
+    assert (parse_certificate("\n".join(str(leaf).splitlines())) == INTERMEDIATE_CERT)

lemur/tests/vectors.py

@@ -512,3 +512,78 @@ BglghkgBZQMEAwIDMAAwLQIVANubSNMSLt8plN9ZV3cp4pe3lMYCAhQPLLE7rTgm
 -----END CERTIFICATE-----
 """
 DSA_CERT = parse_certificate(DSA_CERT_STR)
+
+
+CERT_CHAIN_PKCS7_STR = """
+-----BEGIN PKCS7-----
+MIIMfwYJKoZIhvcNAQcCoIIMcDCCDGwCAQExADALBgkqhkiG9w0BBwGgggxSMIIE
+FjCCAv6gAwIBAgIQbIbX/Ap0Roqzf5HeN5akmzANBgkqhkiG9w0BAQsFADCBpDEq
+MCgGA1UEAwwhTGVtdXJUcnVzdCBVbml0dGVzdHMgUm9vdCBDQSAyMDE4MSMwIQYD
+VQQKDBpMZW11clRydXN0IEVudGVycHJpc2VzIEx0ZDEmMCQGA1UECwwdVW5pdHRl
+c3RpbmcgT3BlcmF0aW9ucyBDZW50ZXIxCzAJBgNVBAYTAkVFMQwwCgYDVQQIDANO
+L0ExDjAMBgNVBAcMBUVhcnRoMB4XDTE3MTIzMTIyMDAwMFoXDTQ3MTIzMTIyMDAw
+MFowgaQxKjAoBgNVBAMMIUxlbXVyVHJ1c3QgVW5pdHRlc3RzIFJvb3QgQ0EgMjAx
+ODEjMCEGA1UECgwaTGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsM
+HVVuaXR0ZXN0aW5nIE9wZXJhdGlvbnMgQ2VudGVyMQswCQYDVQQGEwJFRTEMMAoG
+A1UECAwDTi9BMQ4wDAYDVQQHDAVFYXJ0aDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAL8laXtLXyM64t5dz2B9q+4VvOsChefBi2PlGudqxDuRN3l0Kmcf
+un6x2Gng24pTlGdtmiTEWA0a2F8HRLv4YBWhuYleVeBPtf1fF1/SuYgkJOWT7S5q
+k/od/tUOLHS0Y067st3FydnFQTKpAuYveEkxleFrMS8hX8cuEgbER+8ybiXKn4Gs
+yM/om6lsTyBoaLp5yTAoQb4jAWDbiz1xcjPSkvH2lm7rLGtKoylCYwxRsMh2nZcR
+r1OXVhYHXwpYHVB/jVAjy7PAWQ316hi6mpPYbBV+yfn2GUfGuytqyoXLEsrM3iEE
+AkU0mJjQmYsCDM3r7ONHTM+UFEk47HCZJccCAwEAAaNCMEAwDwYDVR0TAQH/BAUw
+AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFFL12SFeOTTDdGKsHKozeByG
+HY6nMA0GCSqGSIb3DQEBCwUAA4IBAQAJfe0/uAHobkxth38dqrSFmTo+D5/TMlRt
+3hdgjlah6sD2+/DObCyut/XhQWCgTNWyRi4xTKgLh5KSoeJ9EMkADGEgDkU2vjBg
+5FmGZsxg6bqjxehK+2HvASJoTH8r41xmTioav7a2i3wNhaNSntw2QRTQBQEDOIzH
+RpPDQ2quErjA8nSifE2xmAAr3g+FuookTTJuv37s2cS59zRYsg+WC3+TtPpRssvo
+bJ6Xe2D4cCVjUmsqtFEztMgdqgmlcWyGdUKeXdi7CMoeTb4uO+9qRQq46wYWn7K1
+z+W0Kp5yhnnPAoOioAP4vjASDx3z3RnLaZvMmcO7YdCIwhE5oGV0MIIEGjCCAwKg
+AwIBAgIRAJ96dbOdrkw/lSTGiwbaagwwDQYJKoZIhvcNAQELBQAwgaQxKjAoBgNV
+BAMMIUxlbXVyVHJ1c3QgVW5pdHRlc3RzIFJvb3QgQ0EgMjAxODEjMCEGA1UECgwa
+TGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsMHVVuaXR0ZXN0aW5n
+IE9wZXJhdGlvbnMgQ2VudGVyMQswCQYDVQQGEwJFRTEMMAoGA1UECAwDTi9BMQ4w
+DAYDVQQHDAVFYXJ0aDAeFw0xNzEyMzEyMjAwMDBaFw00NzEyMzEyMjAwMDBaMIGn
+MS0wKwYDVQQDDCRMZW11clRydXN0IFVuaXR0ZXN0cyBDbGFzcyAxIENBIDIwMTgx
+IzAhBgNVBAoMGkxlbXVyVHJ1c3QgRW50ZXJwcmlzZXMgTHRkMSYwJAYDVQQLDB1V
+bml0dGVzdGluZyBPcGVyYXRpb25zIENlbnRlcjELMAkGA1UEBhMCRUUxDDAKBgNV
+BAgMA04vQTEOMAwGA1UEBwwFRWFydGgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDR+qNdfNsLhGvgw3IgCQNakL2B9dpQtkVnvAXhdRZqJETm/tHLkGvO
+NWTXAwGdoiKv6+0j3I5InUsW+wzUPewcfj+PLNu4mFMq8jH/gPhTElKiAztPRdm8
+QKchvrqiaU6uEbia8ClM6uPpIi8StxE1aJRYL03p0WeMJjJPrsl6eSSdpR4qL69G
+Td1n5je9OuWAcn5utXXnt/jO4vNeFRjlGp/0n3JmTDd9w4vtAyY9UrdGgo37eBmi
+6mXt5J9i//NenhaiOVU81RqxZM2Jt1kkg2WSjcqcIQfBEWp9StG46VmHLaL+9/v2
+XAV3tL1VilJGj6PoFMb4gY5MXthfGSiXAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB
+Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBQstpQr0iMBVfv0lODIsMgT9+9o
+ezANBgkqhkiG9w0BAQsFAAOCAQEASYQbv1Qwb5zES6Gb5LEhrAcH81ZB2uIpKd3K
+i6AS4fLJVymMGkUs0RZjt39Ep4qX1zf0hn82Yh9YwRalrkgu+tzKrp0JgegNe6+g
+yFRrJC0SIGA4zc3M02m/n4tdaouU2lp6jhmWruL3g25ZkgbQ8LO2zjpSMtblR2eu
+vR2+bI7TepklyG71qx5y6/N8x5PT+hnTlleiZeE/ji9D96MZlpWB4kBihekWmxup
+tED22z/tpQtac+hPBNgt8z1uFVEYN2rKEcCE7V6Qk7icS+M4Vb7M3D8kLyWDubs9
+Yy3l0EWjOXQXxEhTaKEm4gSuY/j+Y35bBVkA2Fcyuq7msiTgrzCCBBYwggL+oAMC
+AQICEGyG1/wKdEaKs3+R3jeWpJswDQYJKoZIhvcNAQELBQAwgaQxKjAoBgNVBAMM
+IUxlbXVyVHJ1c3QgVW5pdHRlc3RzIFJvb3QgQ0EgMjAxODEjMCEGA1UECgwaTGVt
+dXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsMHVVuaXR0ZXN0aW5nIE9w
+ZXJhdGlvbnMgQ2VudGVyMQswCQYDVQQGEwJFRTEMMAoGA1UECAwDTi9BMQ4wDAYD
+VQQHDAVFYXJ0aDAeFw0xNzEyMzEyMjAwMDBaFw00NzEyMzEyMjAwMDBaMIGkMSow
+KAYDVQQDDCFMZW11clRydXN0IFVuaXR0ZXN0cyBSb290IENBIDIwMTgxIzAhBgNV
+BAoMGkxlbXVyVHJ1c3QgRW50ZXJwcmlzZXMgTHRkMSYwJAYDVQQLDB1Vbml0dGVz
+dGluZyBPcGVyYXRpb25zIENlbnRlcjELMAkGA1UEBhMCRUUxDDAKBgNVBAgMA04v
+QTEOMAwGA1UEBwwFRWFydGgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQC/JWl7S18jOuLeXc9gfavuFbzrAoXnwYtj5RrnasQ7kTd5dCpnH7p+sdhp4NuK
+U5RnbZokxFgNGthfB0S7+GAVobmJXlXgT7X9Xxdf0rmIJCTlk+0uapP6Hf7VDix0
+tGNOu7LdxcnZxUEyqQLmL3hJMZXhazEvIV/HLhIGxEfvMm4lyp+BrMjP6JupbE8g
+aGi6eckwKEG+IwFg24s9cXIz0pLx9pZu6yxrSqMpQmMMUbDIdp2XEa9Tl1YWB18K
+WB1Qf41QI8uzwFkN9eoYupqT2GwVfsn59hlHxrsrasqFyxLKzN4hBAJFNJiY0JmL
+AgzN6+zjR0zPlBRJOOxwmSXHAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYD
+VR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRS9dkhXjk0w3RirByqM3gchh2OpzANBgkq
+hkiG9w0BAQsFAAOCAQEACX3tP7gB6G5MbYd/Haq0hZk6Pg+f0zJUbd4XYI5WoerA
+9vvwzmwsrrf14UFgoEzVskYuMUyoC4eSkqHifRDJAAxhIA5FNr4wYORZhmbMYOm6
+o8XoSvth7wEiaEx/K+NcZk4qGr+2tot8DYWjUp7cNkEU0AUBAziMx0aTw0NqrhK4
+wPJ0onxNsZgAK94PhbqKJE0ybr9+7NnEufc0WLIPlgt/k7T6UbLL6Gyel3tg+HAl
+Y1JrKrRRM7TIHaoJpXFshnVCnl3YuwjKHk2+LjvvakUKuOsGFp+ytc/ltCqecoZ5
+zwKDoqAD+L4wEg8d890Zy2mbzJnDu2HQiMIROaBldKEAMQA=
+-----END PKCS7-----
+"""
+
+CERT_CHAIN_PKCS7_PEM = CERT_CHAIN_PKCS7_STR.encode('utf-8')

lemur/utils.py

@@ -81,7 +81,7 @@ class Vault(types.TypeDecorator):
     """
 
     # required by SQLAlchemy. defines the underlying column type
-    impl = types.Binary
+    impl = types.LargeBinary
 
     def process_bind_param(self, value, dialect):
         """