Merge branch 'master' into certificates-for-notification-fix

.travis.yml

@@ -47,4 +47,4 @@ after_success:
 
 notifications:
   email:
-    ccastrapel@netflix.com
+    lemur@netflix.com

docs/administration.rst

@@ -28,6 +28,13 @@ Basic Configuration
 
         LOG_FILE = "/logs/lemur/lemur-test.log"
 
+.. data:: LOG_UPGRADE_FILE
+    :noindex:
+
+    ::
+
+        LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log"
+
 .. data:: DEBUG
     :noindex:
 
@@ -285,6 +292,25 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
         you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
 
 
+.. data:: LEMUR_SES_SOURCE_ARN
+    :noindex:
+
+    Specifies an ARN to use as the SourceArn when sending emails via SES.
+
+    .. note::
+        This parameter is only required if you're using a sending authorization with SES.
+        See: `Using sending authorization with Amazon SES <https://docs.aws.amazon.com/ses/latest/DeveloperGuide/sending-authorization.html>`_
+
+
+.. data:: LEMUR_SES_REGION
+    :noindex:
+
+    Specifies a region for sending emails via SES.
+
+    .. note::
+        This parameter defaults to us-east-1 and is only required if you wish to use a different region.
+
+
 .. data:: LEMUR_EMAIL
     :noindex:
 

lemur/certificates/service.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+import re
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
@@ -779,6 +780,19 @@ def reissue_certificate(certificate, replace=None, user=None):
     if replace:
         primitives["replaces"] = [certificate]
 
+    # Modify description to include the certificate ID being reissued and mention that this is created by Lemur
+    # as part of reissue
+    reissue_message_prefix = "Reissued by Lemur for cert ID "
+    reissue_message = re.compile(f"{reissue_message_prefix}([0-9]+)")
+    if primitives["description"]:
+        match = reissue_message.search(primitives["description"])
+        if match:
+            primitives["description"] = primitives["description"].replace(match.group(1), str(certificate.id))
+        else:
+            primitives["description"] = f"{reissue_message_prefix}{certificate.id}, {primitives['description']}"
+    else:
+        primitives["description"] = f"{reissue_message_prefix}{certificate.id}"
+
     new_cert = create(**primitives)
 
     return new_cert

lemur/manage.py

@@ -120,6 +120,7 @@ METRIC_PROVIDERS = []
 
 LOG_LEVEL = "DEBUG"
 LOG_FILE = "lemur.log"
+LOG_UPGRADE_FILE = "db_upgrade.log"
 
 
 # Database

lemur/migrations/versions/1db4f82bc780_.py

@@ -10,11 +10,21 @@ Create Date: 2018-08-03 12:56:44.565230
 revision = "1db4f82bc780"
 down_revision = "3adfdd6598df"
 
-import logging
-
 from alembic import op
 
-log = logging.getLogger(__name__)
+from flask import current_app
+from logging import Formatter, FileHandler, getLogger
+
+log = getLogger(__name__)
+handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
+handler.setFormatter(
+    Formatter(
+        "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+    )
+)
+handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.addHandler(handler)
 
 
 def upgrade():

lemur/migrations/versions/c301c59688d2_.py

@@ -7,8 +7,9 @@ the rest of the keys, the certificate body is parsed to determine
 the exact key_type information.
 
 Each individual DB change is explicitly committed, and the respective
-log is added to a file named db_upgrade.log in the current working
+log is added to a file configured in LOG_UPGRADE_FILE or, by default,
-directory. Any error encountered while parsing a certificate will
+to a file named db_upgrade.log in the current working directory.
+Any error encountered while parsing a certificate will
 also be logged along with the certificate ID. If faced with any issue
 while running this upgrade, there is no harm in re-running the upgrade.
 Each run processes only rows for which key_type information is not yet
@@ -31,15 +32,28 @@ down_revision = '434c29e40511'
 
 from alembic import op
 from sqlalchemy.sql import text
-from lemur.common import utils
 import time
 import datetime
+from flask import current_app
 
-log_file = open('db_upgrade.log', 'a')
+from logging import Formatter, FileHandler, getLogger
+
+from lemur.common import utils
+
+log = getLogger(__name__)
+handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
+handler.setFormatter(
+    Formatter(
+        "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+    )
+)
+handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.addHandler(handler)
 
 
 def upgrade():
-    log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
+    log.info("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
     start_time = time.time()
 
     # Update RSA keys using the key length information
@@ -50,8 +64,7 @@ def upgrade():
     # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs.
     update_key_type()
 
-    log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- Total %s seconds ---\n" % (time.time() - start_time))
-    log_file.close()
 
 
 def downgrade():
@@ -69,18 +82,18 @@ def downgrade():
 
 
 def update_key_type_rsa(bits):
-    log_file.write("Processing certificate with key type RSA %s\n" % bits)
+    log.info("Processing certificate with key type RSA %s\n" % bits)
 
     stmt = text(
         f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null"
     )
-    log_file.write("Query: %s\n" % stmt)
+    log.info("Query: %s\n" % stmt)
 
     start_time = time.time()
     op.execute(stmt)
     commit()
 
-    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- %s seconds ---\n" % (time.time() - start_time))
 
 
 def update_key_type():
@@ -95,9 +108,9 @@ def update_key_type():
         try:
             cert_key_type = utils.get_key_type_from_certificate(body)
         except ValueError as e:
-            log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
+            log.error("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
         else:
-            log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
+            log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
             stmt = text(
                 "update certificates set key_type=:key_type where id=:id"
             )
@@ -106,7 +119,7 @@ def update_key_type():
 
             commit()
 
-    log_file.write("--- %s seconds ---\n" % (time.time() - start_time))
+    log.info("--- %s seconds ---\n" % (time.time() - start_time))
 
 
 def commit():

lemur/plugins/lemur_aws/sns.py

@@ -15,16 +15,18 @@ from flask import current_app
 def publish(topic_arn, certificates, notification_type, **kwargs):
     sns_client = boto3.client("sns", **kwargs)
     message_ids = {}
+    subject = "Lemur: {0} Notification".format(notification_type.capitalize())
     for certificate in certificates:
-        message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type)
+        message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject)
 
     return message_ids
 
 
-def publish_single(sns_client, topic_arn, certificate, notification_type):
+def publish_single(sns_client, topic_arn, certificate, notification_type, subject):
     response = sns_client.publish(
         TopicArn=topic_arn,
         Message=format_message(certificate, notification_type),
+        Subject=subject,
     )
 
     response_code = response["ResponseMetadata"]["HTTPStatusCode"]
@@ -48,8 +50,9 @@ def format_message(certificate, notification_type):
     json_message = {
         "notification_type": notification_type,
         "certificate_name": certificate["name"],
-        "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"),  # 2047-12-T22:00:00
+        "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),  # 2047-12-31T22:00:00
         "endpoints_detected": len(certificate["endpoints"]),
+        "owner": certificate["owner"],
         "details": create_certificate_url(certificate["name"])
     }
     return json.dumps(json_message)

lemur/plugins/lemur_aws/tests/test_sns.py

@@ -20,8 +20,9 @@ def test_format(certificate, endpoint):
         expected_message = {
             "notification_type": "expiration",
             "certificate_name": certificate["name"],
-            "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"),
+            "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
             "endpoints_detected": 0,
+            "owner": certificate["owner"],
             "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
         }
         assert expected_message == json.loads(format_message(certificate, "expiration"))
@@ -57,7 +58,9 @@ def test_publish(certificate, endpoint):
         expected_message_id = message_ids[certificate["name"]]
         actual_message = next(
             (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None)
-        assert json.loads(actual_message["Body"])["Message"] == format_message(certificate, "expiration")
+        actual_json = json.loads(actual_message["Body"])
+        assert actual_json["Message"] == format_message(certificate, "expiration")
+        assert actual_json["Subject"] == "Lemur: Expiration Notification"
 
 
 def get_options():

lemur/plugins/lemur_digicert/plugin.py

@@ -234,7 +234,7 @@ def handle_cis_response(response):
         return response.json()
 
 
-@retry(stop_max_attempt_number=10, wait_fixed=10000)
+@retry(stop_max_attempt_number=10, wait_fixed=1000)
 def get_certificate_id(session, base_url, order_id):
     """Retrieve certificate order id from Digicert API."""
     order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id)
@@ -245,7 +245,7 @@ def get_certificate_id(session, base_url, order_id):
     return response_data["certificate"]["id"]
 
 
-@retry(stop_max_attempt_number=10, wait_fixed=10000)
+@retry(stop_max_attempt_number=10, wait_fixed=1000)
 def get_cis_certificate(session, base_url, order_id):
     """Retrieve certificate order id from Digicert API, including the chain"""
     certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id)

lemur/plugins/lemur_email/plugin.py

@@ -38,7 +38,7 @@ def render_html(template_name, options, certificates):
 
 def send_via_smtp(subject, body, targets):
     """
-    Attempts to deliver email notification via SES service.
+    Attempts to deliver email notification via SMTP.
 
     :param subject:
     :param body:
@@ -55,21 +55,26 @@ def send_via_smtp(subject, body, targets):
 
 def send_via_ses(subject, body, targets):
     """
-    Attempts to deliver email notification via SMTP.
+    Attempts to deliver email notification via SES service.
     :param subject:
     :param body:
     :param targets:
     :return:
     """
-    client = boto3.client("ses", region_name="us-east-1")
+    ses_region = current_app.config.get("LEMUR_SES_REGION", "us-east-1")
-    client.send_email(
+    client = boto3.client("ses", region_name=ses_region)
-        Source=current_app.config.get("LEMUR_EMAIL"),
+    source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN")
-        Destination={"ToAddresses": targets},
+    args = {
-        Message={
+        "Source": current_app.config.get("LEMUR_EMAIL"),
+        "Destination": {"ToAddresses": targets},
+        "Message": {
             "Subject": {"Data": subject, "Charset": "UTF-8"},
             "Body": {"Html": {"Data": body, "Charset": "UTF-8"}},
         },
-    )
+    }
+    if source_arn:
+        args["SourceArn"] = source_arn
+    client.send_email(**args)
 
 
 class EmailNotificationPlugin(ExpirationNotificationPlugin):

lemur/plugins/lemur_entrust/plugin.py

@@ -1,9 +1,9 @@
-
 import arrow
 import requests
 import json
 import sys
 from flask import current_app
+from retrying import retry
 
 from lemur.plugins import lemur_entrust as entrust
 from lemur.plugins.bases import IssuerPlugin, SourcePlugin
@@ -78,7 +78,6 @@ def process_options(options):
         "eku": "SERVER_AND_CLIENT_AUTH",
         "certType": product_type,
         "certExpiryDate": validity_end,
-        # "keyType": "RSA", Entrust complaining about this parameter
         "tracking": tracking_data
     }
     return data
@@ -87,7 +86,7 @@ def process_options(options):
 def handle_response(my_response):
     """
     Helper function for parsing responses from the Entrust API.
-    :param content:
+    :param my_response:
     :return: :raise Exception:
     """
     msg = {
@@ -100,27 +99,47 @@ def handle_response(my_response):
     }
 
     try:
-        d = json.loads(my_response.content)
+        data = json.loads(my_response.content)
     except ValueError:
         # catch an empty jason object here
-        d = {'response': 'No detailed message'}
+        data = {'response': 'No detailed message'}
-    s = my_response.status_code
+    status_code = my_response.status_code
-    if s > 399:
+    if status_code > 399:
-        raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}")
+        raise Exception(f"ENTRUST error: {msg.get(status_code, status_code)}\n{data['errors']}")
 
     log_data = {
         "function": f"{__name__}.{sys._getframe().f_code.co_name}",
         "message": "Response",
-        "status": s,
+        "status": status_code,
-        "response": d
+        "response": data
     }
     current_app.logger.info(log_data)
-    if d == {'response': 'No detailed message'}:
+    if data == {'response': 'No detailed message'}:
         # status if no data
-        return s
+        return status_code
     else:
         #  return data from the response
-        return d
+        return data
+
+
+@retry(stop_max_attempt_number=3, wait_fixed=5000)
+def order_and_download_certificate(session, url, data):
+    """
+    Helper function to place a certificacte order and download it
+    :param session:
+    :param url: Entrust endpoint url
+    :param data: CSR, and the required order details, such as validity length
+    :return: the cert chain
+    :raise Exception:
+    """
+    try:
+        response = session.post(url, json=data, timeout=(15, 40))
+    except requests.exceptions.Timeout:
+        raise Exception("Timeout for POST")
+    except requests.exceptions.RequestException as e:
+        raise Exception(f"Error for POST {e}")
+
+    return handle_response(response)
 
 
 class EntrustIssuerPlugin(IssuerPlugin):
@@ -178,14 +197,8 @@ class EntrustIssuerPlugin(IssuerPlugin):
         data = process_options(issuer_options)
         data["csr"] = csr
 
-        try:
+        response_dict = order_and_download_certificate(self.session, url, data)
-            response = self.session.post(url, json=data, timeout=(15, 40))
-        except requests.exceptions.Timeout:
-            raise Exception("Timeout for POST")
-        except requests.exceptions.RequestException as e:
-            raise Exception(f"Error for POST {e}")
 
-        response_dict = handle_response(response)
         external_id = response_dict['trackingId']
         cert = response_dict['endEntityCert']
         if len(response_dict['chainCerts']) < 2:
@@ -200,6 +213,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
 
         return cert, chain, external_id
 
+    @retry(stop_max_attempt_number=3, wait_fixed=1000)
     def revoke_certificate(self, certificate, comments):
         """Revoke an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
@@ -216,6 +230,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
         metrics.send("entrust_revoke_certificate", "counter", 1)
         return handle_response(response)
 
+    @retry(stop_max_attempt_number=3, wait_fixed=1000)
     def deactivate_certificate(self, certificate):
         """Deactivates an Entrust certificate."""
         base_url = current_app.config.get("ENTRUST_URL")
@@ -244,7 +259,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
     def get_ordered_certificate(self, order_id):
         raise NotImplementedError("Not implemented\n", self, order_id)
 
-    def canceled_ordered_certificate(self, pending_cert, **kwargs):
+    def cancel_ordered_certificate(self, pending_cert, **kwargs):
         raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs)
 
 

lemur/tests/test_certificates.py

@@ -802,6 +802,7 @@ def test_reissue_certificate(
     assert new_cert.organization != certificate.organization
     # Check for default value since authority does not have cab_compliant option set
     assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION
+    assert new_cert.description.startswith(f"Reissued by Lemur for cert ID {certificate.id}")
 
     # update cab_compliant option to false for crypto_authority to maintain subject details
     update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]')