Merge branch 'master' into certificates-for-notification-fix

This commit is contained in:
Jasmine Schladen 2020-10-29 14:21:25 -07:00 committed by GitHub
commit 86207db93b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 144 additions and 53 deletions

View File

@ -47,4 +47,4 @@ after_success:
notifications: notifications:
email: email:
ccastrapel@netflix.com lemur@netflix.com

View File

@ -28,6 +28,13 @@ Basic Configuration
LOG_FILE = "/logs/lemur/lemur-test.log" LOG_FILE = "/logs/lemur/lemur-test.log"
.. data:: LOG_UPGRADE_FILE
:noindex:
::
LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log"
.. data:: DEBUG .. data:: DEBUG
:noindex: :noindex:
@ -285,6 +292,25 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_ you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
.. data:: LEMUR_SES_SOURCE_ARN
:noindex:
Specifies an ARN to use as the SourceArn when sending emails via SES.
.. note::
This parameter is only required if you're using a sending authorization with SES.
See: `Using sending authorization with Amazon SES <https://docs.aws.amazon.com/ses/latest/DeveloperGuide/sending-authorization.html>`_
.. data:: LEMUR_SES_REGION
:noindex:
Specifies a region for sending emails via SES.
.. note::
This parameter defaults to us-east-1 and is only required if you wish to use a different region.
.. data:: LEMUR_EMAIL .. data:: LEMUR_EMAIL
:noindex: :noindex:

View File

@ -6,6 +6,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com> .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
""" """
import arrow import arrow
import re
from cryptography import x509 from cryptography import x509
from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives import hashes, serialization
@ -779,6 +780,19 @@ def reissue_certificate(certificate, replace=None, user=None):
if replace: if replace:
primitives["replaces"] = [certificate] primitives["replaces"] = [certificate]
# Modify description to include the certificate ID being reissued and mention that this is created by Lemur
# as part of reissue
reissue_message_prefix = "Reissued by Lemur for cert ID "
reissue_message = re.compile(f"{reissue_message_prefix}([0-9]+)")
if primitives["description"]:
match = reissue_message.search(primitives["description"])
if match:
primitives["description"] = primitives["description"].replace(match.group(1), str(certificate.id))
else:
primitives["description"] = f"{reissue_message_prefix}{certificate.id}, {primitives['description']}"
else:
primitives["description"] = f"{reissue_message_prefix}{certificate.id}"
new_cert = create(**primitives) new_cert = create(**primitives)
return new_cert return new_cert

View File

@ -120,6 +120,7 @@ METRIC_PROVIDERS = []
LOG_LEVEL = "DEBUG" LOG_LEVEL = "DEBUG"
LOG_FILE = "lemur.log" LOG_FILE = "lemur.log"
LOG_UPGRADE_FILE = "db_upgrade.log"
# Database # Database

View File

@ -10,11 +10,21 @@ Create Date: 2018-08-03 12:56:44.565230
revision = "1db4f82bc780" revision = "1db4f82bc780"
down_revision = "3adfdd6598df" down_revision = "3adfdd6598df"
import logging
from alembic import op from alembic import op
log = logging.getLogger(__name__) from flask import current_app
from logging import Formatter, FileHandler, getLogger
log = getLogger(__name__)
handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
handler.setFormatter(
Formatter(
"%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
)
)
handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
log.addHandler(handler)
def upgrade(): def upgrade():

View File

@ -7,8 +7,9 @@ the rest of the keys, the certificate body is parsed to determine
the exact key_type information. the exact key_type information.
Each individual DB change is explicitly committed, and the respective Each individual DB change is explicitly committed, and the respective
log is added to a file named db_upgrade.log in the current working log is added to a file configured in LOG_UPGRADE_FILE or, by default,
directory. Any error encountered while parsing a certificate will to a file named db_upgrade.log in the current working directory.
Any error encountered while parsing a certificate will
also be logged along with the certificate ID. If faced with any issue also be logged along with the certificate ID. If faced with any issue
while running this upgrade, there is no harm in re-running the upgrade. while running this upgrade, there is no harm in re-running the upgrade.
Each run processes only rows for which key_type information is not yet Each run processes only rows for which key_type information is not yet
@ -31,15 +32,28 @@ down_revision = '434c29e40511'
from alembic import op from alembic import op
from sqlalchemy.sql import text from sqlalchemy.sql import text
from lemur.common import utils
import time import time
import datetime import datetime
from flask import current_app
log_file = open('db_upgrade.log', 'a') from logging import Formatter, FileHandler, getLogger
from lemur.common import utils
log = getLogger(__name__)
handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
handler.setFormatter(
Formatter(
"%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
)
)
handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
log.addHandler(handler)
def upgrade(): def upgrade():
log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) log.info("\n*** Starting new run(%s) ***\n" % datetime.datetime.now())
start_time = time.time() start_time = time.time()
# Update RSA keys using the key length information # Update RSA keys using the key length information
@ -50,8 +64,7 @@ def upgrade():
# Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs. # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs.
update_key_type() update_key_type()
log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time)) log.info("--- Total %s seconds ---\n" % (time.time() - start_time))
log_file.close()
def downgrade(): def downgrade():
@ -69,18 +82,18 @@ def downgrade():
def update_key_type_rsa(bits): def update_key_type_rsa(bits):
log_file.write("Processing certificate with key type RSA %s\n" % bits) log.info("Processing certificate with key type RSA %s\n" % bits)
stmt = text( stmt = text(
f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null" f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null"
) )
log_file.write("Query: %s\n" % stmt) log.info("Query: %s\n" % stmt)
start_time = time.time() start_time = time.time()
op.execute(stmt) op.execute(stmt)
commit() commit()
log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) log.info("--- %s seconds ---\n" % (time.time() - start_time))
def update_key_type(): def update_key_type():
@ -95,9 +108,9 @@ def update_key_type():
try: try:
cert_key_type = utils.get_key_type_from_certificate(body) cert_key_type = utils.get_key_type_from_certificate(body)
except ValueError as e: except ValueError as e:
log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) log.error("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e)))
else: else:
log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type))
stmt = text( stmt = text(
"update certificates set key_type=:key_type where id=:id" "update certificates set key_type=:key_type where id=:id"
) )
@ -106,7 +119,7 @@ def update_key_type():
commit() commit()
log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) log.info("--- %s seconds ---\n" % (time.time() - start_time))
def commit(): def commit():

View File

@ -15,16 +15,18 @@ from flask import current_app
def publish(topic_arn, certificates, notification_type, **kwargs): def publish(topic_arn, certificates, notification_type, **kwargs):
sns_client = boto3.client("sns", **kwargs) sns_client = boto3.client("sns", **kwargs)
message_ids = {} message_ids = {}
subject = "Lemur: {0} Notification".format(notification_type.capitalize())
for certificate in certificates: for certificate in certificates:
message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type) message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject)
return message_ids return message_ids
def publish_single(sns_client, topic_arn, certificate, notification_type): def publish_single(sns_client, topic_arn, certificate, notification_type, subject):
response = sns_client.publish( response = sns_client.publish(
TopicArn=topic_arn, TopicArn=topic_arn,
Message=format_message(certificate, notification_type), Message=format_message(certificate, notification_type),
Subject=subject,
) )
response_code = response["ResponseMetadata"]["HTTPStatusCode"] response_code = response["ResponseMetadata"]["HTTPStatusCode"]
@ -48,8 +50,9 @@ def format_message(certificate, notification_type):
json_message = { json_message = {
"notification_type": notification_type, "notification_type": notification_type,
"certificate_name": certificate["name"], "certificate_name": certificate["name"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"), # 2047-12-T22:00:00 "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), # 2047-12-31T22:00:00
"endpoints_detected": len(certificate["endpoints"]), "endpoints_detected": len(certificate["endpoints"]),
"owner": certificate["owner"],
"details": create_certificate_url(certificate["name"]) "details": create_certificate_url(certificate["name"])
} }
return json.dumps(json_message) return json.dumps(json_message)

View File

@ -20,8 +20,9 @@ def test_format(certificate, endpoint):
expected_message = { expected_message = {
"notification_type": "expiration", "notification_type": "expiration",
"certificate_name": certificate["name"], "certificate_name": certificate["name"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"), "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
"endpoints_detected": 0, "endpoints_detected": 0,
"owner": certificate["owner"],
"details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
} }
assert expected_message == json.loads(format_message(certificate, "expiration")) assert expected_message == json.loads(format_message(certificate, "expiration"))
@ -57,7 +58,9 @@ def test_publish(certificate, endpoint):
expected_message_id = message_ids[certificate["name"]] expected_message_id = message_ids[certificate["name"]]
actual_message = next( actual_message = next(
(m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None) (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None)
assert json.loads(actual_message["Body"])["Message"] == format_message(certificate, "expiration") actual_json = json.loads(actual_message["Body"])
assert actual_json["Message"] == format_message(certificate, "expiration")
assert actual_json["Subject"] == "Lemur: Expiration Notification"
def get_options(): def get_options():

View File

@ -234,7 +234,7 @@ def handle_cis_response(response):
return response.json() return response.json()
@retry(stop_max_attempt_number=10, wait_fixed=10000) @retry(stop_max_attempt_number=10, wait_fixed=1000)
def get_certificate_id(session, base_url, order_id): def get_certificate_id(session, base_url, order_id):
"""Retrieve certificate order id from Digicert API.""" """Retrieve certificate order id from Digicert API."""
order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id) order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id)
@ -245,7 +245,7 @@ def get_certificate_id(session, base_url, order_id):
return response_data["certificate"]["id"] return response_data["certificate"]["id"]
@retry(stop_max_attempt_number=10, wait_fixed=10000) @retry(stop_max_attempt_number=10, wait_fixed=1000)
def get_cis_certificate(session, base_url, order_id): def get_cis_certificate(session, base_url, order_id):
"""Retrieve certificate order id from Digicert API, including the chain""" """Retrieve certificate order id from Digicert API, including the chain"""
certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id) certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id)

View File

@ -38,7 +38,7 @@ def render_html(template_name, options, certificates):
def send_via_smtp(subject, body, targets): def send_via_smtp(subject, body, targets):
""" """
Attempts to deliver email notification via SES service. Attempts to deliver email notification via SMTP.
:param subject: :param subject:
:param body: :param body:
@ -55,21 +55,26 @@ def send_via_smtp(subject, body, targets):
def send_via_ses(subject, body, targets): def send_via_ses(subject, body, targets):
""" """
Attempts to deliver email notification via SMTP. Attempts to deliver email notification via SES service.
:param subject: :param subject:
:param body: :param body:
:param targets: :param targets:
:return: :return:
""" """
client = boto3.client("ses", region_name="us-east-1") ses_region = current_app.config.get("LEMUR_SES_REGION", "us-east-1")
client.send_email( client = boto3.client("ses", region_name=ses_region)
Source=current_app.config.get("LEMUR_EMAIL"), source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN")
Destination={"ToAddresses": targets}, args = {
Message={ "Source": current_app.config.get("LEMUR_EMAIL"),
"Destination": {"ToAddresses": targets},
"Message": {
"Subject": {"Data": subject, "Charset": "UTF-8"}, "Subject": {"Data": subject, "Charset": "UTF-8"},
"Body": {"Html": {"Data": body, "Charset": "UTF-8"}}, "Body": {"Html": {"Data": body, "Charset": "UTF-8"}},
}, },
) }
if source_arn:
args["SourceArn"] = source_arn
client.send_email(**args)
class EmailNotificationPlugin(ExpirationNotificationPlugin): class EmailNotificationPlugin(ExpirationNotificationPlugin):

View File

@ -1,9 +1,9 @@
import arrow import arrow
import requests import requests
import json import json
import sys import sys
from flask import current_app from flask import current_app
from retrying import retry
from lemur.plugins import lemur_entrust as entrust from lemur.plugins import lemur_entrust as entrust
from lemur.plugins.bases import IssuerPlugin, SourcePlugin from lemur.plugins.bases import IssuerPlugin, SourcePlugin
@ -78,7 +78,6 @@ def process_options(options):
"eku": "SERVER_AND_CLIENT_AUTH", "eku": "SERVER_AND_CLIENT_AUTH",
"certType": product_type, "certType": product_type,
"certExpiryDate": validity_end, "certExpiryDate": validity_end,
# "keyType": "RSA", Entrust complaining about this parameter
"tracking": tracking_data "tracking": tracking_data
} }
return data return data
@ -87,7 +86,7 @@ def process_options(options):
def handle_response(my_response): def handle_response(my_response):
""" """
Helper function for parsing responses from the Entrust API. Helper function for parsing responses from the Entrust API.
:param content: :param my_response:
:return: :raise Exception: :return: :raise Exception:
""" """
msg = { msg = {
@ -100,27 +99,47 @@ def handle_response(my_response):
} }
try: try:
d = json.loads(my_response.content) data = json.loads(my_response.content)
except ValueError: except ValueError:
# catch an empty jason object here # catch an empty jason object here
d = {'response': 'No detailed message'} data = {'response': 'No detailed message'}
s = my_response.status_code status_code = my_response.status_code
if s > 399: if status_code > 399:
raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}") raise Exception(f"ENTRUST error: {msg.get(status_code, status_code)}\n{data['errors']}")
log_data = { log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}", "function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Response", "message": "Response",
"status": s, "status": status_code,
"response": d "response": data
} }
current_app.logger.info(log_data) current_app.logger.info(log_data)
if d == {'response': 'No detailed message'}: if data == {'response': 'No detailed message'}:
# status if no data # status if no data
return s return status_code
else: else:
# return data from the response # return data from the response
return d return data
@retry(stop_max_attempt_number=3, wait_fixed=5000)
def order_and_download_certificate(session, url, data):
"""
Helper function to place a certificacte order and download it
:param session:
:param url: Entrust endpoint url
:param data: CSR, and the required order details, such as validity length
:return: the cert chain
:raise Exception:
"""
try:
response = session.post(url, json=data, timeout=(15, 40))
except requests.exceptions.Timeout:
raise Exception("Timeout for POST")
except requests.exceptions.RequestException as e:
raise Exception(f"Error for POST {e}")
return handle_response(response)
class EntrustIssuerPlugin(IssuerPlugin): class EntrustIssuerPlugin(IssuerPlugin):
@ -178,14 +197,8 @@ class EntrustIssuerPlugin(IssuerPlugin):
data = process_options(issuer_options) data = process_options(issuer_options)
data["csr"] = csr data["csr"] = csr
try: response_dict = order_and_download_certificate(self.session, url, data)
response = self.session.post(url, json=data, timeout=(15, 40))
except requests.exceptions.Timeout:
raise Exception("Timeout for POST")
except requests.exceptions.RequestException as e:
raise Exception(f"Error for POST {e}")
response_dict = handle_response(response)
external_id = response_dict['trackingId'] external_id = response_dict['trackingId']
cert = response_dict['endEntityCert'] cert = response_dict['endEntityCert']
if len(response_dict['chainCerts']) < 2: if len(response_dict['chainCerts']) < 2:
@ -200,6 +213,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
return cert, chain, external_id return cert, chain, external_id
@retry(stop_max_attempt_number=3, wait_fixed=1000)
def revoke_certificate(self, certificate, comments): def revoke_certificate(self, certificate, comments):
"""Revoke an Entrust certificate.""" """Revoke an Entrust certificate."""
base_url = current_app.config.get("ENTRUST_URL") base_url = current_app.config.get("ENTRUST_URL")
@ -216,6 +230,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
metrics.send("entrust_revoke_certificate", "counter", 1) metrics.send("entrust_revoke_certificate", "counter", 1)
return handle_response(response) return handle_response(response)
@retry(stop_max_attempt_number=3, wait_fixed=1000)
def deactivate_certificate(self, certificate): def deactivate_certificate(self, certificate):
"""Deactivates an Entrust certificate.""" """Deactivates an Entrust certificate."""
base_url = current_app.config.get("ENTRUST_URL") base_url = current_app.config.get("ENTRUST_URL")
@ -244,7 +259,7 @@ class EntrustIssuerPlugin(IssuerPlugin):
def get_ordered_certificate(self, order_id): def get_ordered_certificate(self, order_id):
raise NotImplementedError("Not implemented\n", self, order_id) raise NotImplementedError("Not implemented\n", self, order_id)
def canceled_ordered_certificate(self, pending_cert, **kwargs): def cancel_ordered_certificate(self, pending_cert, **kwargs):
raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs) raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs)

View File

@ -802,6 +802,7 @@ def test_reissue_certificate(
assert new_cert.organization != certificate.organization assert new_cert.organization != certificate.organization
# Check for default value since authority does not have cab_compliant option set # Check for default value since authority does not have cab_compliant option set
assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION
assert new_cert.description.startswith(f"Reissued by Lemur for cert ID {certificate.id}")
# update cab_compliant option to false for crypto_authority to maintain subject details # update cab_compliant option to false for crypto_authority to maintain subject details
update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]') update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]')