From fc205713c859d1d04f3dce9f6bac9c463c292b43 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Wed, 7 Dec 2016 16:24:59 -0800 Subject: [PATCH] Certificate rotation enhancements (#570) --- lemur/analyze/service.py | 64 ----- lemur/certificates/cli.py | 167 +++++++++++++ lemur/certificates/exceptions.py | 88 ------- lemur/certificates/models.py | 2 + lemur/certificates/schemas.py | 11 + lemur/certificates/service.py | 41 +++- lemur/deployment/service.py | 19 ++ lemur/endpoints/service.py | 8 +- lemur/manage.py | 231 +----------------- lemur/notifications/cli.py | 29 +++ lemur/notifications/messaging.py | 102 ++++++++ lemur/notifications/service.py | 176 +------------ lemur/plugins/lemur_email/plugin.py | 79 ++++-- .../lemur_email/templates/rotation.html | 48 +++- lemur/plugins/utils.py | 21 ++ lemur/sources/cli.py | 88 +++++++ .../__init__.py => tests/test_deployment.py} | 0 lemur/tests/test_endpoints.py | 2 +- lemur/tests/test_messaging.py | 29 +++ 19 files changed, 607 insertions(+), 598 deletions(-) delete mode 100644 lemur/analyze/service.py create mode 100644 lemur/certificates/cli.py delete mode 100644 lemur/certificates/exceptions.py create mode 100644 lemur/deployment/service.py create mode 100644 lemur/notifications/cli.py create mode 100644 lemur/notifications/messaging.py create mode 100644 lemur/plugins/utils.py create mode 100644 lemur/sources/cli.py rename lemur/{analyze/__init__.py => tests/test_deployment.py} (100%) create mode 100644 lemur/tests/test_messaging.py diff --git a/lemur/analyze/service.py b/lemur/analyze/service.py deleted file mode 100644 index 8c384306..00000000 --- a/lemur/analyze/service.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -.. module: lemur.analyze.service - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. -.. moduleauthor:: Kevin Glisson -""" -# def analyze(endpoints, truststores): -# results = {"headings": ["Endpoint"], -# "results": [], -# "time": datetime.now().strftime("#Y%m%d %H:%M:%S")} -# -# for store in truststores: -# results['headings'].append(os.path.basename(store)) -# -# for endpoint in endpoints: -# result_row = [endpoint] -# for store in truststores: -# result = {'details': []} -# -# tests = [] -# for region, ip in REGIONS.items(): -# try: -# domain = dns.name.from_text(endpoint) -# if not domain.is_absolute(): -# domain = domain.concatenate(dns.name.root) -# -# my_resolver = dns.resolver.Resolver() -# my_resolver.nameservers = [ip] -# answer = my_resolver.query(domain) -# -# #force the testing of regional enpoints by changing the dns server -# response = requests.get('https://' + str(answer[0]), verify=store) -# tests.append('pass') -# result['details'].append("{}: SSL testing completed without errors".format(region)) -# -# except SSLError as e: -# log.debug(e) -# if 'hostname' in str(e): -# tests.append('pass') -# result['details'].append( -# "{}: This test passed ssl negotiation but failed hostname verification because \ -# the hostname is not included in the certificate".format(region)) -# elif 'certificate verify failed' in str(e): -# tests.append('fail') -# result['details'].append("{}: This test failed to verify the SSL certificate".format(region)) -# else: -# tests.append('fail') -# result['details'].append("{}: {}".format(region, str(e))) -# -# except Exception as e: -# log.debug(e) -# tests.append('fail') -# result['details'].append("{}: {}".format(region, str(e))) -# -# #any failing tests fails the whole endpoint -# if 'fail' in tests: -# result['test'] = 'fail' -# else: -# result['test'] = 'pass' -# -# result_row.append(result) -# results['results'].append(result_row) -# return results -# diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py new file mode 100644 index 00000000..8d0e299b --- /dev/null +++ b/lemur/certificates/cli.py @@ -0,0 +1,167 @@ +""" +.. module: lemur.certificate.cli + :platform: Unix + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +import sys + +from flask import current_app + +from flask_script import Manager + +from lemur import database +from lemur.deployment.service import rotate_certificate +from lemur.notifications.messaging import send_rotation_notification +from lemur.certificates.service import reissue_certificate, get_certificate_primitives, get_all_pending_rotation, get_by_name, get_all_certs + +from lemur.certificates.verify import verify_string + +manager = Manager(usage="Handles all certificate related tasks.") + + +def reissue_and_rotate(old_certificate, new_certificate=None, commit=False, message=False): + if not new_certificate: + # we don't want to re-issue if it's already been replaced + if not old_certificate.replaced: + details = get_certificate_primitives(old_certificate) + print_certificate_details(details) + + if commit: + new_certificate = reissue_certificate(old_certificate, replace=True) + print("[+] Issued new certificate named: {0}".format(new_certificate.name)) + + print("[+] Done!") + + else: + new_certificate = old_certificate.replaced + print("[!] Certificate has been replaced by: {0}".format(old_certificate.replaced.name)) + + if len(old_certificate.endpoints) > 0: + for endpoint in old_certificate.endpoints: + print( + "[+] Certificate deployed on endpoint: name:{name} dnsname:{dnsname} port:{port} type:{type}".format( + name=endpoint.name, + dnsname=endpoint.dnsname, + port=endpoint.port, + type=endpoint.type + ) + ) + print("[+] Rotating certificate from: {0} to: {1}".format(old_certificate.name, new_certificate.name)) + + if commit: + rotate_certificate(endpoint, new_certificate) + + print("[+] Done!") + + if message: + send_rotation_notification(old_certificate) + + +def print_certificate_details(details): + """ + Print the certificate details with formatting. + :param details: + :return: + """ + print("[+] Re-issuing certificate with the following details: ") + print( + "[+] Common Name: {common_name}\n" + "[+] Subject Alternate Names: {sans}\n" + "[+] Authority: {authority_name}\n" + "[+] Validity Start: {validity_start}\n" + "[+] Validity End: {validity_end}\n" + "[+] Organization: {organization}\n" + "[+] Organizational Unit: {organizational_unit}\n" + "[+] Country: {country}\n" + "[+] State: {state}\n" + "[+] Location: {location}\n".format( + common_name=details['common_name'], + sans=",".join(x['value'] for x in details['extensions']['sub_alt_names']['names']), + authority_name=details['authority'].name, + validity_start=details['validity_start'].isoformat(), + validity_end=details['validity_end'].isoformat(), + organization=details['organization'], + organizational_unit=details['organizational_unit'], + country=details['country'], + state=details['state'], + location=details['location'] + ) + ) + + +@manager.command +def rotate(new_certificate_name=False, old_certificate_name=False, message=False, commit=False): + new_cert = old_cert = None + + if commit: + print("[!] Running in COMMIT mode.") + + if old_certificate_name: + old_cert = get_by_name(old_certificate_name) + + if not old_cert: + print("[-] No certificate found with name: {0}".format(old_certificate_name)) + sys.exit(1) + + if new_certificate_name: + new_cert = get_by_name(new_certificate_name) + + if not new_cert: + print("[-] No certificate found with name: {0}".format(old_certificate_name)) + sys.exit(1) + + if old_cert and new_cert: + reissue_and_rotate(old_cert, new_certificate=new_cert, commit=commit, message=message) + else: + for certificate in get_all_pending_rotation(): + reissue_and_rotate(certificate, commit=commit, message=message) + + +@manager.command +def reissue(old_certificate_name, commit=False): + from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives + + old_cert = get_by_name(old_certificate_name) + + if not old_cert: + print("[-] No certificate found with name: {0}".format(old_certificate_name)) + sys.exit(1) + + if commit: + print("[!] Running in COMMIT mode.") + + details = get_certificate_primitives(old_cert) + print_certificate_details(details) + + if commit: + new_cert = reissue_certificate(old_cert, replace=True) + print("[+] Issued new certificate named: {0}".format(new_cert.name)) + + print("[+] Done!") + + +@manager.command +def check_revoked(): + """ + Function attempts to update Lemur's internal cache with revoked + certificates. This is called periodically by Lemur. It checks both + CRLs and OCSP to see if a certificate is revoked. If Lemur is unable + encounters an issue with verification it marks the certificate status + as `unknown`. + """ + for cert in get_all_certs(): + try: + if cert.chain: + status = verify_string(cert.body, cert.chain) + else: + status = verify_string(cert.body, "") + + cert.status = 'valid' if status else 'invalid' + + except Exception as e: + current_app.logger.exception(e) + cert.status = 'unknown' + + database.update(cert) diff --git a/lemur/certificates/exceptions.py b/lemur/certificates/exceptions.py deleted file mode 100644 index 83437704..00000000 --- a/lemur/certificates/exceptions.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -.. module: lemur.certificates.exceptions - :synopsis: Defines all monterey specific exceptions - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. -.. moduleauthor:: Kevin Glisson -""" -from flask import current_app -from lemur.exceptions import LemurException - - -class UnknownAuthority(LemurException): - def __init__(self, authority): - self.code = 404 - self.authority = authority - self.data = {"message": "The authority specified '{}' is not a valid authority".format(self.authority)} - - current_app.logger.warning(self) - - def __str__(self): - return repr(self.data['message']) - - -class InsufficientDomains(LemurException): - def __init__(self): - self.code = 400 - self.data = {"message": "Need at least one domain specified in order create a certificate"} - - current_app.logger.warning(self) - - def __str__(self): - return repr(self.data['message']) - - -class InvalidCertificate(LemurException): - def __init__(self): - self.code = 400 - self.data = {"message": "Need at least one domain specified in order create a certificate"} - - current_app.logger.warning(self) - - def __str__(self): - return repr(self.data['message']) - - -class UnableToCreateCSR(LemurException): - def __init__(self): - self.code = 500 - self.data = {"message": "Unable to generate CSR"} - - current_app.logger.error(self) - - def __str__(self): - return repr(self.data['message']) - - -class UnableToCreatePrivateKey(LemurException): - def __init__(self): - self.code = 500 - self.data = {"message": "Unable to generate Private Key"} - - current_app.logger.error(self) - - def __str__(self): - return repr(self.data['message']) - - -class MissingFiles(LemurException): - def __init__(self, path): - self.code = 500 - self.path = path - self.data = {"path": self.path, "message": "Expecting missing files"} - - current_app.logger.error(self) - - def __str__(self): - return repr(self.data['message']) - - -class NoPersistanceFound(LemurException): - def __init__(self): - self.code = 500 - self.data = {"code": 500, "message": "No peristence method found, Lemur cannot persist sensitive information"} - - current_app.logger.error(self) - - def __str__(self): - return repr(self.data['message']) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 31f3c6cc..2c4d0c03 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -66,6 +66,8 @@ class Certificate(db.Model): bits = Column(Integer()) san = Column(String(1024)) # TODO this should be migrated to boolean + rotation = Column(Boolean) + user_id = Column(Integer, ForeignKey('users.id')) authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE")) root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE")) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 59b220aa..072cb653 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -201,9 +201,20 @@ class CertificateExportInputSchema(LemurInputSchema): plugin = fields.Nested(PluginInputSchema) +class CertificateNotificationOutputSchema(LemurOutputSchema): + description = fields.String() + issuer = fields.String() + name = fields.String() + owner = fields.Email() + user = fields.Nested(UserNestedOutputSchema) + replaces = fields.Nested(CertificateNestedOutputSchema, many=True) + endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[]) + + certificate_input_schema = CertificateInputSchema() certificate_output_schema = CertificateOutputSchema() certificates_output_schema = CertificateOutputSchema(many=True) certificate_upload_input_schema = CertificateUploadInputSchema() certificate_export_input_schema = CertificateExportInputSchema() certificate_edit_input_schema = CertificateEditInputSchema() +certificate_notification_output_schema = CertificateNotificationOutputSchema() diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index ea2ebc5d..d4d8221c 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -1,14 +1,15 @@ """ -.. module: service +.. module: lemur.certificate.service :platform: Unix :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ import arrow +from datetime import timedelta -from sqlalchemy import func, or_ from flask import current_app +from sqlalchemy import func, or_ from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -17,15 +18,15 @@ from cryptography.hazmat.primitives import hashes, serialization from lemur import database from lemur.extensions import metrics from lemur.plugins.base import plugins -from lemur.certificates.models import Certificate from lemur.common.utils import generate_private_key -from lemur.destinations.models import Destination -from lemur.notifications.models import Notification -from lemur.authorities.models import Authority -from lemur.domains.models import Domain - from lemur.roles.models import Role +from lemur.domains.models import Domain +from lemur.authorities.models import Authority +from lemur.destinations.models import Destination +from lemur.certificates.models import Certificate +from lemur.notifications.models import Notification + from lemur.roles import service as role_service @@ -77,6 +78,24 @@ def get_by_source(source_label): return Certificate.query.filter(Certificate.sources.any(label=source_label)) +def get_all_pending_rotation(): + """ + Retrieves all certificates that need to be rotated. + + Must be X days from expiration, uses `LEMUR_DEFAULT_ROTATION_INTERVAL` + to determine how many days from expiration the certificate must be + for rotation to be pending. + + :return: + """ + now = arrow.utcnow() + interval = current_app.config.get('LEMUR_DEFAULT_ROTATION_INTERVAL', 30) + end = now + timedelta(days=interval) + + return Certificate.query.filter(Certificate.rotation == True)\ + .filter(Certificate.not_after <= end.format('YYYY-MM-DD')).all() # noqa + + def find_duplicates(cert): """ Finds certificates that already exist within Lemur. We do this by looking for @@ -527,9 +546,9 @@ def reissue_certificate(certificate, replace=None, user=None): else: primitives['creator'] = user + if replace: + primitives['replaces'] = certificate + new_cert = create(**primitives) - if replace: - certificate.notify = False - return new_cert diff --git a/lemur/deployment/service.py b/lemur/deployment/service.py new file mode 100644 index 00000000..e11e5fd7 --- /dev/null +++ b/lemur/deployment/service.py @@ -0,0 +1,19 @@ +from flask import current_app +from lemur.extensions import metrics + + +def rotate_certificate(endpoint, new_cert): + """ + Rotates a certificate on a given endpoint. + + :param endpoint: + :param new_cert: + :return: + """ + try: + endpoint.source.plugin.update_endpoint(endpoint, new_cert) + endpoint.certificate = new_cert + metrics.send('rotation_success', 'counter', 1, metric_tags={'endpoint': endpoint.name}) + except Exception as e: + metrics.send('rotation_failure', 'counter', 1, metric_tags={'endpoint': endpoint.name}) + current_app.logger.exception(e) diff --git a/lemur/endpoints/service.py b/lemur/endpoints/service.py index 2d2f5ecc..1d95570c 100644 --- a/lemur/endpoints/service.py +++ b/lemur/endpoints/service.py @@ -10,12 +10,12 @@ """ from flask import current_app -from lemur import database -from lemur.extensions import metrics -from lemur.endpoints.models import Endpoint, Policy, Cipher - from sqlalchemy import func +from lemur import database +from lemur.endpoints.models import Endpoint, Policy, Cipher +from lemur.extensions import metrics + def get_all(): """ diff --git a/lemur/manage.py b/lemur/manage.py index 513e1530..42bee6b5 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -7,7 +7,6 @@ from collections import Counter import os import sys import base64 -import time import requests import json @@ -21,16 +20,15 @@ from flask_script import Manager, Command, Option, prompt_pass from flask_migrate import Migrate, MigrateCommand, stamp from flask_script.commands import ShowUrls, Clean, Server +from lemur.sources.cli import manager as source_manager +from lemur.certificates.cli import manager as certificate_manager + from lemur import database -from lemur.extensions import metrics from lemur.users import service as user_service from lemur.roles import service as role_service -from lemur.certificates import service as cert_service from lemur.authorities import service as authority_service from lemur.notifications import service as notification_service -from lemur.certificates.verify import verify_string -from lemur.sources import service as source_service from lemur.common.utils import validate_conf @@ -148,31 +146,6 @@ def drop_all(): database.db.drop_all() -@manager.command -def check_revoked(): - """ - Function attempts to update Lemur's internal cache with revoked - certificates. This is called periodically by Lemur. It checks both - CRLs and OCSP to see if a certificate is revoked. If Lemur is unable - encounters an issue with verification it marks the certificate status - as `unknown`. - """ - for cert in cert_service.get_all_certs(): - try: - if cert.chain: - status = verify_string(cert.body, cert.chain) - else: - status = verify_string(cert.body, "") - - cert.status = 'valid' if status else 'invalid' - - except Exception as e: - current_app.logger.exception(e) - cert.status = 'unknown' - - database.update(cert) - - @manager.shell def make_shell_context(): """ @@ -201,23 +174,6 @@ def generate_settings(): return output -@manager.command -def notify(): - """ - Runs Lemur's notification engine, that looks for expired certificates and sends - notifications out to those that bave subscribed to them. - - :return: - """ - sys.stdout.write("Starting to notify subscribers about expiring certificates!\n") - count = notification_service.send_expiration_notifications() - sys.stdout.write( - "Finished notifying subscribers about expiring certificates! Sent {count} notifications!\n".format( - count=count - ) - ) - - class InitializeApp(Command): """ This command will bootstrap our database with any destinations as @@ -521,117 +477,6 @@ def unlock(path=None): sys.stdout.write("[+] Keys have been unencrypted!\n") -def print_certificate_details(details): - """ - Print the certificate details with formatting. - :param details: - :return: - """ - sys.stdout.write("[+] Re-issuing certificate with the following details: \n") - sys.stdout.write( - "[+] Common Name: {common_name}\n" - "[+] Subject Alternate Names: {sans}\n" - "[+] Authority: {authority_name}\n" - "[+] Validity Start: {validity_start}\n" - "[+] Validity End: {validity_end}\n" - "[+] Organization: {organization}\n" - "[+] Organizational Unit: {organizational_unit}\n" - "[+] Country: {country}\n" - "[+] State: {state}\n" - "[+] Location: {location}\n".format( - common_name=details['common_name'], - sans=",".join(x['value'] for x in details['extensions']['sub_alt_names']['names']), - authority_name=details['authority'].name, - validity_start=details['validity_start'].isoformat(), - validity_end=details['validity_end'].isoformat(), - organization=details['organization'], - organizational_unit=details['organizational_unit'], - country=details['country'], - state=details['state'], - location=details['location'] - ) - ) - - -certificate_manager = Manager(usage="Handles all certificate related tasks.") - - -@certificate_manager.command -def rotate(new_certificate_name, old_certificate_name, commit=False): - from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives - from lemur.endpoints.service import rotate_certificate - - old_cert = get_by_name(old_certificate_name) - - if not old_cert: - sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name)) - sys.exit(1) - - if new_certificate_name: - new_cert = get_by_name(new_certificate_name) - - if not new_cert: - sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name)) - sys.exit(1) - - if commit: - sys.stdout.write("[!] Running in COMMIT mode.\n") - - if not new_certificate_name: - sys.stdout.write("[!] No new certificate provided. Attempting to re-issue old certificate: {0}.\n".format(old_certificate_name)) - - details = get_certificate_primitives(old_cert) - print_certificate_details(details) - - if commit: - new_cert = reissue_certificate(old_cert, replace=True) - sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name)) - - sys.stdout.write("[+] Done! \n") - - if len(old_cert.endpoints) > 0: - for endpoint in old_cert.endpoints: - sys.stdout.write( - "[+] Certificate deployed on endpoint: name:{name} dnsname:{dnsname} port:{port} type:{type}\n".format( - name=endpoint.name, - dnsname=endpoint.dnsname, - port=endpoint.port, - type=endpoint.type - ) - ) - sys.stdout.write("[+] Rotating certificate from: {0} to: {1}\n".format(old_certificate_name, new_cert.name)) - - if commit: - rotate_certificate(endpoint, new_cert) - - sys.stdout.write("[+] Done! \n") - else: - sys.stdout.write("[!] Certificate not found on any existing endpoints. Nothing to rotate.\n") - - -@certificate_manager.command -def reissue(old_certificate_name, commit=False): - from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives - - old_cert = get_by_name(old_certificate_name) - - if not old_cert: - sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name)) - sys.exit(1) - - if commit: - sys.stdout.write("[!] Running in COMMIT mode.\n") - - details = get_certificate_primitives(old_cert) - print_certificate_details(details) - - if commit: - new_cert = reissue_certificate(old_cert, replace=True) - sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name)) - - sys.stdout.write("[+] Done! \n") - - @manager.command def publish_verisign_units(): """ @@ -771,76 +616,6 @@ class Report(Command): sys.stdout.write(tabulate(rows, headers=["Authority Name", "Description", "Daily Average", "Monthy Average", "Yearly Average"]) + "\n") -source_manager = Manager(usage="Handles all source related tasks.") - - -def validate_sources(source_strings): - sources = [] - if not source_strings: - table = [] - for source in source_service.get_all(): - table.append([source.label, source.active, source.description]) - - sys.stdout.write("No source specified choose from below:\n") - sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description'])) - sys.exit(1) - - if 'all' in source_strings: - sources = source_service.get_all() - else: - for source_str in source_strings: - source = source_service.get_by_label(source_str) - - if not source: - sys.stderr.write("Unable to find specified source with label: {0}\n".format(source_str)) - sys.exit(1) - - sources.append(source) - return sources - - -@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') -def sync(source_strings): - source_objs = validate_sources(source_strings) - for source in source_objs: - start_time = time.time() - sys.stdout.write("[+] Staring to sync source: {label}!\n".format(label=source.label)) - - user = user_service.get_by_username('lemur') - - try: - source_service.sync(source, user) - sys.stdout.write( - "[+] Finished syncing source: {label}. Run Time: {time}\n".format( - label=source.label, - time=(time.time() - start_time) - ) - ) - except Exception as e: - current_app.logger.exception(e) - - sys.stdout.write( - "[X] Failed syncing source {label}!\n".format(label=source.label) - ) - - metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label}) - - -@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') -def clean(source_strings): - source_objs = validate_sources(source_strings) - for source in source_objs: - start_time = time.time() - sys.stdout.write("[+] Staring to clean source: {label}!\n".format(label=source.label)) - source_service.clean(source) - sys.stdout.write( - "[+] Finished cleaning source: {label}. Run Time: {time}\n".format( - label=source.label, - time=(time.time() - start_time) - ) - ) - - def main(): manager.add_command("start", LemurServer()) manager.add_command("runserver", Server(host='127.0.0.1', threaded=True)) diff --git a/lemur/notifications/cli.py b/lemur/notifications/cli.py new file mode 100644 index 00000000..b659a03d --- /dev/null +++ b/lemur/notifications/cli.py @@ -0,0 +1,29 @@ +""" +.. module: lemur.notifications.cli + :platform: Unix + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from flask_script import Manager + +from lemur.notifications.messaging import send_expiration_notifications + +manager = Manager(usage="Handles notification related tasks.") + + +@manager.command +def notify(): + """ + Runs Lemur's notification engine, that looks for expired certificates and sends + notifications out to those that have subscribed to them. + + :return: + """ + print("Starting to notify subscribers about expiring certificates!") + count = send_expiration_notifications() + print( + "Finished notifying subscribers about expiring certificates! Sent {count} notifications!".format( + count=count + ) + ) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py new file mode 100644 index 00000000..cfff37b7 --- /dev/null +++ b/lemur/notifications/messaging.py @@ -0,0 +1,102 @@ +""" +.. module: lemur.notifications.messaging + :platform: Unix + + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Kevin Glisson + +""" + +import arrow +from flask import current_app + +from lemur import database, metrics +from lemur.certificates.schemas import certificate_notification_output_schema +from lemur.notifications.models import Notification +from lemur.plugins import plugins +from lemur.plugins.utils import get_plugin_option + + +def send_expiration_notifications(): + """ + This function will check for upcoming certificate expiration, + and send out notification emails at given intervals. + """ + for plugin in plugins.all(plugin_type='notification'): + notifications = database.db.session.query(Notification)\ + .filter(Notification.plugin_name == plugin.slug)\ + .filter(Notification.active == True).all() # noqa + + messages = [] + for n in notifications: + for certificate in n.certificates: + if needs_notification(certificate): + data = certificate_notification_output_schema.dump(certificate).data + messages.append((data, n.options)) + + for data, targets, options in messages: + try: + plugin.send('expiration', data, targets, options) + metrics.send('expiration_notification_sent', 'counter', 1) + except Exception as e: + metrics.send('expiration_notification_failure', 'counter', 1) + current_app.logger.exception(e) + + +def send_rotation_notification(certificate): + """ + Sends a report to certificate owners when their certificate as been + rotated. + + :param certificate: + :return: + """ + plugin = plugins.get(current_app.config.get('LEMUR_DEFAULT_NOTIFICATION_PLUGIN')) + + data = certificate_notification_output_schema.dump(certificate).data + + try: + plugin.send('rotation', data, [data.owner]) + metrics.send('rotation_notification_sent', 'counter', 1) + except Exception as e: + metrics.send('rotation_notification_failure', 'counter', 1) + current_app.logger.exception(e) + + +def needs_notification(certificate): + """ + Determine if notifications for a given certificate should + currently be sent + + :param certificate: + :return: + """ + if not certificate.notify: + return + + if not certificate.notifications: + return + + now = arrow.utcnow() + days = (certificate.not_after - now).days + + for notification in certificate.notifications: + interval = get_plugin_option('interval', notification.options) + unit = get_plugin_option('unit', notification.options) + + if unit == 'weeks': + interval *= 7 + + elif unit == 'months': + interval *= 30 + + elif unit == 'days': # it's nice to be explicit about the base unit + pass + + else: + raise Exception("Invalid base unit for expiration interval: {0}".format(unit)) + + if days == interval: + return certificate diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 9fbbab86..af00d28b 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -1,5 +1,5 @@ """ -.. module: lemur.notifications +.. module: lemur.notifications.service :platform: Unix :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more @@ -8,181 +8,11 @@ .. moduleauthor:: Kevin Glisson """ -import ssl - -import arrow - from flask import current_app + from lemur import database -from lemur.domains.models import Domain -from lemur.notifications.models import Notification from lemur.certificates.models import Certificate - -from lemur.certificates import service as cert_service - -from lemur.plugins.base import plugins - - -def get_options(name, options): - for o in options: - if o.get('name') == name: - return o - - -def _get_message_data(cert): - """ - Parse our the certification information needed for our notification - - :param cert: - :return: - """ - cert_dict = {} - - if cert.user: - cert_dict['creator'] = cert.user.email - - cert_dict['not_after'] = cert.not_after - cert_dict['owner'] = cert.owner - cert_dict['name'] = cert.name - cert_dict['body'] = cert.body - cert_dict['endpoints'] = [{'name': x.name, 'dnsname': x.dnsname} for x in cert.endpoints] - - return cert_dict - - -def _deduplicate(messages): - """ - Take all of the messages that should be sent and provide - a roll up to the same set if the recipients are the same - """ - roll_ups = [] - for data, options in messages: - o = get_options('recipients', options) - targets = o['value'].split(',') - - for m, r, o in roll_ups: - if r == targets: - for cert in m: - if cert['body'] == data['body']: - break - else: - m.append(data) - current_app.logger.info( - "Sending expiration alert about {0} to {1}".format( - data['name'], ",".join(targets))) - break - else: - roll_ups.append(([data], targets, options)) - - return roll_ups - - -def send_expiration_notifications(): - """ - This function will check for upcoming certificate expiration, - and send out notification emails at given intervals. - """ - sent = 0 - - for plugin in plugins.all(plugin_type='notification'): - notifications = database.db.session.query(Notification)\ - .filter(Notification.plugin_name == plugin.slug)\ - .filter(Notification.active == True).all() # noqa - - messages = [] - for n in notifications: - for c in n.certificates: - if _is_eligible_for_notifications(c): - messages.append((_get_message_data(c), n.options)) - - messages = _deduplicate(messages) - - for data, targets, options in messages: - sent += 1 - plugin.send('expiration', data, targets, options) - - current_app.logger.info("Lemur has sent {0} certification notifications".format(sent)) - return sent - - -def _get_domain_certificate(name): - """ - Fetch the SSL certificate currently hosted at a given domain (if any) and - compare it against our all of our know certificates to determine if a new - SSL certificate has already been deployed - - :param name: - :return: - """ - try: - pub_key = ssl.get_server_certificate((name, 443)) - return cert_service.find_duplicates(pub_key.strip()) - except Exception as e: - current_app.logger.info(str(e)) - return [] - - -def _find_superseded(cert): - """ - Here we try to fetch any domain in the certificate to see if we can resolve it - and to try and see if it is currently serving the certificate we are - alerting on. - - :param domains: - :return: - """ - query = database.session_query(Certificate) - ss_list = [] - - # determine what is current host at our domains - for domain in cert.domains: - dups = _get_domain_certificate(domain.name) - for c in dups: - if c.body != cert.body: - ss_list.append(dups) - - current_app.logger.info("Trying to resolve {0}".format(domain.name)) - - # look for other certificates that may not be hosted but cover the same domains - query = query.filter(Certificate.domains.any(Domain.name.in_([x.name for x in cert.domains]))) - query = query.filter(Certificate.active == True) # noqa - query = query.filter(Certificate.not_after >= arrow.utcnow().format('YYYY-MM-DD')) - query = query.filter(Certificate.body != cert.body) - ss_list.extend(query.all()) - return ss_list - - -def _is_eligible_for_notifications(cert): - """ - Determine if notifications for a given certificate should - currently be sent - - :param cert: - :return: - """ - if not cert.notify: - return - - now = arrow.utcnow() - days = (cert.not_after - now.naive).days - - for notification in cert.notifications: - interval = get_options('interval', notification.options)['value'] - unit = get_options('unit', notification.options)['value'] - if unit == 'weeks': - interval *= 7 - - elif unit == 'months': - interval *= 30 - - elif unit == 'days': # it's nice to be explicit about the base unit - pass - - else: - raise Exception("Invalid base unit for expiration interval: {0}".format(unit)) - - if days == interval: - return cert +from lemur.notifications.models import Notification def create_default_expiration_notifications(name, recipients): diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index f2166d78..65e2ee09 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -1,5 +1,5 @@ """ -.. module: lemur.plugins.lemur_aws.aws +.. module: lemur.plugins.lemur_email.plugin :platform: Unix :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more :license: Apache, see LICENSE for more details. @@ -11,14 +11,53 @@ from flask import current_app from flask_mail import Message from lemur.extensions import smtp_mail +from lemur.exceptions import InvalidConfiguration from lemur.plugins.bases import ExpirationNotificationPlugin from lemur.plugins import lemur_email as email - from lemur.plugins.lemur_email.templates.config import env +def render_html(template_name, message): + """ + Renders the html for our email notification. + + :param template_name: + :param message: + :return: + """ + template = env.get_template('{}.html'.format(template_name)) + return template.render(dict(messages=message, hostname=current_app.config.get('LEMUR_HOSTNAME'))) + + +def send_via_ses(subject, body, targets): + """ + Attempts to deliver email notification via SES service. + + :param subject: + :param body: + :param targets: + :return: + """ + msg = Message(subject, recipients=targets) + msg.body = "" # kinda a weird api for sending html emails + msg.html = body + smtp_mail.send(msg) + + +def send_via_smtp(subject, body, targets): + """ + Attempts to deliver email notification via SMTP. + :param subject: + :param body: + :param targets: + :return: + """ + conn = boto.connect_ses() + conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, targets, format='html') + + class EmailNotificationPlugin(ExpirationNotificationPlugin): title = 'Email' slug = 'email-notification' @@ -38,33 +77,23 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): }, ] + def __init__(self, *args, **kwargs): + """Initialize the plugin with the appropriate details.""" + sender = current_app.config.get('LEMUR_EMAIL_SENDER', 'ses').lower() + + if sender not in ['ses', 'smtp']: + raise InvalidConfiguration('Email sender type {0} is not recognized.') + @staticmethod - def send(event_type, message, targets, options, **kwargs): - """ - Configures all Lemur email messaging + def send(template_name, message, targets, **kwargs): + subject = 'Lemur: Expiration Notification' - :param event_type: - :param options: - """ - subject = 'Notification: Lemur' - - if event_type == 'expiration': - subject = 'Notification: SSL Certificate Expiration ' - - # jinja template depending on type - template = env.get_template('{}.html'.format(event_type)) - body = template.render(dict(messages=message, hostname=current_app.config.get('LEMUR_HOSTNAME'))) + body = render_html(template_name, message) s_type = current_app.config.get("LEMUR_EMAIL_SENDER", 'ses').lower() + if s_type == 'ses': - conn = boto.connect_ses() - conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, targets, format='html') + send_via_ses(subject, body, targets) elif s_type == 'smtp': - msg = Message(subject, recipients=targets) - msg.body = "" # kinda a weird api for sending html emails - msg.html = body - smtp_mail.send(msg) - - else: - current_app.logger.error("No mail carrier specified, notification emails were not able to be sent!") + send_via_smtp(subject, body, targets) diff --git a/lemur/plugins/lemur_email/templates/rotation.html b/lemur/plugins/lemur_email/templates/rotation.html index ef9f2762..8e84a2ea 100644 --- a/lemur/plugins/lemur_email/templates/rotation.html +++ b/lemur/plugins/lemur_email/templates/rotation.html @@ -34,7 +34,7 @@ - @@ -43,7 +43,7 @@ @@ -56,7 +56,7 @@
- Your certificate(s) are being rotated! + Your certificate(s) have been rotated!
- @@ -71,11 +71,51 @@ + + + + + + + + +
Hi, +
This is a Lemur certificate rotation notice. Your certificate has been re-issued and re-provisioned. +
+ Impacted Certificate: +
+ + + {% for message in messages %} + + + + + + {% if not loop.last %} + + + + {% endif %} + {% endfor %} + +
+ {{ message.name }} +
+ + {{ message.endpoints | length }} Endpoints +
{{ message.owner }} +
{{ message.not_after | time }} + Details + +
+
+ Impacted Endpoints:
-
This is a Lemur certificate rotation notice. This a purely informational notice. The following certificates will be rotated on any associated endpoints. diff --git a/lemur/plugins/utils.py b/lemur/plugins/utils.py new file mode 100644 index 00000000..2e727a00 --- /dev/null +++ b/lemur/plugins/utils.py @@ -0,0 +1,21 @@ +""" +.. module: lemur.plugins.utils + :platform: Unix + + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Kevin Glisson + +""" + + +def get_plugin_option(name, options): + """ + Retrieve option name from options dict. + :param options: + :return: + """ + for o in options: + if o.get('name') == name: + return o['value'] diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py new file mode 100644 index 00000000..bd72b343 --- /dev/null +++ b/lemur/sources/cli.py @@ -0,0 +1,88 @@ +""" +.. module: lemur.sources.cli + :platform: Unix + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +import sys +import time + +from tabulate import tabulate + +from flask_script import Manager + +from flask import current_app + +from lemur.extensions import metrics +from lemur.sources import service as source_service +from lemur.users import service as user_service + +manager = Manager(usage="Handles all source related tasks.") + + +def validate_sources(source_strings): + sources = [] + if not source_strings: + table = [] + for source in source_service.get_all(): + table.append([source.label, source.active, source.description]) + + print("No source specified choose from below:") + print(tabulate(table, headers=['Label', 'Active', 'Description'])) + sys.exit(1) + + if 'all' in source_strings: + sources = source_service.get_all() + else: + for source_str in source_strings: + source = source_service.get_by_label(source_str) + + if not source: + print("Unable to find specified source with label: {0}".format(source_str)) + sys.exit(1) + + sources.append(source) + return sources + + +@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') +def sync(source_strings): + source_objs = validate_sources(source_strings) + for source in source_objs: + start_time = time.time() + print("[+] Staring to sync source: {label}!\n".format(label=source.label)) + + user = user_service.get_by_username('lemur') + + try: + source_service.sync(source, user) + print( + "[+] Finished syncing source: {label}. Run Time: {time}".format( + label=source.label, + time=(time.time() - start_time) + ) + ) + except Exception as e: + current_app.logger.exception(e) + + print( + "[X] Failed syncing source {label}!\n".format(label=source.label) + ) + + metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label}) + + +@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') +def clean(source_strings): + source_objs = validate_sources(source_strings) + for source in source_objs: + start_time = time.time() + print("[+] Staring to clean source: {label}!\n".format(label=source.label)) + source_service.clean(source) + print( + "[+] Finished cleaning source: {label}. Run Time: {time}\n".format( + label=source.label, + time=(time.time() - start_time) + ) + ) diff --git a/lemur/analyze/__init__.py b/lemur/tests/test_deployment.py similarity index 100% rename from lemur/analyze/__init__.py rename to lemur/tests/test_deployment.py diff --git a/lemur/tests/test_endpoints.py b/lemur/tests/test_endpoints.py index a20e866e..d68681d7 100644 --- a/lemur/tests/test_endpoints.py +++ b/lemur/tests/test_endpoints.py @@ -8,7 +8,7 @@ from .vectors import VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN def test_rotate_certificate(client, source_plugin): - from lemur.endpoints.service import rotate_certificate + from lemur.deployment.service import rotate_certificate new_certificate = CertificateFactory() endpoint = EndpointFactory() diff --git a/lemur/tests/test_messaging.py b/lemur/tests/test_messaging.py new file mode 100644 index 00000000..4bafce2d --- /dev/null +++ b/lemur/tests/test_messaging.py @@ -0,0 +1,29 @@ +import pytest +from freezegun import freeze_time + +from datetime import timedelta + + +def test_needs_notification(app, certificate, notification): + from lemur.notifications.messaging import needs_notification + assert not needs_notification(certificate) + + with pytest.raises(Exception): + notification.options = [{'name': 'interval', 'value': 10}, {'name': 'unit', 'value': 'min'}] + certificate.notifications.append(notification) + needs_notification(certificate) + + certificate.notifications[0].options = [{'name': 'interval', 'value': 10}, {'name': 'unit', 'value': 'days'}] + assert not needs_notification(certificate) + + delta = certificate.not_after - timedelta(days=10) + with freeze_time(delta.datetime): + assert needs_notification(certificate) + + +def test_send_expiration_notification(): + assert False + + +def test_send_rotation_notification(): + assert False