Certificate rotation enhancements (#570)

lemur/analyze/service.py

@@ -1,64 +0,0 @@
-"""
-.. module: lemur.analyze.service
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-"""
-# def analyze(endpoints, truststores):
-#    results = {"headings": ["Endpoint"],
-#               "results": [],
-#               "time": datetime.now().strftime("#Y%m%d %H:%M:%S")}
-#
-#    for store in truststores:
-#        results['headings'].append(os.path.basename(store))
-#
-#    for endpoint in endpoints:
-#        result_row = [endpoint]
-#        for store in truststores:
-#            result = {'details': []}
-#
-#            tests = []
-#            for region, ip in REGIONS.items():
-#                try:
-#                    domain = dns.name.from_text(endpoint)
-#                    if not domain.is_absolute():
-#                        domain = domain.concatenate(dns.name.root)
-#
-#                    my_resolver = dns.resolver.Resolver()
-#                    my_resolver.nameservers = [ip]
-#                    answer = my_resolver.query(domain)
-#
-#                    #force the testing of regional enpoints by changing the dns server
-#                    response = requests.get('https://' + str(answer[0]), verify=store)
-#                    tests.append('pass')
-#                    result['details'].append("{}: SSL testing completed without errors".format(region))
-#
-#                except SSLError as e:
-#                    log.debug(e)
-#                    if 'hostname' in str(e):
-#                        tests.append('pass')
-#                        result['details'].append(
-#                              "{}: This test passed ssl negotiation but failed hostname verification because \
-#                                   the hostname is not included in the certificate".format(region))
-#                    elif 'certificate verify failed' in str(e):
-#                        tests.append('fail')
-#                        result['details'].append("{}: This test failed to verify the SSL certificate".format(region))
-#                    else:
-#                        tests.append('fail')
-#                        result['details'].append("{}: {}".format(region, str(e)))
-#
-#                except Exception as e:
-#                    log.debug(e)
-#                    tests.append('fail')
-#                    result['details'].append("{}: {}".format(region, str(e)))
-#
-#            #any failing tests fails the whole endpoint
-#            if 'fail' in tests:
-#                result['test'] = 'fail'
-#            else:
-#                result['test'] = 'pass'
-#
-#            result_row.append(result)
-#        results['results'].append(result_row)
-#    return results
-#

lemur/certificates/cli.py

@@ -0,0 +1,167 @@
+"""
+.. module: lemur.certificate.cli
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+import sys
+
+from flask import current_app
+
+from flask_script import Manager
+
+from lemur import database
+from lemur.deployment.service import rotate_certificate
+from lemur.notifications.messaging import send_rotation_notification
+from lemur.certificates.service import reissue_certificate, get_certificate_primitives, get_all_pending_rotation, get_by_name, get_all_certs
+
+from lemur.certificates.verify import verify_string
+
+manager = Manager(usage="Handles all certificate related tasks.")
+
+
+def reissue_and_rotate(old_certificate, new_certificate=None, commit=False, message=False):
+    if not new_certificate:
+        # we don't want to re-issue if it's already been replaced
+        if not old_certificate.replaced:
+            details = get_certificate_primitives(old_certificate)
+            print_certificate_details(details)
+
+            if commit:
+                new_certificate = reissue_certificate(old_certificate, replace=True)
+                print("[+] Issued new certificate named: {0}".format(new_certificate.name))
+
+            print("[+] Done!")
+
+        else:
+            new_certificate = old_certificate.replaced
+            print("[!] Certificate has been replaced by: {0}".format(old_certificate.replaced.name))
+
+    if len(old_certificate.endpoints) > 0:
+        for endpoint in old_certificate.endpoints:
+            print(
+                "[+] Certificate deployed on endpoint: name:{name} dnsname:{dnsname} port:{port} type:{type}".format(
+                    name=endpoint.name,
+                    dnsname=endpoint.dnsname,
+                    port=endpoint.port,
+                    type=endpoint.type
+                )
+            )
+            print("[+] Rotating certificate from: {0} to: {1}".format(old_certificate.name, new_certificate.name))
+
+            if commit:
+                rotate_certificate(endpoint, new_certificate)
+
+            print("[+] Done!")
+
+    if message:
+        send_rotation_notification(old_certificate)
+
+
+def print_certificate_details(details):
+    """
+    Print the certificate details with formatting.
+    :param details:
+    :return:
+    """
+    print("[+] Re-issuing certificate with the following details: ")
+    print(
+        "[+] Common Name: {common_name}\n"
+        "[+] Subject Alternate Names: {sans}\n"
+        "[+] Authority: {authority_name}\n"
+        "[+] Validity Start: {validity_start}\n"
+        "[+] Validity End: {validity_end}\n"
+        "[+] Organization: {organization}\n"
+        "[+] Organizational Unit: {organizational_unit}\n"
+        "[+] Country: {country}\n"
+        "[+] State: {state}\n"
+        "[+] Location: {location}\n".format(
+            common_name=details['common_name'],
+            sans=",".join(x['value'] for x in details['extensions']['sub_alt_names']['names']),
+            authority_name=details['authority'].name,
+            validity_start=details['validity_start'].isoformat(),
+            validity_end=details['validity_end'].isoformat(),
+            organization=details['organization'],
+            organizational_unit=details['organizational_unit'],
+            country=details['country'],
+            state=details['state'],
+            location=details['location']
+        )
+    )
+
+
+@manager.command
+def rotate(new_certificate_name=False, old_certificate_name=False, message=False, commit=False):
+    new_cert = old_cert = None
+
+    if commit:
+        print("[!] Running in COMMIT mode.")
+
+    if old_certificate_name:
+        old_cert = get_by_name(old_certificate_name)
+
+        if not old_cert:
+            print("[-] No certificate found with name: {0}".format(old_certificate_name))
+            sys.exit(1)
+
+    if new_certificate_name:
+        new_cert = get_by_name(new_certificate_name)
+
+        if not new_cert:
+            print("[-] No certificate found with name: {0}".format(old_certificate_name))
+            sys.exit(1)
+
+    if old_cert and new_cert:
+        reissue_and_rotate(old_cert, new_certificate=new_cert, commit=commit, message=message)
+    else:
+        for certificate in get_all_pending_rotation():
+            reissue_and_rotate(certificate, commit=commit, message=message)
+
+
+@manager.command
+def reissue(old_certificate_name, commit=False):
+    from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives
+
+    old_cert = get_by_name(old_certificate_name)
+
+    if not old_cert:
+        print("[-] No certificate found with name: {0}".format(old_certificate_name))
+        sys.exit(1)
+
+    if commit:
+        print("[!] Running in COMMIT mode.")
+
+    details = get_certificate_primitives(old_cert)
+    print_certificate_details(details)
+
+    if commit:
+        new_cert = reissue_certificate(old_cert, replace=True)
+        print("[+] Issued new certificate named: {0}".format(new_cert.name))
+
+    print("[+] Done!")
+
+
+@manager.command
+def check_revoked():
+    """
+    Function attempts to update Lemur's internal cache with revoked
+    certificates. This is called periodically by Lemur. It checks both
+    CRLs and OCSP to see if a certificate is revoked. If Lemur is unable
+    encounters an issue with verification it marks the certificate status
+    as `unknown`.
+    """
+    for cert in get_all_certs():
+        try:
+            if cert.chain:
+                status = verify_string(cert.body, cert.chain)
+            else:
+                status = verify_string(cert.body, "")
+
+            cert.status = 'valid' if status else 'invalid'
+
+        except Exception as e:
+            current_app.logger.exception(e)
+            cert.status = 'unknown'
+
+        database.update(cert)

lemur/certificates/exceptions.py

@@ -1,88 +0,0 @@
-"""
-.. module: lemur.certificates.exceptions
-    :synopsis: Defines all monterey specific exceptions
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-"""
-from flask import current_app
-from lemur.exceptions import LemurException
-
-
-class UnknownAuthority(LemurException):
-    def __init__(self, authority):
-        self.code = 404
-        self.authority = authority
-        self.data = {"message": "The authority specified '{}' is not a valid authority".format(self.authority)}
-
-        current_app.logger.warning(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class InsufficientDomains(LemurException):
-    def __init__(self):
-        self.code = 400
-        self.data = {"message": "Need at least one domain specified in order create a certificate"}
-
-        current_app.logger.warning(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class InvalidCertificate(LemurException):
-    def __init__(self):
-        self.code = 400
-        self.data = {"message": "Need at least one domain specified in order create a certificate"}
-
-        current_app.logger.warning(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class UnableToCreateCSR(LemurException):
-    def __init__(self):
-        self.code = 500
-        self.data = {"message": "Unable to generate CSR"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class UnableToCreatePrivateKey(LemurException):
-    def __init__(self):
-        self.code = 500
-        self.data = {"message": "Unable to generate Private Key"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class MissingFiles(LemurException):
-    def __init__(self, path):
-        self.code = 500
-        self.path = path
-        self.data = {"path": self.path, "message": "Expecting missing files"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])
-
-
-class NoPersistanceFound(LemurException):
-    def __init__(self):
-        self.code = 500
-        self.data = {"code": 500, "message": "No peristence method found, Lemur cannot persist sensitive information"}
-
-        current_app.logger.error(self)
-
-    def __str__(self):
-        return repr(self.data['message'])

lemur/certificates/models.py

@@ -66,6 +66,8 @@ class Certificate(db.Model):
     bits = Column(Integer())
     san = Column(String(1024))  # TODO this should be migrated to boolean
 
+    rotation = Column(Boolean)
+
     user_id = Column(Integer, ForeignKey('users.id'))
     authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
     root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))

lemur/certificates/schemas.py

@@ -201,9 +201,20 @@ class CertificateExportInputSchema(LemurInputSchema):
     plugin = fields.Nested(PluginInputSchema)
 
 
+class CertificateNotificationOutputSchema(LemurOutputSchema):
+    description = fields.String()
+    issuer = fields.String()
+    name = fields.String()
+    owner = fields.Email()
+    user = fields.Nested(UserNestedOutputSchema)
+    replaces = fields.Nested(CertificateNestedOutputSchema, many=True)
+    endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
+
+
 certificate_input_schema = CertificateInputSchema()
 certificate_output_schema = CertificateOutputSchema()
 certificates_output_schema = CertificateOutputSchema(many=True)
 certificate_upload_input_schema = CertificateUploadInputSchema()
 certificate_export_input_schema = CertificateExportInputSchema()
 certificate_edit_input_schema = CertificateEditInputSchema()
+certificate_notification_output_schema = CertificateNotificationOutputSchema()

lemur/certificates/service.py

@@ -1,14 +1,15 @@
 """
-.. module: service
+.. module: lemur.certificate.service
     :platform: Unix
     :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+from datetime import timedelta
 
-from sqlalchemy import func, or_
 from flask import current_app
+from sqlalchemy import func, or_
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -17,15 +18,15 @@ from cryptography.hazmat.primitives import hashes, serialization
 from lemur import database
 from lemur.extensions import metrics
 from lemur.plugins.base import plugins
-from lemur.certificates.models import Certificate
 from lemur.common.utils import generate_private_key
 
-from lemur.destinations.models import Destination
-from lemur.notifications.models import Notification
-from lemur.authorities.models import Authority
-from lemur.domains.models import Domain
-
 from lemur.roles.models import Role
+from lemur.domains.models import Domain
+from lemur.authorities.models import Authority
+from lemur.destinations.models import Destination
+from lemur.certificates.models import Certificate
+from lemur.notifications.models import Notification
+
 from lemur.roles import service as role_service
 
 
@@ -77,6 +78,24 @@ def get_by_source(source_label):
     return Certificate.query.filter(Certificate.sources.any(label=source_label))
 
 
+def get_all_pending_rotation():
+    """
+    Retrieves all certificates that need to be rotated.
+
+    Must be X days from expiration, uses `LEMUR_DEFAULT_ROTATION_INTERVAL`
+    to determine how many days from expiration the certificate must be
+    for rotation to be pending.
+
+    :return:
+    """
+    now = arrow.utcnow()
+    interval = current_app.config.get('LEMUR_DEFAULT_ROTATION_INTERVAL', 30)
+    end = now + timedelta(days=interval)
+
+    return Certificate.query.filter(Certificate.rotation == True)\
+        .filter(Certificate.not_after <= end.format('YYYY-MM-DD')).all()  # noqa
+
+
 def find_duplicates(cert):
     """
     Finds certificates that already exist within Lemur. We do this by looking for
@@ -527,9 +546,9 @@ def reissue_certificate(certificate, replace=None, user=None):
     else:
         primitives['creator'] = user
 
+    if replace:
+        primitives['replaces'] = certificate
+
     new_cert = create(**primitives)
 
-    if replace:
-        certificate.notify = False
-
     return new_cert

lemur/deployment/service.py

@@ -0,0 +1,19 @@
+from flask import current_app
+from lemur.extensions import metrics
+
+
+def rotate_certificate(endpoint, new_cert):
+    """
+    Rotates a certificate on a given endpoint.
+
+    :param endpoint:
+    :param new_cert:
+    :return:
+    """
+    try:
+        endpoint.source.plugin.update_endpoint(endpoint, new_cert)
+        endpoint.certificate = new_cert
+        metrics.send('rotation_success', 'counter', 1, metric_tags={'endpoint': endpoint.name})
+    except Exception as e:
+        metrics.send('rotation_failure', 'counter', 1, metric_tags={'endpoint': endpoint.name})
+        current_app.logger.exception(e)

lemur/endpoints/service.py

@@ -10,12 +10,12 @@
 """
 from flask import current_app
 
-from lemur import database
-from lemur.extensions import metrics
-from lemur.endpoints.models import Endpoint, Policy, Cipher
-
 from sqlalchemy import func
 
+from lemur import database
+from lemur.endpoints.models import Endpoint, Policy, Cipher
+from lemur.extensions import metrics
+
 
 def get_all():
     """

lemur/manage.py

@@ -7,7 +7,6 @@ from collections import Counter
 import os
 import sys
 import base64
-import time
 import requests
 import json
 
@@ -21,16 +20,15 @@ from flask_script import Manager, Command, Option, prompt_pass
 from flask_migrate import Migrate, MigrateCommand, stamp
 from flask_script.commands import ShowUrls, Clean, Server
 
+from lemur.sources.cli import manager as source_manager
+from lemur.certificates.cli import manager as certificate_manager
+
 from lemur import database
-from lemur.extensions import metrics
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
-from lemur.certificates import service as cert_service
 from lemur.authorities import service as authority_service
 from lemur.notifications import service as notification_service
 
-from lemur.certificates.verify import verify_string
-from lemur.sources import service as source_service
 
 from lemur.common.utils import validate_conf
 
@@ -148,31 +146,6 @@ def drop_all():
     database.db.drop_all()
 
 
-@manager.command
-def check_revoked():
-    """
-    Function attempts to update Lemur's internal cache with revoked
-    certificates. This is called periodically by Lemur. It checks both
-    CRLs and OCSP to see if a certificate is revoked. If Lemur is unable
-    encounters an issue with verification it marks the certificate status
-    as `unknown`.
-    """
-    for cert in cert_service.get_all_certs():
-        try:
-            if cert.chain:
-                status = verify_string(cert.body, cert.chain)
-            else:
-                status = verify_string(cert.body, "")
-
-            cert.status = 'valid' if status else 'invalid'
-
-        except Exception as e:
-            current_app.logger.exception(e)
-            cert.status = 'unknown'
-
-        database.update(cert)
-
-
 @manager.shell
 def make_shell_context():
     """
@@ -201,23 +174,6 @@ def generate_settings():
     return output
 
 
-@manager.command
-def notify():
-    """
-    Runs Lemur's notification engine, that looks for expired certificates and sends
-    notifications out to those that bave subscribed to them.
-
-    :return:
-    """
-    sys.stdout.write("Starting to notify subscribers about expiring certificates!\n")
-    count = notification_service.send_expiration_notifications()
-    sys.stdout.write(
-        "Finished notifying subscribers about expiring certificates! Sent {count} notifications!\n".format(
-            count=count
-        )
-    )
-
-
 class InitializeApp(Command):
     """
     This command will bootstrap our database with any destinations as
@@ -521,117 +477,6 @@ def unlock(path=None):
     sys.stdout.write("[+] Keys have been unencrypted!\n")
 
 
-def print_certificate_details(details):
-    """
-    Print the certificate details with formatting.
-    :param details:
-    :return:
-    """
-    sys.stdout.write("[+] Re-issuing certificate with the following details:  \n")
-    sys.stdout.write(
-        "[+] Common Name: {common_name}\n"
-        "[+] Subject Alternate Names: {sans}\n"
-        "[+] Authority: {authority_name}\n"
-        "[+] Validity Start: {validity_start}\n"
-        "[+] Validity End: {validity_end}\n"
-        "[+] Organization: {organization}\n"
-        "[+] Organizational Unit: {organizational_unit}\n"
-        "[+] Country: {country}\n"
-        "[+] State: {state}\n"
-        "[+] Location: {location}\n".format(
-            common_name=details['common_name'],
-            sans=",".join(x['value'] for x in details['extensions']['sub_alt_names']['names']),
-            authority_name=details['authority'].name,
-            validity_start=details['validity_start'].isoformat(),
-            validity_end=details['validity_end'].isoformat(),
-            organization=details['organization'],
-            organizational_unit=details['organizational_unit'],
-            country=details['country'],
-            state=details['state'],
-            location=details['location']
-        )
-    )
-
-
-certificate_manager = Manager(usage="Handles all certificate related tasks.")
-
-
-@certificate_manager.command
-def rotate(new_certificate_name, old_certificate_name, commit=False):
-    from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives
-    from lemur.endpoints.service import rotate_certificate
-
-    old_cert = get_by_name(old_certificate_name)
-
-    if not old_cert:
-        sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name))
-        sys.exit(1)
-
-    if new_certificate_name:
-        new_cert = get_by_name(new_certificate_name)
-
-        if not new_cert:
-            sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name))
-            sys.exit(1)
-
-    if commit:
-        sys.stdout.write("[!] Running in COMMIT mode.\n")
-
-    if not new_certificate_name:
-        sys.stdout.write("[!] No new certificate provided. Attempting to re-issue old certificate: {0}.\n".format(old_certificate_name))
-
-        details = get_certificate_primitives(old_cert)
-        print_certificate_details(details)
-
-        if commit:
-            new_cert = reissue_certificate(old_cert, replace=True)
-            sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name))
-
-        sys.stdout.write("[+] Done! \n")
-
-    if len(old_cert.endpoints) > 0:
-        for endpoint in old_cert.endpoints:
-            sys.stdout.write(
-                "[+] Certificate deployed on endpoint: name:{name} dnsname:{dnsname} port:{port} type:{type}\n".format(
-                    name=endpoint.name,
-                    dnsname=endpoint.dnsname,
-                    port=endpoint.port,
-                    type=endpoint.type
-                )
-            )
-            sys.stdout.write("[+] Rotating certificate from: {0} to: {1}\n".format(old_certificate_name, new_cert.name))
-
-            if commit:
-                rotate_certificate(endpoint, new_cert)
-
-            sys.stdout.write("[+] Done! \n")
-    else:
-        sys.stdout.write("[!] Certificate not found on any existing endpoints. Nothing to rotate.\n")
-
-
-@certificate_manager.command
-def reissue(old_certificate_name, commit=False):
-    from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives
-
-    old_cert = get_by_name(old_certificate_name)
-
-    if not old_cert:
-        sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name))
-        sys.exit(1)
-
-    if commit:
-        sys.stdout.write("[!] Running in COMMIT mode.\n")
-
-    details = get_certificate_primitives(old_cert)
-    print_certificate_details(details)
-
-    if commit:
-        new_cert = reissue_certificate(old_cert, replace=True)
-        sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name))
-
-    sys.stdout.write("[+] Done! \n")
-
-
 @manager.command
 def publish_verisign_units():
     """
@@ -771,76 +616,6 @@ class Report(Command):
         sys.stdout.write(tabulate(rows, headers=["Authority Name", "Description", "Daily Average", "Monthy Average", "Yearly Average"]) + "\n")
 
 
-source_manager = Manager(usage="Handles all source related tasks.")
-
-
-def validate_sources(source_strings):
-    sources = []
-    if not source_strings:
-        table = []
-        for source in source_service.get_all():
-            table.append([source.label, source.active, source.description])
-
-        sys.stdout.write("No source specified choose from below:\n")
-        sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description']))
-        sys.exit(1)
-
-    if 'all' in source_strings:
-        sources = source_service.get_all()
-    else:
-        for source_str in source_strings:
-            source = source_service.get_by_label(source_str)
-
-            if not source:
-                sys.stderr.write("Unable to find specified source with label: {0}\n".format(source_str))
-                sys.exit(1)
-
-            sources.append(source)
-    return sources
-
-
-@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
-def sync(source_strings):
-    source_objs = validate_sources(source_strings)
-    for source in source_objs:
-        start_time = time.time()
-        sys.stdout.write("[+] Staring to sync source: {label}!\n".format(label=source.label))
-
-        user = user_service.get_by_username('lemur')
-
-        try:
-            source_service.sync(source, user)
-            sys.stdout.write(
-                "[+] Finished syncing source: {label}. Run Time: {time}\n".format(
-                    label=source.label,
-                    time=(time.time() - start_time)
-                )
-            )
-        except Exception as e:
-            current_app.logger.exception(e)
-
-            sys.stdout.write(
-                "[X] Failed syncing source {label}!\n".format(label=source.label)
-            )
-
-            metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label})
-
-
-@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
-def clean(source_strings):
-    source_objs = validate_sources(source_strings)
-    for source in source_objs:
-        start_time = time.time()
-        sys.stdout.write("[+] Staring to clean source: {label}!\n".format(label=source.label))
-        source_service.clean(source)
-        sys.stdout.write(
-            "[+] Finished cleaning source: {label}. Run Time: {time}\n".format(
-                label=source.label,
-                time=(time.time() - start_time)
-            )
-        )
-
-
 def main():
     manager.add_command("start", LemurServer())
     manager.add_command("runserver", Server(host='127.0.0.1', threaded=True))

lemur/notifications/cli.py

@@ -0,0 +1,29 @@
+"""
+.. module: lemur.notifications.cli
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask_script import Manager
+
+from lemur.notifications.messaging import send_expiration_notifications
+
+manager = Manager(usage="Handles notification related tasks.")
+
+
+@manager.command
+def notify():
+    """
+    Runs Lemur's notification engine, that looks for expired certificates and sends
+    notifications out to those that have subscribed to them.
+
+    :return:
+    """
+    print("Starting to notify subscribers about expiring certificates!")
+    count = send_expiration_notifications()
+    print(
+        "Finished notifying subscribers about expiring certificates! Sent {count} notifications!".format(
+            count=count
+        )
+    )

lemur/notifications/messaging.py

@@ -0,0 +1,102 @@
+"""
+.. module: lemur.notifications.messaging
+    :platform: Unix
+
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+
+"""
+
+import arrow
+from flask import current_app
+
+from lemur import database, metrics
+from lemur.certificates.schemas import certificate_notification_output_schema
+from lemur.notifications.models import Notification
+from lemur.plugins import plugins
+from lemur.plugins.utils import get_plugin_option
+
+
+def send_expiration_notifications():
+    """
+    This function will check for upcoming certificate expiration,
+    and send out notification emails at given intervals.
+    """
+    for plugin in plugins.all(plugin_type='notification'):
+        notifications = database.db.session.query(Notification)\
+            .filter(Notification.plugin_name == plugin.slug)\
+            .filter(Notification.active == True).all()  # noqa
+
+        messages = []
+        for n in notifications:
+            for certificate in n.certificates:
+                if needs_notification(certificate):
+                    data = certificate_notification_output_schema.dump(certificate).data
+                    messages.append((data, n.options))
+
+        for data, targets, options in messages:
+            try:
+                plugin.send('expiration', data, targets, options)
+                metrics.send('expiration_notification_sent', 'counter', 1)
+            except Exception as e:
+                metrics.send('expiration_notification_failure', 'counter', 1)
+                current_app.logger.exception(e)
+
+
+def send_rotation_notification(certificate):
+    """
+    Sends a report to certificate owners when their certificate as been
+    rotated.
+
+    :param certificate:
+    :return:
+    """
+    plugin = plugins.get(current_app.config.get('LEMUR_DEFAULT_NOTIFICATION_PLUGIN'))
+
+    data = certificate_notification_output_schema.dump(certificate).data
+
+    try:
+        plugin.send('rotation', data, [data.owner])
+        metrics.send('rotation_notification_sent', 'counter', 1)
+    except Exception as e:
+        metrics.send('rotation_notification_failure', 'counter', 1)
+        current_app.logger.exception(e)
+
+
+def needs_notification(certificate):
+    """
+    Determine if notifications for a given certificate should
+    currently be sent
+
+    :param certificate:
+    :return:
+    """
+    if not certificate.notify:
+        return
+
+    if not certificate.notifications:
+        return
+
+    now = arrow.utcnow()
+    days = (certificate.not_after - now).days
+
+    for notification in certificate.notifications:
+        interval = get_plugin_option('interval', notification.options)
+        unit = get_plugin_option('unit', notification.options)
+
+        if unit == 'weeks':
+            interval *= 7
+
+        elif unit == 'months':
+            interval *= 30
+
+        elif unit == 'days':  # it's nice to be explicit about the base unit
+            pass
+
+        else:
+            raise Exception("Invalid base unit for expiration interval: {0}".format(unit))
+
+        if days == interval:
+            return certificate

lemur/notifications/service.py

@@ -1,5 +1,5 @@
 """
-.. module: lemur.notifications
+.. module: lemur.notifications.service
     :platform: Unix
 
     :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
@@ -8,181 +8,11 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-import ssl
-
-import arrow
-
 from flask import current_app
+
 from lemur import database
-from lemur.domains.models import Domain
-from lemur.notifications.models import Notification
 from lemur.certificates.models import Certificate
-
+from lemur.notifications.models import Notification
-from lemur.certificates import service as cert_service
-
-from lemur.plugins.base import plugins
-
-
-def get_options(name, options):
-    for o in options:
-        if o.get('name') == name:
-            return o
-
-
-def _get_message_data(cert):
-    """
-    Parse our the certification information needed for our notification
-
-    :param cert:
-    :return:
-    """
-    cert_dict = {}
-
-    if cert.user:
-        cert_dict['creator'] = cert.user.email
-
-    cert_dict['not_after'] = cert.not_after
-    cert_dict['owner'] = cert.owner
-    cert_dict['name'] = cert.name
-    cert_dict['body'] = cert.body
-    cert_dict['endpoints'] = [{'name': x.name, 'dnsname': x.dnsname} for x in cert.endpoints]
-
-    return cert_dict
-
-
-def _deduplicate(messages):
-    """
-    Take all of the messages that should be sent and provide
-    a roll up to the same set if the recipients are the same
-    """
-    roll_ups = []
-    for data, options in messages:
-        o = get_options('recipients', options)
-        targets = o['value'].split(',')
-
-        for m, r, o in roll_ups:
-            if r == targets:
-                for cert in m:
-                    if cert['body'] == data['body']:
-                        break
-                else:
-                    m.append(data)
-                    current_app.logger.info(
-                        "Sending expiration alert about {0} to {1}".format(
-                            data['name'], ",".join(targets)))
-                break
-        else:
-            roll_ups.append(([data], targets, options))
-
-    return roll_ups
-
-
-def send_expiration_notifications():
-    """
-    This function will check for upcoming certificate expiration,
-    and send out notification emails at given intervals.
-    """
-    sent = 0
-
-    for plugin in plugins.all(plugin_type='notification'):
-        notifications = database.db.session.query(Notification)\
-            .filter(Notification.plugin_name == plugin.slug)\
-            .filter(Notification.active == True).all()  # noqa
-
-        messages = []
-        for n in notifications:
-            for c in n.certificates:
-                if _is_eligible_for_notifications(c):
-                    messages.append((_get_message_data(c), n.options))
-
-        messages = _deduplicate(messages)
-
-        for data, targets, options in messages:
-            sent += 1
-            plugin.send('expiration', data, targets, options)
-
-        current_app.logger.info("Lemur has sent {0} certification notifications".format(sent))
-    return sent
-
-
-def _get_domain_certificate(name):
-    """
-    Fetch the SSL certificate currently hosted at a given domain (if any) and
-    compare it against our all of our know certificates to determine if a new
-    SSL certificate has already been deployed
-
-    :param name:
-    :return:
-    """
-    try:
-        pub_key = ssl.get_server_certificate((name, 443))
-        return cert_service.find_duplicates(pub_key.strip())
-    except Exception as e:
-        current_app.logger.info(str(e))
-        return []
-
-
-def _find_superseded(cert):
-    """
-    Here we try to fetch any domain in the certificate to see if we can resolve it
-    and to try and see if it is currently serving the certificate we are
-    alerting on.
-
-    :param domains:
-    :return:
-    """
-    query = database.session_query(Certificate)
-    ss_list = []
-
-    # determine what is current host at our domains
-    for domain in cert.domains:
-        dups = _get_domain_certificate(domain.name)
-        for c in dups:
-            if c.body != cert.body:
-                ss_list.append(dups)
-
-        current_app.logger.info("Trying to resolve {0}".format(domain.name))
-
-    # look for other certificates that may not be hosted but cover the same domains
-    query = query.filter(Certificate.domains.any(Domain.name.in_([x.name for x in cert.domains])))
-    query = query.filter(Certificate.active == True)  # noqa
-    query = query.filter(Certificate.not_after >= arrow.utcnow().format('YYYY-MM-DD'))
-    query = query.filter(Certificate.body != cert.body)
-    ss_list.extend(query.all())
-    return ss_list
-
-
-def _is_eligible_for_notifications(cert):
-    """
-    Determine if notifications for a given certificate should
-    currently be sent
-
-    :param cert:
-    :return:
-    """
-    if not cert.notify:
-        return
-
-    now = arrow.utcnow()
-    days = (cert.not_after - now.naive).days
-
-    for notification in cert.notifications:
-        interval = get_options('interval', notification.options)['value']
-        unit = get_options('unit', notification.options)['value']
-        if unit == 'weeks':
-            interval *= 7
-
-        elif unit == 'months':
-            interval *= 30
-
-        elif unit == 'days':  # it's nice to be explicit about the base unit
-            pass
-
-        else:
-            raise Exception("Invalid base unit for expiration interval: {0}".format(unit))
-
-        if days == interval:
-            return cert
 
 
 def create_default_expiration_notifications(name, recipients):

lemur/plugins/lemur_email/plugin.py

@@ -1,5 +1,5 @@
 """
-.. module: lemur.plugins.lemur_aws.aws
+.. module: lemur.plugins.lemur_email.plugin
     :platform: Unix
     :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
     :license: Apache, see LICENSE for more details.
@@ -11,14 +11,53 @@ from flask import current_app
 from flask_mail import Message
 
 from lemur.extensions import smtp_mail
+from lemur.exceptions import InvalidConfiguration
 
 from lemur.plugins.bases import ExpirationNotificationPlugin
 from lemur.plugins import lemur_email as email
 
-
 from lemur.plugins.lemur_email.templates.config import env
 
 
+def render_html(template_name, message):
+    """
+    Renders the html for our email notification.
+
+    :param template_name:
+    :param message:
+    :return:
+    """
+    template = env.get_template('{}.html'.format(template_name))
+    return template.render(dict(messages=message, hostname=current_app.config.get('LEMUR_HOSTNAME')))
+
+
+def send_via_ses(subject, body, targets):
+    """
+    Attempts to deliver email notification via SES service.
+
+    :param subject:
+    :param body:
+    :param targets:
+    :return:
+    """
+    msg = Message(subject, recipients=targets)
+    msg.body = ""  # kinda a weird api for sending html emails
+    msg.html = body
+    smtp_mail.send(msg)
+
+
+def send_via_smtp(subject, body, targets):
+    """
+    Attempts to deliver email notification via SMTP.
+    :param subject:
+    :param body:
+    :param targets:
+    :return:
+    """
+    conn = boto.connect_ses()
+    conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, targets, format='html')
+
+
 class EmailNotificationPlugin(ExpirationNotificationPlugin):
     title = 'Email'
     slug = 'email-notification'
@@ -38,33 +77,23 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
         },
     ]
 
+    def __init__(self, *args, **kwargs):
+        """Initialize the plugin with the appropriate details."""
+        sender = current_app.config.get('LEMUR_EMAIL_SENDER', 'ses').lower()
+
+        if sender not in ['ses', 'smtp']:
+            raise InvalidConfiguration('Email sender type {0} is not recognized.')
+
     @staticmethod
-    def send(event_type, message, targets, options, **kwargs):
+    def send(template_name, message, targets, **kwargs):
-        """
+        subject = 'Lemur: Expiration Notification'
-        Configures all Lemur email messaging
 
-        :param event_type:
+        body = render_html(template_name, message)
-        :param options:
-        """
-        subject = 'Notification: Lemur'
-
-        if event_type == 'expiration':
-            subject = 'Notification: SSL Certificate Expiration '
-
-        # jinja template depending on type
-        template = env.get_template('{}.html'.format(event_type))
-        body = template.render(dict(messages=message, hostname=current_app.config.get('LEMUR_HOSTNAME')))
 
         s_type = current_app.config.get("LEMUR_EMAIL_SENDER", 'ses').lower()
+
         if s_type == 'ses':
-            conn = boto.connect_ses()
+            send_via_ses(subject, body, targets)
-            conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, targets, format='html')
 
         elif s_type == 'smtp':
-            msg = Message(subject, recipients=targets)
+            send_via_smtp(subject, body, targets)
-            msg.body = ""  # kinda a weird api for sending html emails
-            msg.html = body
-            smtp_mail.send(msg)
-
-        else:
-            current_app.logger.error("No mail carrier specified, notification emails were not able to be sent!")

lemur/plugins/lemur_email/templates/rotation.html

@@ -34,7 +34,7 @@
                     <tr height="16"></tr>
                     <tr>
                         <td>
-                            <table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
+                            <table bgcolor="#3681E4" width="100%" border="0" cellspacing="0" cellpadding="0"
                                    style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
                                 <tbody>
                                 <tr>
@@ -43,7 +43,7 @@
                                 <tr>
                                     <td width="32px"></td>
                                     <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
-                                       Your certificate(s) are being rotated!
+                                       Your certificate(s) have been rotated!
                                     </td>
                                     <td width="32px"></td>
                                 </tr>
@@ -56,7 +56,7 @@
                     </tr>
                     <tr>
                         <td>
-                            <table bgcolor="#3681E4" width="100%" border="0" cellspacing="0" cellpadding="0"
+                            <table width="100%" border="0" cellspacing="0" cellpadding="0"
                                    style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
                                 <tbody>
                                 <tr height="16px">
@@ -71,11 +71,51 @@
                                             <tr>
                                                 <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
                                                     Hi,
+                                                    <br>This is a Lemur certificate rotation notice. Your certificate has been re-issued and re-provisioned.
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020;line-height:1.5">
+                                                    Impacted Certificate:
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    <table border="0" cellspacing="0" cellpadding="0"
+                                                           style="margin-top:48px;margin-bottom:48px">
+                                                        <tbody>
+                                                        {%  for message in messages %}
+                                                            <tr valign="middle">
+                                                                <td width="32px"></td>
+                                                                <td width="16px"></td>
+                                                                <td style="line-height:1.2">
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.name }}</span>
+                                                                    <br>
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
+                                                                        {{ message.endpoints | length }} Endpoints
+                                                                        <br>{{ message.owner }}
+                                                                        <br>{{ message.not_after | time }}
+                                                                        <a href="https://{{ hostname }}/#/certificates/{{ message.name }}" target="_blank">Details</a>
+                                                                    </span>
+                                                                </td>
+                                                            </tr>
+                                                                {%  if not loop.last %}
+                                                                <tr valign="middle">
+                                                                    <td width="32px" height="24px"></td>
+                                                                </tr>
+                                                                {% endif %}
+                                                        {%  endfor %}
+                                                        </tbody>
+                                                    </table>
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020;line-height:1.5">
+                                                    Impacted Endpoints:
                                                 </td>
                                             </tr>
                                             <tr>
                                                 <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
-                                                    <br>This is a Lemur certificate rotation notice. This a purely informational notice. The following certificates will be rotated on any associated endpoints.
                                                     <table border="0" cellspacing="0" cellpadding="0"
                                                            style="margin-top:48px;margin-bottom:48px">
                                                         <tbody>

lemur/plugins/utils.py

@@ -0,0 +1,21 @@
+"""
+.. module: lemur.plugins.utils
+    :platform: Unix
+
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+
+"""
+
+
+def get_plugin_option(name, options):
+    """
+    Retrieve option name from options dict.
+    :param options:
+    :return:
+    """
+    for o in options:
+        if o.get('name') == name:
+            return o['value']

lemur/sources/cli.py

@@ -0,0 +1,88 @@
+"""
+.. module: lemur.sources.cli
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+import sys
+import time
+
+from tabulate import tabulate
+
+from flask_script import Manager
+
+from flask import current_app
+
+from lemur.extensions import metrics
+from lemur.sources import service as source_service
+from lemur.users import service as user_service
+
+manager = Manager(usage="Handles all source related tasks.")
+
+
+def validate_sources(source_strings):
+    sources = []
+    if not source_strings:
+        table = []
+        for source in source_service.get_all():
+            table.append([source.label, source.active, source.description])
+
+        print("No source specified choose from below:")
+        print(tabulate(table, headers=['Label', 'Active', 'Description']))
+        sys.exit(1)
+
+    if 'all' in source_strings:
+        sources = source_service.get_all()
+    else:
+        for source_str in source_strings:
+            source = source_service.get_by_label(source_str)
+
+            if not source:
+                print("Unable to find specified source with label: {0}".format(source_str))
+                sys.exit(1)
+
+            sources.append(source)
+    return sources
+
+
+@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
+def sync(source_strings):
+    source_objs = validate_sources(source_strings)
+    for source in source_objs:
+        start_time = time.time()
+        print("[+] Staring to sync source: {label}!\n".format(label=source.label))
+
+        user = user_service.get_by_username('lemur')
+
+        try:
+            source_service.sync(source, user)
+            print(
+                "[+] Finished syncing source: {label}. Run Time: {time}".format(
+                    label=source.label,
+                    time=(time.time() - start_time)
+                )
+            )
+        except Exception as e:
+            current_app.logger.exception(e)
+
+            print(
+                "[X] Failed syncing source {label}!\n".format(label=source.label)
+            )
+
+            metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label})
+
+
+@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
+def clean(source_strings):
+    source_objs = validate_sources(source_strings)
+    for source in source_objs:
+        start_time = time.time()
+        print("[+] Staring to clean source: {label}!\n".format(label=source.label))
+        source_service.clean(source)
+        print(
+            "[+] Finished cleaning source: {label}. Run Time: {time}\n".format(
+                label=source.label,
+                time=(time.time() - start_time)
+            )
+        )

lemur/tests/test_endpoints.py

@@ -8,7 +8,7 @@ from .vectors import VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN
 
 
 def test_rotate_certificate(client, source_plugin):
-    from lemur.endpoints.service import rotate_certificate
+    from lemur.deployment.service import rotate_certificate
     new_certificate = CertificateFactory()
     endpoint = EndpointFactory()
 

lemur/tests/test_messaging.py

@@ -0,0 +1,29 @@
+import pytest
+from freezegun import freeze_time
+
+from datetime import timedelta
+
+
+def test_needs_notification(app, certificate, notification):
+    from lemur.notifications.messaging import needs_notification
+    assert not needs_notification(certificate)
+
+    with pytest.raises(Exception):
+        notification.options = [{'name': 'interval', 'value': 10}, {'name': 'unit', 'value': 'min'}]
+        certificate.notifications.append(notification)
+        needs_notification(certificate)
+
+    certificate.notifications[0].options = [{'name': 'interval', 'value': 10}, {'name': 'unit', 'value': 'days'}]
+    assert not needs_notification(certificate)
+
+    delta = certificate.not_after - timedelta(days=10)
+    with freeze_time(delta.datetime):
+        assert needs_notification(certificate)
+
+
+def test_send_expiration_notification():
+    assert False
+
+
+def test_send_rotation_notification():
+    assert False