Adding additional reporting and refactoring existing setup. (#620)

lemur/certificates/models.py

@@ -158,6 +158,14 @@ class Certificate(db.Model):
         if isinstance(cert.public_key(), rsa.RSAPublicKey):
             return 'RSA{key_size}'.format(key_size=cert.public_key().key_size)
 
+    @property
+    def validity_remaining(self):
+        return abs(self.not_after - arrow.utcnow())
+
+    @property
+    def validity_range(self):
+        return self.not_after - self.not_before
+
     @hybrid_property
     def expired(self):
         if self.not_after <= arrow.utcnow():

lemur/manage.py

@@ -1,16 +1,11 @@
 from __future__ import unicode_literals    # at top of module
 
-import arrow
-from datetime import datetime, timedelta
-from collections import Counter
-
 import os
 import sys
 import base64
 import requests
 import json
 
-from tabulate import tabulate
 from gunicorn.config import make_settings
 
 from cryptography.fernet import Fernet
@@ -24,11 +19,10 @@ from lemur.sources.cli import manager as source_manager
 from lemur.certificates.cli import manager as certificate_manager
 from lemur.notifications.cli import manager as notification_manager
 from lemur.endpoints.cli import manager as endpoint_manager
-
+from lemur.reporting.cli import manager as report_manager
 from lemur import database
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
-from lemur.authorities import service as authority_service
 from lemur.notifications import service as notification_service
 
 
@@ -522,100 +516,6 @@ def publish_unapproved_verisign_certificates():
     metrics.send('pending_certificates', 'gauge', certs)
 
 
-class Report(Command):
-    """
-    Defines a set of reports to be run periodically against Lemur.
-    """
-    option_list = (
-        Option('-n', '--name', dest='name', default=None, help='Name of the report to run.'),
-        Option('-d', '--duration', dest='duration', default=356, help='Number of days to run the report'),
-    )
-
-    def run(self, name, duration):
-        end = datetime.utcnow()
-        start = end - timedelta(days=duration)
-
-        if name == 'authority':
-            self.certificates_issued(name, start, end)
-
-        elif name == 'activeFQDNS':
-            self.active_fqdns()
-
-    @staticmethod
-    def active_fqdns():
-        """
-        Generates a report that gives the number of active fqdns, but root domain.
-        :return:
-        """
-        from lemur.certificates.service import get_all_certs
-        sys.stdout.write("FQDN, Root Domain, Issuer, Total Length (days), Time until expiration (days)\n")
-        for cert in get_all_certs():
-            if not cert.expired:
-                now = arrow.utcnow()
-                ttl = now - cert.not_before
-                total_length = cert.not_after - cert.not_before
-
-                for fqdn in cert.domains:
-                    root_domain = ".".join(fqdn.name.split('.')[-2:])
-                    sys.stdout.write(", ".join([fqdn.name, root_domain, cert.issuer, str(total_length.days), str(ttl.days)]) + "\n")
-
-    @staticmethod
-    def certificates_issued(name=None, start=None, end=None):
-        """
-        Generates simple report of number of certificates issued by the authority, if no authority
-        is specified report on total number of certificates.
-
-        :param name:
-        :param start:
-        :param end:
-        :return:
-        """
-
-        def _calculate_row(authority):
-            day_cnt = Counter()
-            month_cnt = Counter()
-            year_cnt = Counter()
-
-            for cert in authority.certificates:
-                date = cert.date_created.date()
-                day_cnt[date.day] += 1
-                month_cnt[date.month] += 1
-                year_cnt[date.year] += 1
-
-            try:
-                day_avg = int(sum(day_cnt.values()) / len(day_cnt.keys()))
-            except ZeroDivisionError:
-                day_avg = 0
-
-            try:
-                month_avg = int(sum(month_cnt.values()) / len(month_cnt.keys()))
-            except ZeroDivisionError:
-                month_avg = 0
-
-            try:
-                year_avg = int(sum(year_cnt.values()) / len(year_cnt.keys()))
-            except ZeroDivisionError:
-                year_avg = 0
-
-            return [authority.name, authority.description, day_avg, month_avg, year_avg]
-
-        rows = []
-        if not name:
-            for authority in authority_service.get_all():
-                rows.append(_calculate_row(authority))
-
-        else:
-            authority = authority_service.get_by_name(name)
-
-            if not authority:
-                sys.stderr.write('[!] Authority {0} was not found.'.format(name))
-                sys.exit(1)
-
-            rows.append(_calculate_row(authority))
-
-        sys.stdout.write(tabulate(rows, headers=["Authority Name", "Description", "Daily Average", "Monthy Average", "Yearly Average"]) + "\n")
-
-
 def main():
     manager.add_command("start", LemurServer())
     manager.add_command("runserver", Server(host='127.0.0.1', threaded=True))
@@ -630,7 +530,7 @@ def main():
     manager.add_command("certificate", certificate_manager)
     manager.add_command("notify", notification_manager)
     manager.add_command("endpoint", endpoint_manager)
-    manager.add_command("report", Report())
+    manager.add_command("report", report_manager)
     manager.run()
 
 

lemur/reporting/cli.py

@@ -0,0 +1,61 @@
+"""
+.. module: lemur.reporting.cli
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from tabulate import tabulate
+from flask_script import Manager
+
+from lemur.reporting.service import fqdns, expiring_certificates
+
+manager = Manager(usage="Reporting related tasks.")
+
+
+@manager.option('-v', '--validity', dest='validity', choices=['all', 'expired', 'valid'], default='all', help='Filter certificates by validity.')
+@manager.option('-d', '--deployment', dest='deployment', choices=['all', 'deployed', 'ready'], default='all', help='Filter by deployment status.')
+def fqdn(deployment, validity):
+    """
+    Generates a report in order to determine the number of FQDNs covered by Lemur issued certificates.
+    """
+    headers = ['FQDN', 'Root Domain', 'Issuer', 'Owner', 'Validity End', 'Total Length (days), Time Until Expiration (days)']
+    rows = []
+
+    for cert in fqdns(validity=validity, deployment=deployment).all():
+        for domain in cert.domains:
+            rows.append([
+                domain.name,
+                '.'.join(domain.name.split('.')[1:]),
+                cert.issuer,
+                cert.owner,
+                cert.not_after,
+                cert.validity_range.days,
+                cert.validity_remaining.days
+            ])
+
+    print(tabulate(rows, headers=headers))
+
+
+@manager.option('-ttl', '--ttl', dest='ttl', default=30, help='Days til expiration.')
+@manager.option('-d', '--deployment', dest='deployment', choices=['all', 'deployed', 'ready'], default='all', help='Filter by deployment status.')
+def expiring(ttl, deployment):
+    """
+    Returns certificates expiring in the next n days.
+    """
+    headers = ['Common Name', 'Owner', 'Issuer', 'Validity End', 'Endpoint']
+    rows = []
+
+    for cert in expiring_certificates(ttl=ttl, deployment=deployment).all():
+        for endpoint in cert.endpoints:
+            rows.append(
+                [
+                    cert.cn,
+                    cert.owner,
+                    cert.issuer,
+                    cert.not_after,
+                    endpoint.dnsname
+                ]
+            )
+
+    print(tabulate(rows, headers=headers))

lemur/reporting/service.py

@@ -0,0 +1,77 @@
+import arrow
+from datetime import timedelta
+
+from sqlalchemy import cast, not_
+from sqlalchemy_utils import ArrowType
+
+from lemur import database
+from lemur.certificates.models import Certificate
+
+
+def filter_by_validity(query, validity=None):
+    if validity == 'expired':
+        query = query.filter(Certificate.expired == True)  # noqa
+
+    elif validity == 'valid':
+        query = query.filter(Certificate.expired == False)  # noqa
+
+    return query
+
+
+def filter_by_owner(query, owner=None):
+    if owner:
+        return query.filter(Certificate.owner == owner)
+
+    return query
+
+
+def filter_by_issuer(query, issuer=None):
+    if issuer:
+        return query.filter(Certificate.issuer == issuer)
+
+    return query
+
+
+def filter_by_deployment(query, deployment=None):
+    if deployment == 'deployed':
+        query = query.filter(Certificate.endpoints.any())
+
+    elif deployment == 'ready':
+        query = query.filter(not_(Certificate.endpoints.any()))
+
+    return query
+
+
+def filter_by_validity_end(query, validity_end=None):
+    if validity_end:
+        return query.filter(cast(Certificate.not_after, ArrowType) <= validity_end)
+
+    return query
+
+
+def fqdns(**kwargs):
+    """
+    Returns an FQDN report.
+    :return:
+    """
+    query = database.session_query(Certificate)
+    query = filter_by_deployment(query, deployment=kwargs.get('deployed'))
+    query = filter_by_validity(query, validity=kwargs.get('validity'))
+    return query
+
+
+def expiring_certificates(**kwargs):
+    """
+    Returns an Expiring report.
+    :return:
+    """
+    ttl = kwargs.get('ttl', 30)
+    now = arrow.utcnow()
+    validity_end = now + timedelta(days=ttl)
+
+    query = database.session_query(Certificate)
+    query = filter_by_deployment(query, deployment=kwargs.get('deployed'))
+    query = filter_by_validity(query, validity='valid')
+    query = filter_by_validity_end(query, validity_end=validity_end)
+
+    return query