Fixing notification deduplication and roll up

lemur/certificates/models.py

@@ -279,5 +279,4 @@ class Certificate(db.Model):
 @event.listens_for(Certificate.destinations, 'append')
 def update_destinations(target, value, initiator):
     destination_plugin = plugins.get(value.plugin_name)
-
     destination_plugin.upload(target.body, target.private_key, target.chain, value.options)

lemur/certificates/service.py

@@ -18,7 +18,6 @@ from lemur.destinations.models import Destination
 from lemur.notifications.models import Notification
 from lemur.authorities.models import Authority
 
-
 from lemur.roles.models import Role
 
 from cryptography import x509
@@ -400,14 +399,6 @@ def stats(**kwargs):
     :param kwargs:
     :return:
     """
-    query = database.session_query(Certificate)
-
-    if kwargs.get('active') == 'true':
-        query = query.filter(Certificate.elb_listeners.any())
-
-    if kwargs.get('destination_id'):
-        query = query.filter(Certificate.destinations.any(Destination.id == kwargs.get('destination_id')))
-
     if kwargs.get('metric') == 'not_after':
         start = arrow.utcnow()
         end = start.replace(weeks=+32)
@@ -420,10 +411,6 @@ def stats(**kwargs):
         attr = getattr(Certificate, kwargs.get('metric'))
         query = database.db.session.query(attr, func.count(attr))
 
-        # TODO this could be cleaned up
-        if kwargs.get('active') == 'true':
-            query = query.filter(Certificate.elb_listeners.any())
-
         items = query.group_by(attr).all()
 
     keys = []

lemur/destinations/service.py

@@ -5,6 +5,8 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+from sqlalchemy import func
+
 from lemur import database
 from lemur.destinations.models import Destination
 from lemur.certificates.models import Certificate
@@ -28,9 +30,8 @@ def update(destination_id, label, options, description):
     Updates an existing destination.
 
     :param destination_id:  Lemur assigned ID
-    :param destination_number: AWS assigned ID
     :param label: Destination common name
-    :param comments:
+    :param description:
     :rtype : Destination
     :return:
     """
@@ -107,3 +108,24 @@ def render(args):
         query = database.sort(query, Destination, sort_by, sort_dir)
 
     return database.paginate(query, page, count)
+
+
+def stats(**kwargs):
+    """
+    Helper that defines some useful statistics about destinations.
+
+    :param kwargs:
+    :return:
+    """
+    attr = getattr(Destination, kwargs.get('metric'))
+    query = database.db.session.query(attr, func.count(attr))
+
+    items = query.group_by(attr).all()
+
+    keys = []
+    values = []
+    for key, count in items:
+        keys.append(key)
+        values.append(count)
+
+    return {'labels': keys, 'values': values}

lemur/destinations/views.py

@@ -353,7 +353,21 @@ class CertificateDestinations(AuthenticatedResource):
         return service.render(args)
 
 
+class DestinationsStats(AuthenticatedResource):
+    """ Defines the 'certificates' stats endpoint """
+    def __init__(self):
+        self.reqparse = reqparse.RequestParser()
+        super(DestinationsStats, self).__init__()
+
+    def get(self):
+        self.reqparse.add_argument('metric', type=str, location='args')
+        args = self.reqparse.parse_args()
+        items = service.stats(**args)
+        return dict(items=items, total=len(items))
+
+
 api.add_resource(DestinationsList, '/destinations', endpoint='destinations')
-api.add_resource(Destinations, '/destinations/<int:destination_id>', endpoint='account')
+api.add_resource(Destinations, '/destinations/<int:destination_id>', endpoint='destination')
 api.add_resource(CertificateDestinations, '/certificates/<int:certificate_id>/destinations',
                  endpoint='certificateDestinations')
+api.add_resource(DestinationsStats, '/destinations/stats', endpoint='destinationStats')

lemur/manage.py

@@ -222,6 +222,23 @@ def sync_sources(labels, view):
         sync_lock.release()
 
 
+@manager.command
+def notify():
+    """
+    Runs Lemur's notification engine, that looks for expired certificates and sends
+    notifications out to those that bave subscribed to them.
+
+    :return:
+    """
+    sys.stdout.write("Starting to notify subscribers about expiring certificates!\n")
+    count = notification_service.send_expiration_notifications()
+    sys.stdout.write(
+        "Finished notifying subscribers about expiring certificates! Sent {count} notifications!\n".format(
+            count=count
+        )
+    )
+
+
 class InitializeApp(Command):
     """
     This command will bootstrap our database with any destinations as

lemur/notifications/service.py

@@ -34,7 +34,7 @@ def _get_message_data(cert):
     cert_dict = cert.as_dict()
     cert_dict['creator'] = cert.user.email
     cert_dict['domains'] = [x .name for x in cert.domains]
-    cert_dict['superseded'] = list(set([x.name for x in find_superseded(cert.domains) if cert.name != x]))
+    cert_dict['superseded'] = list(set([x.name for x in _find_superseded(cert) if cert.name != x]))
     return cert_dict
 
 
@@ -44,8 +44,13 @@ def _deduplicate(messages):
     a roll up to the same set if the recipients are the same
     """
     roll_ups = []
-    for targets, data in messages:
+    for data, options in messages:
-        for m, r in roll_ups:
+        targets = []
+        for o in options:
+            if o.get('name') == 'recipients':
+                targets = o['value'].split(',')
+
+        for m, r, o in roll_ups:
             if r == targets:
                 m.append(data)
                 current_app.logger.info(
@@ -53,7 +58,7 @@ def _deduplicate(messages):
                         data['name'], ",".join(targets)))
                 break
         else:
-            roll_ups.append(([data], targets, data.plugin_options))
+            roll_ups.append(([data], targets, options))
     return roll_ups
 
 
@@ -62,21 +67,30 @@ def send_expiration_notifications():
     This function will check for upcoming certificate expiration,
     and send out notification emails at given intervals.
     """
-    notifications = 0
+    sent = 0
 
-    for plugin_name, notifications in database.get_all(Notification, True, field='active').group_by(Notification.plugin_name):
+    for plugin in plugins.all(plugin_type='notification'):
-        notifications += 1
+        notifications = database.db.session.query(Notification)\
+            .filter(Notification.plugin_name == plugin.slug)\
+            .filter(Notification.active == True).all()  # noqa
 
-        messages = _deduplicate(notifications)
+        messages = []
-        plugin = plugins.get(plugin_name)
+        for n in notifications:
+            for c in n.certificates:
+                if _is_eligible_for_notifications(c):
+                    messages.append((_get_message_data(c), n.options))
+
+        messages = _deduplicate(messages)
 
         for data, targets, options in messages:
+            sent += 1
             plugin.send('expiration', data, targets, options)
 
-    current_app.logger.info("Lemur has sent {0} certification notifications".format(notifications))
+        current_app.logger.info("Lemur has sent {0} certification notifications".format(sent))
+    return sent
 
 
-def get_domain_certificate(name):
+def _get_domain_certificate(name):
     """
     Fetch the SSL certificate currently hosted at a given domain (if any) and
     compare it against our all of our know certificates to determine if a new
@@ -92,7 +106,7 @@ def get_domain_certificate(name):
         current_app.logger.info(str(e))
 
 
-def find_superseded(domains):
+def _find_superseded(cert):
     """
     Here we try to fetch any domain in the certificate to see if we can resolve it
     and to try and see if it is currently serving the certificate we are
@@ -103,17 +117,22 @@ def find_superseded(domains):
     """
     query = database.session_query(Certificate)
     ss_list = []
-    for domain in domains:
+
-        dc = get_domain_certificate(domain.name)
+    # determine what is current host at our domains
-        if dc:
+    for domain in cert.domains:
-            ss_list.append(dc)
+        dups = _get_domain_certificate(domain.name)
+        for c in dups:
+            if c.body != cert.body:
+                ss_list.append(dups)
+
         current_app.logger.info("Trying to resolve {0}".format(domain.name))
 
-    query = query.filter(Certificate.domains.any(Domain.name.in_([x.name for x in domains])))
+    # look for other certificates that may not be hosted but cover the same domains
+    query = query.filter(Certificate.domains.any(Domain.name.in_([x.name for x in cert.domains])))
     query = query.filter(Certificate.active == True)  # noqa
     query = query.filter(Certificate.not_after >= arrow.utcnow().format('YYYY-MM-DD'))
+    query = query.filter(Certificate.body != cert.body)
     ss_list.extend(query.all())
-
     return ss_list
 
 

lemur/plugins/base/v1.py

@@ -101,13 +101,18 @@ class IPlugin(local):
         Returns a list of tuples pointing to various resources for this plugin.
         >>> def get_resource_links(self):
         >>>     return [
-        >>>         ('Documentation', 'http://sentry.readthedocs.org'),
+        >>>         ('Documentation', 'http://lemury.readthedocs.org'),
-        >>>         ('Bug Tracker', 'https://github.com/getsentry/sentry/issues'),
+        >>>         ('Bug Tracker', 'https://github.com/Netflix/lemur/issues'),
-        >>>         ('Source', 'https://github.com/getsentry/sentry'),
+        >>>         ('Source', 'https://github.com/Netflix/lemur'),
         >>>     ]
         """
         return self.resource_links
 
+    def get_option(self, name, options):
+        for o in options:
+            if o.get(name):
+                return o['value']
+
 
 class Plugin(IPlugin):
     """

lemur/plugins/lemur_email/plugin.py

@@ -19,12 +19,6 @@ from lemur.plugins import lemur_email as email
 from lemur.plugins.lemur_email.templates.config import env
 
 
-def find_value(name, options):
-    for o in options:
-        if o.get(name):
-            return o['value']
-
-
 class EmailNotificationPlugin(ExpirationNotificationPlugin):
     title = 'Email'
     slug = 'email-notification'

lemur/plugins/service.py

@@ -0,0 +1,7 @@
+"""
+.. module: service
+    :platform: Unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""

lemur/static/app/angular/dashboard/dashboard.js

@@ -11,11 +11,6 @@ angular.module('lemur')
 
     var baseAccounts = LemurRestangular.all('accounts');
 
-    baseAccounts.getList()
-      .then(function (data) {
-        $scope.accounts = data;
-      });
-
     $scope.colours = [
       {
         fillColor: 'rgba(41, 171, 224, 0.2)',
@@ -89,4 +84,9 @@ angular.module('lemur')
       .then(function (data) {
         $scope.expiring = {labels: data.items.labels, values: [data.items.values]};
       });
+
+    LemurRestangular.all('destinations').customGET('stats', {metric: 'certificates'})
+      .then(function (data) {
+        $scope.destinations = {labels: data.items.labels, values: [data.items.values]};
+      });
   });

lemur/static/app/angular/dashboard/dashboard.tpl.html

@@ -36,6 +36,17 @@
         </div>
       </div>
     </div>
+    <div class="row"></div>
+    <div class="col-md-6">
+        <div class="panel panel-default">
+            <div class="panel-heading">
+                <h3 class="panel-title">Destinations</h3>
+            </div>
+            <div class="panel-body">
+                <canvas id="destinationPie" class="chart chart-pie" data="destinations.values" labels="destinations.labels" colours="colours" legend="true"></canvas>
+            </div>
+        </div>
+    </div>
   </div>
   <!-- /.row -->
 </div>

lemur/tests/test_notifications.py

@@ -115,3 +115,23 @@ def test_admin_notifications_get(client):
     resp = client.get(api.url_for(NotificationsList), headers=VALID_ADMIN_HEADER_TOKEN)
     assert resp.status_code == 200
     assert resp.json == {'items': [], 'total': 0}
+
+
+def test_get_message_data(session):
+    assert 1 == 2
+
+
+def test_deduplicate(session):
+    assert 1 == 2
+
+
+def test_find_superseded(session):
+    assert 1 == 2
+
+
+def test_is_eligible_for_notifications(session):
+    assert 1 == 2
+
+
+def test_create_default_expiration_notifications(session):
+    assert 1 == 2