From cdb3814469d4c3f39d3439d3d5a178696794e70e Mon Sep 17 00:00:00 2001 From: kevgliss Date: Sun, 2 Aug 2015 09:14:27 -0700 Subject: [PATCH] Fixing notification deduplication and roll up --- lemur/certificates/models.py | 1 - lemur/certificates/service.py | 13 ----- lemur/destinations/service.py | 26 ++++++++- lemur/destinations/views.py | 16 +++++- lemur/manage.py | 17 ++++++ lemur/notifications/service.py | 55 +++++++++++++------ lemur/plugins/base/v1.py | 11 +++- lemur/plugins/lemur_email/plugin.py | 6 -- lemur/plugins/service.py | 7 +++ .../static/app/angular/dashboard/dashboard.js | 10 ++-- .../app/angular/dashboard/dashboard.tpl.html | 11 ++++ lemur/tests/test_notifications.py | 20 +++++++ 12 files changed, 144 insertions(+), 49 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index cbe37019..ef7cbdc6 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -279,5 +279,4 @@ class Certificate(db.Model): @event.listens_for(Certificate.destinations, 'append') def update_destinations(target, value, initiator): destination_plugin = plugins.get(value.plugin_name) - destination_plugin.upload(target.body, target.private_key, target.chain, value.options) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 8eb2c1e9..237f145f 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -18,7 +18,6 @@ from lemur.destinations.models import Destination from lemur.notifications.models import Notification from lemur.authorities.models import Authority - from lemur.roles.models import Role from cryptography import x509 @@ -400,14 +399,6 @@ def stats(**kwargs): :param kwargs: :return: """ - query = database.session_query(Certificate) - - if kwargs.get('active') == 'true': - query = query.filter(Certificate.elb_listeners.any()) - - if kwargs.get('destination_id'): - query = query.filter(Certificate.destinations.any(Destination.id == kwargs.get('destination_id'))) - if kwargs.get('metric') == 'not_after': start = arrow.utcnow() end = start.replace(weeks=+32) @@ -420,10 +411,6 @@ def stats(**kwargs): attr = getattr(Certificate, kwargs.get('metric')) query = database.db.session.query(attr, func.count(attr)) - # TODO this could be cleaned up - if kwargs.get('active') == 'true': - query = query.filter(Certificate.elb_listeners.any()) - items = query.group_by(attr).all() keys = [] diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 38dc600f..f27a138c 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -5,6 +5,8 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from sqlalchemy import func + from lemur import database from lemur.destinations.models import Destination from lemur.certificates.models import Certificate @@ -28,9 +30,8 @@ def update(destination_id, label, options, description): Updates an existing destination. :param destination_id: Lemur assigned ID - :param destination_number: AWS assigned ID :param label: Destination common name - :param comments: + :param description: :rtype : Destination :return: """ @@ -107,3 +108,24 @@ def render(args): query = database.sort(query, Destination, sort_by, sort_dir) return database.paginate(query, page, count) + + +def stats(**kwargs): + """ + Helper that defines some useful statistics about destinations. + + :param kwargs: + :return: + """ + attr = getattr(Destination, kwargs.get('metric')) + query = database.db.session.query(attr, func.count(attr)) + + items = query.group_by(attr).all() + + keys = [] + values = [] + for key, count in items: + keys.append(key) + values.append(count) + + return {'labels': keys, 'values': values} diff --git a/lemur/destinations/views.py b/lemur/destinations/views.py index 55ff7071..21e7886e 100644 --- a/lemur/destinations/views.py +++ b/lemur/destinations/views.py @@ -353,7 +353,21 @@ class CertificateDestinations(AuthenticatedResource): return service.render(args) +class DestinationsStats(AuthenticatedResource): + """ Defines the 'certificates' stats endpoint """ + def __init__(self): + self.reqparse = reqparse.RequestParser() + super(DestinationsStats, self).__init__() + + def get(self): + self.reqparse.add_argument('metric', type=str, location='args') + args = self.reqparse.parse_args() + items = service.stats(**args) + return dict(items=items, total=len(items)) + + api.add_resource(DestinationsList, '/destinations', endpoint='destinations') -api.add_resource(Destinations, '/destinations/', endpoint='account') +api.add_resource(Destinations, '/destinations/', endpoint='destination') api.add_resource(CertificateDestinations, '/certificates//destinations', endpoint='certificateDestinations') +api.add_resource(DestinationsStats, '/destinations/stats', endpoint='destinationStats') diff --git a/lemur/manage.py b/lemur/manage.py index 1ad37291..4f5794ac 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -222,6 +222,23 @@ def sync_sources(labels, view): sync_lock.release() +@manager.command +def notify(): + """ + Runs Lemur's notification engine, that looks for expired certificates and sends + notifications out to those that bave subscribed to them. + + :return: + """ + sys.stdout.write("Starting to notify subscribers about expiring certificates!\n") + count = notification_service.send_expiration_notifications() + sys.stdout.write( + "Finished notifying subscribers about expiring certificates! Sent {count} notifications!\n".format( + count=count + ) + ) + + class InitializeApp(Command): """ This command will bootstrap our database with any destinations as diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 0ffdc52d..9653f957 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -34,7 +34,7 @@ def _get_message_data(cert): cert_dict = cert.as_dict() cert_dict['creator'] = cert.user.email cert_dict['domains'] = [x .name for x in cert.domains] - cert_dict['superseded'] = list(set([x.name for x in find_superseded(cert.domains) if cert.name != x])) + cert_dict['superseded'] = list(set([x.name for x in _find_superseded(cert) if cert.name != x])) return cert_dict @@ -44,8 +44,13 @@ def _deduplicate(messages): a roll up to the same set if the recipients are the same """ roll_ups = [] - for targets, data in messages: - for m, r in roll_ups: + for data, options in messages: + targets = [] + for o in options: + if o.get('name') == 'recipients': + targets = o['value'].split(',') + + for m, r, o in roll_ups: if r == targets: m.append(data) current_app.logger.info( @@ -53,7 +58,7 @@ def _deduplicate(messages): data['name'], ",".join(targets))) break else: - roll_ups.append(([data], targets, data.plugin_options)) + roll_ups.append(([data], targets, options)) return roll_ups @@ -62,21 +67,30 @@ def send_expiration_notifications(): This function will check for upcoming certificate expiration, and send out notification emails at given intervals. """ - notifications = 0 + sent = 0 - for plugin_name, notifications in database.get_all(Notification, True, field='active').group_by(Notification.plugin_name): - notifications += 1 + for plugin in plugins.all(plugin_type='notification'): + notifications = database.db.session.query(Notification)\ + .filter(Notification.plugin_name == plugin.slug)\ + .filter(Notification.active == True).all() # noqa - messages = _deduplicate(notifications) - plugin = plugins.get(plugin_name) + messages = [] + for n in notifications: + for c in n.certificates: + if _is_eligible_for_notifications(c): + messages.append((_get_message_data(c), n.options)) + + messages = _deduplicate(messages) for data, targets, options in messages: + sent += 1 plugin.send('expiration', data, targets, options) - current_app.logger.info("Lemur has sent {0} certification notifications".format(notifications)) + current_app.logger.info("Lemur has sent {0} certification notifications".format(sent)) + return sent -def get_domain_certificate(name): +def _get_domain_certificate(name): """ Fetch the SSL certificate currently hosted at a given domain (if any) and compare it against our all of our know certificates to determine if a new @@ -92,7 +106,7 @@ def get_domain_certificate(name): current_app.logger.info(str(e)) -def find_superseded(domains): +def _find_superseded(cert): """ Here we try to fetch any domain in the certificate to see if we can resolve it and to try and see if it is currently serving the certificate we are @@ -103,17 +117,22 @@ def find_superseded(domains): """ query = database.session_query(Certificate) ss_list = [] - for domain in domains: - dc = get_domain_certificate(domain.name) - if dc: - ss_list.append(dc) + + # determine what is current host at our domains + for domain in cert.domains: + dups = _get_domain_certificate(domain.name) + for c in dups: + if c.body != cert.body: + ss_list.append(dups) + current_app.logger.info("Trying to resolve {0}".format(domain.name)) - query = query.filter(Certificate.domains.any(Domain.name.in_([x.name for x in domains]))) + # look for other certificates that may not be hosted but cover the same domains + query = query.filter(Certificate.domains.any(Domain.name.in_([x.name for x in cert.domains]))) query = query.filter(Certificate.active == True) # noqa query = query.filter(Certificate.not_after >= arrow.utcnow().format('YYYY-MM-DD')) + query = query.filter(Certificate.body != cert.body) ss_list.extend(query.all()) - return ss_list diff --git a/lemur/plugins/base/v1.py b/lemur/plugins/base/v1.py index ce378b98..7026c99f 100644 --- a/lemur/plugins/base/v1.py +++ b/lemur/plugins/base/v1.py @@ -101,13 +101,18 @@ class IPlugin(local): Returns a list of tuples pointing to various resources for this plugin. >>> def get_resource_links(self): >>> return [ - >>> ('Documentation', 'http://sentry.readthedocs.org'), - >>> ('Bug Tracker', 'https://github.com/getsentry/sentry/issues'), - >>> ('Source', 'https://github.com/getsentry/sentry'), + >>> ('Documentation', 'http://lemury.readthedocs.org'), + >>> ('Bug Tracker', 'https://github.com/Netflix/lemur/issues'), + >>> ('Source', 'https://github.com/Netflix/lemur'), >>> ] """ return self.resource_links + def get_option(self, name, options): + for o in options: + if o.get(name): + return o['value'] + class Plugin(IPlugin): """ diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index cecc6e37..90c53a67 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -19,12 +19,6 @@ from lemur.plugins import lemur_email as email from lemur.plugins.lemur_email.templates.config import env -def find_value(name, options): - for o in options: - if o.get(name): - return o['value'] - - class EmailNotificationPlugin(ExpirationNotificationPlugin): title = 'Email' slug = 'email-notification' diff --git a/lemur/plugins/service.py b/lemur/plugins/service.py index e69de29b..33965963 100644 --- a/lemur/plugins/service.py +++ b/lemur/plugins/service.py @@ -0,0 +1,7 @@ +""" +.. module: service + :platform: Unix + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" diff --git a/lemur/static/app/angular/dashboard/dashboard.js b/lemur/static/app/angular/dashboard/dashboard.js index f0d94eab..630e0439 100644 --- a/lemur/static/app/angular/dashboard/dashboard.js +++ b/lemur/static/app/angular/dashboard/dashboard.js @@ -11,11 +11,6 @@ angular.module('lemur') var baseAccounts = LemurRestangular.all('accounts'); - baseAccounts.getList() - .then(function (data) { - $scope.accounts = data; - }); - $scope.colours = [ { fillColor: 'rgba(41, 171, 224, 0.2)', @@ -89,4 +84,9 @@ angular.module('lemur') .then(function (data) { $scope.expiring = {labels: data.items.labels, values: [data.items.values]}; }); + + LemurRestangular.all('destinations').customGET('stats', {metric: 'certificates'}) + .then(function (data) { + $scope.destinations = {labels: data.items.labels, values: [data.items.values]}; + }); }); diff --git a/lemur/static/app/angular/dashboard/dashboard.tpl.html b/lemur/static/app/angular/dashboard/dashboard.tpl.html index a00880a4..c8d2c6a6 100644 --- a/lemur/static/app/angular/dashboard/dashboard.tpl.html +++ b/lemur/static/app/angular/dashboard/dashboard.tpl.html @@ -36,6 +36,17 @@ +
+
+
+
+

Destinations

+
+
+ +
+
+
diff --git a/lemur/tests/test_notifications.py b/lemur/tests/test_notifications.py index e66c1984..cfea8afa 100644 --- a/lemur/tests/test_notifications.py +++ b/lemur/tests/test_notifications.py @@ -115,3 +115,23 @@ def test_admin_notifications_get(client): resp = client.get(api.url_for(NotificationsList), headers=VALID_ADMIN_HEADER_TOKEN) assert resp.status_code == 200 assert resp.json == {'items': [], 'total': 0} + + +def test_get_message_data(session): + assert 1 == 2 + + +def test_deduplicate(session): + assert 1 == 2 + + +def test_find_superseded(session): + assert 1 == 2 + + +def test_is_eligible_for_notifications(session): + assert 1 == 2 + + +def test_create_default_expiration_notifications(session): + assert 1 == 2