Modifying the way we report metrics. Relying on metric tags instead of the the metric name for additional dimensions. (#1036)

lemur/notifications/cli.py

@@ -7,6 +7,8 @@
 """
 from flask_script import Manager
 
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
+from lemur.extensions import sentry, metrics
 from lemur.notifications.messaging import send_expiration_notifications
 
 manager = Manager(usage="Handles notification related tasks.")
@@ -25,11 +27,18 @@ def expirations(exclude):
 
     :return:
     """
-    print("Starting to notify subscribers about expiring certificates!")
-    success, failed = send_expiration_notifications(exclude)
-    print(
-        "Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}".format(
-            success=success,
-            failed=failed
+    status = FAILURE_METRIC_STATUS
+    try:
+        print("Starting to notify subscribers about expiring certificates!")
+        success, failed = send_expiration_notifications(exclude)
+        print(
+            "Finished notifying subscribers about expiring certificates! Sent: {success} Failed: {failed}".format(
+                success=success,
+                failed=failed
+            )
         )
-    )
+        status = SUCCESS_METRIC_STATUS
+    except Exception as e:
+        sentry.captureException()
+
+    metrics.send('expiration_notification_job', 'counter', 1, metric_tags={'status': status})

lemur/notifications/messaging.py

@@ -18,6 +18,7 @@ from flask import current_app
 from sqlalchemy import and_
 
 from lemur import database, metrics
+from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS
 from lemur.extensions import sentry
 from lemur.common.utils import windowed_query
 
@@ -94,14 +95,17 @@ def send_notification(event_type, data, targets, notification):
     :param notification:
     :return:
     """
+    status = FAILURE_METRIC_STATUS
     try:
         notification.plugin.send(event_type, data, targets, notification.options)
-        metrics.send('{0}_notification_sent'.format(event_type), 'counter', 1)
-        return True
+        status = SUCCESS_METRIC_STATUS
     except Exception as e:
         sentry.captureException()
-        metrics.send('{0}_notification_failure'.format(event_type), 'counter', 1)
-        current_app.logger.exception(e)
+
+    metrics.send('notification', 'counter', 1, metric_tags={'status': status, 'event_type': event_type})
+
+    if status == SUCCESS_METRIC_STATUS:
+        return True
 
 
 def send_expiration_notifications(exclude):
@@ -147,8 +151,10 @@ def send_rotation_notification(certificate, notification_plugin=None):
     rotated.
 
     :param certificate:
+    :param notification_plugin:
     :return:
     """
+    status = FAILURE_METRIC_STATUS
     if not notification_plugin:
         notification_plugin = plugins.get(current_app.config.get('LEMUR_DEFAULT_NOTIFICATION_PLUGIN'))
 
@@ -156,12 +162,14 @@ def send_rotation_notification(certificate, notification_plugin=None):
 
     try:
         notification_plugin.send('rotation', data, [data['owner']])
-        metrics.send('rotation_notification_sent', 'counter', 1)
-        return True
+        status = SUCCESS_METRIC_STATUS
     except Exception as e:
         sentry.captureException()
-        metrics.send('rotation_notification_failure', 'counter', 1)
-        current_app.logger.exception(e)
+
+    metrics.send('notification', 'counter', 1, metric_tags={'status': status, 'event_type': 'rotation'})
+
+    if status == SUCCESS_METRIC_STATUS:
+        return True
 
 
 def needs_notification(certificate):