From f803fab41300ba2acba3634b5d5795072112d7b0 Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 6 Nov 2019 10:14:49 -0800 Subject: [PATCH 1/2] add plugin to send atlas metric via redis --- lemur/plugins/lemur_atlas_redis/__init__.py | 4 + lemur/plugins/lemur_atlas_redis/plugin.py | 97 +++++++++++++++++++++ setup.py | 1 + 3 files changed, 102 insertions(+) create mode 100644 lemur/plugins/lemur_atlas_redis/__init__.py create mode 100644 lemur/plugins/lemur_atlas_redis/plugin.py diff --git a/lemur/plugins/lemur_atlas_redis/__init__.py b/lemur/plugins/lemur_atlas_redis/__init__.py new file mode 100644 index 00000000..f8afd7e3 --- /dev/null +++ b/lemur/plugins/lemur_atlas_redis/__init__.py @@ -0,0 +1,4 @@ +try: + VERSION = __import__("pkg_resources").get_distribution(__name__).version +except Exception as e: + VERSION = "unknown" diff --git a/lemur/plugins/lemur_atlas_redis/plugin.py b/lemur/plugins/lemur_atlas_redis/plugin.py new file mode 100644 index 00000000..9a36743f --- /dev/null +++ b/lemur/plugins/lemur_atlas_redis/plugin.py @@ -0,0 +1,97 @@ +""" +.. module: lemur.plugins.lemur_atlas_redis.plugin + :platform: Unix + :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Jay Zarfoss +""" +from time import time +from redis import Redis +import json +from datetime import datetime + +from flask import current_app +from lemur.plugins import lemur_atlas as atlas +from lemur.plugins.bases.metric import MetricPlugin + + +def millis_since_epoch(): + """ + current time since epoch in milliseconds + """ + epoch = datetime.utcfromtimestamp(0) + delta = datetime.now() - epoch + return int(delta.total_seconds() * 1000.0) + + +class AtlasMetricRedisPlugin(MetricPlugin): + title = "AtlasRedis" + slug = "atlas-metric-Redius" + description = "Adds support for sending key metrics to Atlas via local Redis" + version = atlas.VERSION + + author = "Jay Zarfoss" + author_url = "https://github.com/netflix/lemur" + + options = [ + { + "name": "redis_host", + "type": "str", + "required": False, + "help_message": "If no host is provided localhost is assumed", + "default": "localhost", + }, + {"name": "redis_port", "type": "int", "required": False, "default": 28527}, + ] + + metric_data = {} + redis_host = None + redis_port = None + + def submit( + self, metric_name, metric_type, metric_value, metric_tags=None, options=None + ): + if not options: + options = self.options + + valid_types = ["COUNTER", "GAUGE", "TIMER"] + if metric_type.upper() not in valid_types: + raise Exception( + "Invalid Metric Type for Atlas: '{metric}' choose from: {options}".format( + metric=metric_type, options=",".join(valid_types) + ) + ) + + if metric_tags: + if not isinstance(metric_tags, dict): + raise Exception( + "Invalid Metric Tags for Atlas: Tags must be in dict format" + ) + + self.metric_data["timestamp"] = millis_since_epoch() + self.metric_data["type"] = metric_type.upper() + self.metric_data["name"] = str(metric_name) + self.metric_data["tags"] = metric_tags + + if ( + metric_value == "NaN" + or isinstance(metric_value, int) + or isinstance(metric_value, float) + ): + self.metric_data["value"] = metric_value + else: + raise Exception("Invalid Metric Value for Atlas: Metric must be a number") + + self.redis_host = self.get_option("redis_host", options) + self.redis_port = self.get_option("redis_port", options) + + try: + r = Redis(host=self.redis_host, port=self.redis_port, socket_timeout=0.1) + r.rpush('atlas-agent', json.dumps(self.metric_data)) + except Exception: + current_app.logger.warning( + "AtlasMetricsRedis: could not post atlas metrics to AtlasRedis {host}:{port}".format( + host=self.redis_host, port=self.redis_port + ) + ) diff --git a/setup.py b/setup.py index a01c110f..1c61e9f9 100644 --- a/setup.py +++ b/setup.py @@ -147,6 +147,7 @@ setup( 'java_keystore_export = lemur.plugins.lemur_jks.plugin:JavaKeystoreExportPlugin', 'openssl_export = lemur.plugins.lemur_openssl.plugin:OpenSSLExportPlugin', 'atlas_metric = lemur.plugins.lemur_atlas.plugin:AtlasMetricPlugin', + 'atlas_metric_redis = lemur.plugins.lemur_atlas.plugin:AtlasMetricRedisPlugin', 'kubernetes_destination = lemur.plugins.lemur_kubernetes.plugin:KubernetesDestinationPlugin', 'cryptography_issuer = lemur.plugins.lemur_cryptography.plugin:CryptographyIssuerPlugin', 'cfssl_issuer = lemur.plugins.lemur_cfssl.plugin:CfsslIssuerPlugin', From 113c9dd65744a4783e4ec2498797f2fe5e341061 Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 6 Nov 2019 10:42:59 -0800 Subject: [PATCH 2/2] atlas redis plugin typo cleanup and better exception handling --- lemur/plugins/lemur_atlas_redis/plugin.py | 10 +++++----- setup.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_atlas_redis/plugin.py b/lemur/plugins/lemur_atlas_redis/plugin.py index 9a36743f..e69ae672 100644 --- a/lemur/plugins/lemur_atlas_redis/plugin.py +++ b/lemur/plugins/lemur_atlas_redis/plugin.py @@ -6,7 +6,7 @@ .. moduleauthor:: Jay Zarfoss """ -from time import time + from redis import Redis import json from datetime import datetime @@ -27,7 +27,7 @@ def millis_since_epoch(): class AtlasMetricRedisPlugin(MetricPlugin): title = "AtlasRedis" - slug = "atlas-metric-Redius" + slug = "atlas-metric-redis" description = "Adds support for sending key metrics to Atlas via local Redis" version = atlas.VERSION @@ -89,9 +89,9 @@ class AtlasMetricRedisPlugin(MetricPlugin): try: r = Redis(host=self.redis_host, port=self.redis_port, socket_timeout=0.1) r.rpush('atlas-agent', json.dumps(self.metric_data)) - except Exception: + except Exception as e: current_app.logger.warning( - "AtlasMetricsRedis: could not post atlas metrics to AtlasRedis {host}:{port}".format( - host=self.redis_host, port=self.redis_port + "AtlasMetricsRedis: exception [{exception}] could not post atlas metrics to AtlasRedis [{host}:{port}], metric [{metricdata}]".format( + exception=e, host=self.redis_host, port=self.redis_port, metricdata=json.dumps(self.metric_data) ) ) diff --git a/setup.py b/setup.py index 1c61e9f9..90c0b2f8 100644 --- a/setup.py +++ b/setup.py @@ -147,7 +147,7 @@ setup( 'java_keystore_export = lemur.plugins.lemur_jks.plugin:JavaKeystoreExportPlugin', 'openssl_export = lemur.plugins.lemur_openssl.plugin:OpenSSLExportPlugin', 'atlas_metric = lemur.plugins.lemur_atlas.plugin:AtlasMetricPlugin', - 'atlas_metric_redis = lemur.plugins.lemur_atlas.plugin:AtlasMetricRedisPlugin', + 'atlas_metric_redis = lemur.plugins.lemur_atlas_redis.plugin:AtlasMetricRedisPlugin', 'kubernetes_destination = lemur.plugins.lemur_kubernetes.plugin:KubernetesDestinationPlugin', 'cryptography_issuer = lemur.plugins.lemur_cryptography.plugin:CryptographyIssuerPlugin', 'cfssl_issuer = lemur.plugins.lemur_cfssl.plugin:CfsslIssuerPlugin',