From 0d983bd2b5f2e0ea7fec565b70bfa8f423358236 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 15:39:36 -0700 Subject: [PATCH 1/5] missed edge case --- lemur/sources/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f69f70f5..f4783313 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -78,7 +78,7 @@ def sync_endpoints(source): source.label ) ) - return new, updated + return new, updated, updated_by_hash for endpoint in endpoints: exists = endpoint_service.get_by_dnsname_and_port( From f803fab41300ba2acba3634b5d5795072112d7b0 Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 6 Nov 2019 10:14:49 -0800 Subject: [PATCH 2/5] add plugin to send atlas metric via redis --- lemur/plugins/lemur_atlas_redis/__init__.py | 4 + lemur/plugins/lemur_atlas_redis/plugin.py | 97 +++++++++++++++++++++ setup.py | 1 + 3 files changed, 102 insertions(+) create mode 100644 lemur/plugins/lemur_atlas_redis/__init__.py create mode 100644 lemur/plugins/lemur_atlas_redis/plugin.py diff --git a/lemur/plugins/lemur_atlas_redis/__init__.py b/lemur/plugins/lemur_atlas_redis/__init__.py new file mode 100644 index 00000000..f8afd7e3 --- /dev/null +++ b/lemur/plugins/lemur_atlas_redis/__init__.py @@ -0,0 +1,4 @@ +try: + VERSION = __import__("pkg_resources").get_distribution(__name__).version +except Exception as e: + VERSION = "unknown" diff --git a/lemur/plugins/lemur_atlas_redis/plugin.py b/lemur/plugins/lemur_atlas_redis/plugin.py new file mode 100644 index 00000000..9a36743f --- /dev/null +++ b/lemur/plugins/lemur_atlas_redis/plugin.py @@ -0,0 +1,97 @@ +""" +.. module: lemur.plugins.lemur_atlas_redis.plugin + :platform: Unix + :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Jay Zarfoss +""" +from time import time +from redis import Redis +import json +from datetime import datetime + +from flask import current_app +from lemur.plugins import lemur_atlas as atlas +from lemur.plugins.bases.metric import MetricPlugin + + +def millis_since_epoch(): + """ + current time since epoch in milliseconds + """ + epoch = datetime.utcfromtimestamp(0) + delta = datetime.now() - epoch + return int(delta.total_seconds() * 1000.0) + + +class AtlasMetricRedisPlugin(MetricPlugin): + title = "AtlasRedis" + slug = "atlas-metric-Redius" + description = "Adds support for sending key metrics to Atlas via local Redis" + version = atlas.VERSION + + author = "Jay Zarfoss" + author_url = "https://github.com/netflix/lemur" + + options = [ + { + "name": "redis_host", + "type": "str", + "required": False, + "help_message": "If no host is provided localhost is assumed", + "default": "localhost", + }, + {"name": "redis_port", "type": "int", "required": False, "default": 28527}, + ] + + metric_data = {} + redis_host = None + redis_port = None + + def submit( + self, metric_name, metric_type, metric_value, metric_tags=None, options=None + ): + if not options: + options = self.options + + valid_types = ["COUNTER", "GAUGE", "TIMER"] + if metric_type.upper() not in valid_types: + raise Exception( + "Invalid Metric Type for Atlas: '{metric}' choose from: {options}".format( + metric=metric_type, options=",".join(valid_types) + ) + ) + + if metric_tags: + if not isinstance(metric_tags, dict): + raise Exception( + "Invalid Metric Tags for Atlas: Tags must be in dict format" + ) + + self.metric_data["timestamp"] = millis_since_epoch() + self.metric_data["type"] = metric_type.upper() + self.metric_data["name"] = str(metric_name) + self.metric_data["tags"] = metric_tags + + if ( + metric_value == "NaN" + or isinstance(metric_value, int) + or isinstance(metric_value, float) + ): + self.metric_data["value"] = metric_value + else: + raise Exception("Invalid Metric Value for Atlas: Metric must be a number") + + self.redis_host = self.get_option("redis_host", options) + self.redis_port = self.get_option("redis_port", options) + + try: + r = Redis(host=self.redis_host, port=self.redis_port, socket_timeout=0.1) + r.rpush('atlas-agent', json.dumps(self.metric_data)) + except Exception: + current_app.logger.warning( + "AtlasMetricsRedis: could not post atlas metrics to AtlasRedis {host}:{port}".format( + host=self.redis_host, port=self.redis_port + ) + ) diff --git a/setup.py b/setup.py index a01c110f..1c61e9f9 100644 --- a/setup.py +++ b/setup.py @@ -147,6 +147,7 @@ setup( 'java_keystore_export = lemur.plugins.lemur_jks.plugin:JavaKeystoreExportPlugin', 'openssl_export = lemur.plugins.lemur_openssl.plugin:OpenSSLExportPlugin', 'atlas_metric = lemur.plugins.lemur_atlas.plugin:AtlasMetricPlugin', + 'atlas_metric_redis = lemur.plugins.lemur_atlas.plugin:AtlasMetricRedisPlugin', 'kubernetes_destination = lemur.plugins.lemur_kubernetes.plugin:KubernetesDestinationPlugin', 'cryptography_issuer = lemur.plugins.lemur_cryptography.plugin:CryptographyIssuerPlugin', 'cfssl_issuer = lemur.plugins.lemur_cfssl.plugin:CfsslIssuerPlugin', From 113c9dd65744a4783e4ec2498797f2fe5e341061 Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 6 Nov 2019 10:42:59 -0800 Subject: [PATCH 3/5] atlas redis plugin typo cleanup and better exception handling --- lemur/plugins/lemur_atlas_redis/plugin.py | 10 +++++----- setup.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_atlas_redis/plugin.py b/lemur/plugins/lemur_atlas_redis/plugin.py index 9a36743f..e69ae672 100644 --- a/lemur/plugins/lemur_atlas_redis/plugin.py +++ b/lemur/plugins/lemur_atlas_redis/plugin.py @@ -6,7 +6,7 @@ .. moduleauthor:: Jay Zarfoss """ -from time import time + from redis import Redis import json from datetime import datetime @@ -27,7 +27,7 @@ def millis_since_epoch(): class AtlasMetricRedisPlugin(MetricPlugin): title = "AtlasRedis" - slug = "atlas-metric-Redius" + slug = "atlas-metric-redis" description = "Adds support for sending key metrics to Atlas via local Redis" version = atlas.VERSION @@ -89,9 +89,9 @@ class AtlasMetricRedisPlugin(MetricPlugin): try: r = Redis(host=self.redis_host, port=self.redis_port, socket_timeout=0.1) r.rpush('atlas-agent', json.dumps(self.metric_data)) - except Exception: + except Exception as e: current_app.logger.warning( - "AtlasMetricsRedis: could not post atlas metrics to AtlasRedis {host}:{port}".format( - host=self.redis_host, port=self.redis_port + "AtlasMetricsRedis: exception [{exception}] could not post atlas metrics to AtlasRedis [{host}:{port}], metric [{metricdata}]".format( + exception=e, host=self.redis_host, port=self.redis_port, metricdata=json.dumps(self.metric_data) ) ) diff --git a/setup.py b/setup.py index 1c61e9f9..90c0b2f8 100644 --- a/setup.py +++ b/setup.py @@ -147,7 +147,7 @@ setup( 'java_keystore_export = lemur.plugins.lemur_jks.plugin:JavaKeystoreExportPlugin', 'openssl_export = lemur.plugins.lemur_openssl.plugin:OpenSSLExportPlugin', 'atlas_metric = lemur.plugins.lemur_atlas.plugin:AtlasMetricPlugin', - 'atlas_metric_redis = lemur.plugins.lemur_atlas.plugin:AtlasMetricRedisPlugin', + 'atlas_metric_redis = lemur.plugins.lemur_atlas_redis.plugin:AtlasMetricRedisPlugin', 'kubernetes_destination = lemur.plugins.lemur_kubernetes.plugin:KubernetesDestinationPlugin', 'cryptography_issuer = lemur.plugins.lemur_cryptography.plugin:CryptographyIssuerPlugin', 'cfssl_issuer = lemur.plugins.lemur_cfssl.plugin:CfsslIssuerPlugin', From 00a0a27826f81ea7ca5859191dfaa8b7ef35efef Mon Sep 17 00:00:00 2001 From: Jay Zarfoss Date: Wed, 20 Nov 2019 09:44:31 -0800 Subject: [PATCH 4/5] used fixedName variable to transport db lookup optimization --- lemur/certificates/service.py | 3 +++ lemur/static/app/angular/certificates/view/view.js | 2 +- lemur/static/app/angular/certificates/view/view.tpl.html | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1a0cdc5a..0e91b563 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -393,6 +393,9 @@ def render(args): Certificate.cn.ilike(term), ) ) + elif "fixedName" in terms: + # only what matches the fixed name directly if a fixedname is provided + query = query.filter(Certificate.name == terms[1]) else: query = database.filter(query, Certificate, terms) diff --git a/lemur/static/app/angular/certificates/view/view.js b/lemur/static/app/angular/certificates/view/view.js index e4ae0314..0ee0d2c2 100644 --- a/lemur/static/app/angular/certificates/view/view.js +++ b/lemur/static/app/angular/certificates/view/view.js @@ -11,7 +11,7 @@ angular.module('lemur') controller: 'CertificatesViewController' }) .state('certificate', { - url: '/certificates/:name', + url: '/certificates/:fixedName', // use "fixedName" if in URL to indicate 'like' query can be avoided templateUrl: '/angular/certificates/view/view.tpl.html', controller: 'CertificatesViewController' }); diff --git a/lemur/static/app/angular/certificates/view/view.tpl.html b/lemur/static/app/angular/certificates/view/view.tpl.html index 9d5c7772..3f952aa2 100644 --- a/lemur/static/app/angular/certificates/view/view.tpl.html +++ b/lemur/static/app/angular/certificates/view/view.tpl.html @@ -52,7 +52,7 @@
- Permalink + Permalink From f188aea2c22c4342cb6d2610586be67590b43f86 Mon Sep 17 00:00:00 2001 From: Niels Bischof Date: Mon, 2 Dec 2019 06:22:09 -0500 Subject: [PATCH 5/5] type on quickstart/index.rst --- docs/quickstart/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 280bb612..01a5c7ca 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -31,7 +31,7 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver. npm is needed if you're installing Lemur from the source (e.g., from git). -.. note:: Installing node from a package manager may creat the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following +.. note:: Installing node from a package manager may create the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following sudo ln -s /user/bin/nodejs /usr/bin/node Now, install Python ``virtualenv`` package: