From 429e6a967c4133e2da0b0e576abb9537b810f33d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 25 Jul 2019 18:49:19 -0700 Subject: [PATCH 1/2] better error handling for redis --- lemur/common/redis.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lemur/common/redis.py b/lemur/common/redis.py index 34a8778f..ca15734f 100644 --- a/lemur/common/redis.py +++ b/lemur/common/redis.py @@ -3,7 +3,9 @@ Helper Class for Redis """ import redis +import sys from flask import current_app +from lemur.extensions import sentry from lemur.factory import create_app if current_app: @@ -23,7 +25,19 @@ class RedisHandler: def redis(self, db=0): # The decode_responses flag here directs the client to convert the responses from Redis into Python strings # using the default encoding utf-8. This is client specific. - red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True) + function = f"{__name__}.{sys._getframe().f_code.co_name}" + try: + red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True) + red.set("test", 0) + except redis.ConnectionError: + log_data = { + "function": function, + "message": "Redis Connection error", + "host": self.host, + "port": self.port + } + current_app.logger.error(log_data) + sentry.captureException() return red From adabe18c905cb4a9fe4a44f3a9e710e07cdf63d4 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 25 Jul 2019 18:56:28 -0700 Subject: [PATCH 2/2] metric tags, to be able to track which domains where failing during the LetsEncrypt domain validation --- lemur/plugins/lemur_acme/dyn.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 00a48eb6..fff2e632 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -33,22 +33,22 @@ def get_dynect_session(): return dynect_session -def _has_dns_propagated(name, token): +def _has_dns_propagated(fqdn, token): txt_records = [] try: dns_resolver = dns.resolver.Resolver() - dns_resolver.nameservers = [get_authoritative_nameserver(name)] - dns_response = dns_resolver.query(name, "TXT") + dns_resolver.nameservers = [get_authoritative_nameserver(fqdn)] + dns_response = dns_resolver.query(fqdn, "TXT") for rdata in dns_response: for txt_record in rdata.strings: txt_records.append(txt_record.decode("utf-8")) except dns.exception.DNSException: - metrics.send("has_dns_propagated_fail", "counter", 1) + metrics.send("has_dns_propagated_fail", "counter", 1, metric_tags={"dns": fqdn}) return False for txt_record in txt_records: if txt_record == token: - metrics.send("has_dns_propagated_success", "counter", 1) + metrics.send("has_dns_propagated_success", "counter", 1, metric_tags={"dns": fqdn}) return True return False @@ -61,12 +61,12 @@ def wait_for_dns_change(change_id, account_number=None): status = _has_dns_propagated(fqdn, token) current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status)) if status: - metrics.send("wait_for_dns_change_success", "counter", 1) + metrics.send("wait_for_dns_change_success", "counter", 1, metric_tags={"dns": fqdn}) break time.sleep(10) if not status: # TODO: Delete associated DNS text record here - metrics.send("wait_for_dns_change_fail", "counter", 1) + metrics.send("wait_for_dns_change_fail", "counter", 1, metric_tags={"dns": fqdn}) sentry.captureException(extra={"fqdn": str(fqdn), "txt_record": str(token)}) metrics.send( "wait_for_dns_change_error",