From 61c493fc9196a61bb48b07ceb719d6bc36f12ee9 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Thu, 13 Jul 2017 14:49:04 -0700 Subject: [PATCH] Adding additional failure conditions to sentry tracking. (#853) * Adding additional failure conditions to sentry tracking. * Removing sentry extension as a circular import. --- lemur/certificates/cli.py | 53 +++++++++++++++++--------------- lemur/certificates/models.py | 3 ++ lemur/common/schema.py | 3 ++ lemur/endpoints/cli.py | 22 +++++++------ lemur/notifications/messaging.py | 3 ++ lemur/sources/cli.py | 4 ++- 6 files changed, 54 insertions(+), 34 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index d30225ff..4038fb49 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -12,6 +12,7 @@ from flask import current_app from flask_script import Manager from lemur import database +from lemur.extensions import sentry from lemur.extensions import metrics from lemur.deployment import service as deployment_service from lemur.endpoints import service as endpoint_service @@ -146,34 +147,36 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c print("[+] Starting endpoint rotation.") - old_cert = validate_certificate(old_certificate_name) - new_cert = validate_certificate(new_certificate_name) - endpoint = validate_endpoint(endpoint_name) + try: + old_cert = validate_certificate(old_certificate_name) + new_cert = validate_certificate(new_certificate_name) + endpoint = validate_endpoint(endpoint_name) - if endpoint and new_cert: - print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name)) - request_rotation(endpoint, new_cert, message, commit) - - elif old_cert and new_cert: - print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name)) - - for endpoint in old_cert.endpoints: - print("[+] Rotating {0}".format(endpoint.name)) + if endpoint and new_cert: + print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name)) request_rotation(endpoint, new_cert, message, commit) - else: - print("[+] Rotating all endpoints that have new certificates available") - for endpoint in endpoint_service.get_all_pending_rotation(): - if len(endpoint.certificate.replaced) == 1: - print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name)) - request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) - else: - metrics.send('endpoint_rotation_failure', 'counter', 1) - print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format( - endpoint.name - )) + elif old_cert and new_cert: + print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name)) - print("[+] Done!") + for endpoint in old_cert.endpoints: + print("[+] Rotating {0}".format(endpoint.name)) + request_rotation(endpoint, new_cert, message, commit) + + else: + print("[+] Rotating all endpoints that have new certificates available") + for endpoint in endpoint_service.get_all_pending_rotation(): + if len(endpoint.certificate.replaced) == 1: + print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name)) + request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) + else: + metrics.send('endpoint_rotation_failure', 'counter', 1) + print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format( + endpoint.name + )) + print("[+] Done!") + except Exception as e: + sentry.captureException() @manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to reissue.') @@ -201,6 +204,7 @@ def reissue(old_certificate_name, commit): print("[+] Done!") except Exception as e: + sentry.captureException() metrics.send('certificate_reissue_failure', 'counter', 1) print( "[!] Failed to reissue certificate {0} reason: {1}".format( @@ -229,6 +233,7 @@ def check_revoked(): cert.status = 'valid' if status else 'revoked' except Exception as e: + sentry.captureException() current_app.logger.exception(e) cert.status = 'unknown' diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 59c9e90a..8c0a56d5 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -25,6 +25,7 @@ from sqlalchemy_utils.types.arrow import ArrowType import lemur.common.utils from lemur.database import db +from lemur.extensions import sentry from lemur.utils import Vault from lemur.common import defaults @@ -323,8 +324,10 @@ class Certificate(db.Model): else: current_app.logger.warning('Custom OIDs not yet supported for clone operation.') except InvalidCodepoint as e: + sentry.captureException() current_app.logger.warning('Unable to parse extensions due to underscore in dns name') except ValueError as e: + sentry.captureException() current_app.logger.warning('Unable to parse') current_app.logger.exception(e) diff --git a/lemur/common/schema.py b/lemur/common/schema.py index e01081a8..1e081f8c 100644 --- a/lemur/common/schema.py +++ b/lemur/common/schema.py @@ -15,6 +15,8 @@ from sqlalchemy.orm.collections import InstrumentedList from inflection import camelize, underscore from marshmallow import Schema, post_dump, pre_load +from lemur.extensions import sentry + class LemurSchema(Schema): """ @@ -157,6 +159,7 @@ def validate_schema(input_schema, output_schema): try: resp = f(*args, **kwargs) except Exception as e: + sentry.captureException() current_app.logger.exception(e) return dict(message=str(e)), 500 diff --git a/lemur/endpoints/cli.py b/lemur/endpoints/cli.py index 689461b1..0f576808 100644 --- a/lemur/endpoints/cli.py +++ b/lemur/endpoints/cli.py @@ -14,7 +14,7 @@ from sqlalchemy import cast from sqlalchemy_utils import ArrowType from lemur import database -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.endpoints.models import Endpoint @@ -27,13 +27,17 @@ def expire(ttl): Removed all endpoints that have not been recently updated. """ print("[+] Staring expiration of old endpoints.") - now = arrow.utcnow() - expiration = now - timedelta(hours=ttl) - endpoints = database.session_query(Endpoint).filter(cast(Endpoint.last_updated, ArrowType) <= expiration) - for endpoint in endpoints: - print("[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(name=endpoint.name, last_updated=endpoint.last_updated)) - database.delete(endpoint) - metrics.send('endpoint_expired', 'counter', 1) + try: + now = arrow.utcnow() + expiration = now - timedelta(hours=ttl) + endpoints = database.session_query(Endpoint).filter(cast(Endpoint.last_updated, ArrowType) <= expiration) - print("[+] Finished expiration.") + for endpoint in endpoints: + print("[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(name=endpoint.name, last_updated=endpoint.last_updated)) + database.delete(endpoint) + metrics.send('endpoint_expired', 'counter', 1) + + print("[+] Finished expiration.") + except Exception as e: + sentry.captureException() diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index b287fefd..590fcad2 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -18,6 +18,7 @@ from flask import current_app from sqlalchemy import and_ from lemur import database, metrics +from lemur.extensions import sentry from lemur.common.utils import windowed_query from lemur.certificates.schemas import certificate_notification_output_schema @@ -98,6 +99,7 @@ def send_notification(event_type, data, targets, notification): metrics.send('{0}_notification_sent'.format(event_type), 'counter', 1) return True except Exception as e: + sentry.captureException() metrics.send('{0}_notification_failure'.format(event_type), 'counter', 1) current_app.logger.exception(e) @@ -157,6 +159,7 @@ def send_rotation_notification(certificate, notification_plugin=None): metrics.send('rotation_notification_sent', 'counter', 1) return True except Exception as e: + sentry.captureException() metrics.send('rotation_notification_failure', 'counter', 1) current_app.logger.exception(e) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index 78c5a0b1..03c8c50d 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -14,7 +14,7 @@ from flask_script import Manager from flask import current_app -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.plugins.base import plugins from lemur.sources import service as source_service @@ -87,6 +87,7 @@ def sync(source_strings): ) metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label}) + sentry.captureException() @manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') @@ -117,6 +118,7 @@ def clean(source_strings, commit): except Exception as e: current_app.logger.exception(e) metrics.send('clean_failed', 'counter', 1, metric_tags={'source': source.label}) + sentry.captureException() current_app.logger.warning("Removed {0} from source {1} during cleaning".format( certificate.name,