Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding additional failure conditions to sentry tracking. (#853) 

* Adding additional failure conditions to sentry tracking.

* Removing sentry extension as a circular import.
This commit is contained in:
kevgliss 2017-07-13 14:49:04 -07:00 committed by GitHub
parent 6779e19ac9
commit 61c493fc91
6 changed files with 54 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
lemur/certificates/cli.py
View File

@ -12,6 +12,7 @@ from flask import current_app
from flask_script import Manager
from lemur import database
from lemur.extensions import sentry
from lemur.extensions import metrics
from lemur.deployment import service as deployment_service
from lemur.endpoints import service as endpoint_service
@ -146,34 +147,36 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
print("[+] Starting endpoint rotation.")
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
endpoint = validate_endpoint(endpoint_name)
if endpoint and new_cert:
print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name))
request_rotation(endpoint, new_cert, message, commit)
elif old_cert and new_cert:
print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name))
for endpoint in old_cert.endpoints:
print("[+] Rotating {0}".format(endpoint.name))
if endpoint and new_cert:
print("[+] Rotating endpoint: {0} to certificate {1}".format(endpoint.name, new_cert.name))
request_rotation(endpoint, new_cert, message, commit)
else:
print("[+] Rotating all endpoints that have new certificates available")
for endpoint in endpoint_service.get_all_pending_rotation():
if len(endpoint.certificate.replaced) == 1:
print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name))
request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
else:
metrics.send('endpoint_rotation_failure', 'counter', 1)
print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format(
endpoint.name
))
elif old_cert and new_cert:
print("[+] Rotating all endpoints from {0} to {1}".format(old_cert.name, new_cert.name))
print("[+] Done!")
for endpoint in old_cert.endpoints:
print("[+] Rotating {0}".format(endpoint.name))
request_rotation(endpoint, new_cert, message, commit)
else:
print("[+] Rotating all endpoints that have new certificates available")
for endpoint in endpoint_service.get_all_pending_rotation():
if len(endpoint.certificate.replaced) == 1:
print("[+] Rotating {0} to {1}".format(endpoint.name, endpoint.certificate.replaced[0].name))
request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
else:
metrics.send('endpoint_rotation_failure', 'counter', 1)
print("[!] Failed to rotate endpoint {0} reason: Multiple replacement certificates found.".format(
endpoint.name
))
print("[+] Done!")
except Exception as e:
sentry.captureException()
@manager.option('-o', '--old-certificate', dest='old_certificate_name', help='Name of the certificate you wish to reissue.')
@ -201,6 +204,7 @@ def reissue(old_certificate_name, commit):
print("[+] Done!")
except Exception as e:
sentry.captureException()
metrics.send('certificate_reissue_failure', 'counter', 1)
print(
"[!] Failed to reissue certificate {0} reason: {1}".format(
@ -229,6 +233,7 @@ def check_revoked():
cert.status = 'valid' if status else 'revoked'
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
cert.status = 'unknown'

3
lemur/certificates/models.py
View File

@ -25,6 +25,7 @@ from sqlalchemy_utils.types.arrow import ArrowType
import lemur.common.utils
from lemur.database import db
from lemur.extensions import sentry
from lemur.utils import Vault
from lemur.common import defaults
@ -323,8 +324,10 @@ class Certificate(db.Model):
else:
current_app.logger.warning('Custom OIDs not yet supported for clone operation.')
except InvalidCodepoint as e:
sentry.captureException()
current_app.logger.warning('Unable to parse extensions due to underscore in dns name')
except ValueError as e:
sentry.captureException()
current_app.logger.warning('Unable to parse')
current_app.logger.exception(e)

3
lemur/common/schema.py
View File

@ -15,6 +15,8 @@ from sqlalchemy.orm.collections import InstrumentedList
from inflection import camelize, underscore
from marshmallow import Schema, post_dump, pre_load
from lemur.extensions import sentry
class LemurSchema(Schema):
"""
@ -157,6 +159,7 @@ def validate_schema(input_schema, output_schema):
try:
resp = f(*args, **kwargs)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
return dict(message=str(e)), 500

22
lemur/endpoints/cli.py
View File

@ -14,7 +14,7 @@ from sqlalchemy import cast
from sqlalchemy_utils import ArrowType
from lemur import database
from lemur.extensions import metrics
from lemur.extensions import metrics, sentry
from lemur.endpoints.models import Endpoint
@ -27,13 +27,17 @@ def expire(ttl):
Removed all endpoints that have not been recently updated.
"""
print("[+] Staring expiration of old endpoints.")
now = arrow.utcnow()
expiration = now - timedelta(hours=ttl)
endpoints = database.session_query(Endpoint).filter(cast(Endpoint.last_updated, ArrowType) <= expiration)
for endpoint in endpoints:
print("[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(name=endpoint.name, last_updated=endpoint.last_updated))
database.delete(endpoint)
metrics.send('endpoint_expired', 'counter', 1)
try:
now = arrow.utcnow()
expiration = now - timedelta(hours=ttl)
endpoints = database.session_query(Endpoint).filter(cast(Endpoint.last_updated, ArrowType) <= expiration)
print("[+] Finished expiration.")
for endpoint in endpoints:
print("[!] Expiring endpoint: {name} Last Updated: {last_updated}".format(name=endpoint.name, last_updated=endpoint.last_updated))
database.delete(endpoint)
metrics.send('endpoint_expired', 'counter', 1)
print("[+] Finished expiration.")
except Exception as e:
sentry.captureException()

3
lemur/notifications/messaging.py
View File

@ -18,6 +18,7 @@ from flask import current_app
from sqlalchemy import and_
from lemur import database, metrics
from lemur.extensions import sentry
from lemur.common.utils import windowed_query
from lemur.certificates.schemas import certificate_notification_output_schema
@ -98,6 +99,7 @@ def send_notification(event_type, data, targets, notification):
metrics.send('{0}_notification_sent'.format(event_type), 'counter', 1)
return True
except Exception as e:
sentry.captureException()
metrics.send('{0}_notification_failure'.format(event_type), 'counter', 1)
current_app.logger.exception(e)
@ -157,6 +159,7 @@ def send_rotation_notification(certificate, notification_plugin=None):
metrics.send('rotation_notification_sent', 'counter', 1)
return True
except Exception as e:
sentry.captureException()
metrics.send('rotation_notification_failure', 'counter', 1)
current_app.logger.exception(e)

4
lemur/sources/cli.py
View File

@ -14,7 +14,7 @@ from flask_script import Manager
from flask import current_app
from lemur.extensions import metrics
from lemur.extensions import metrics, sentry
from lemur.plugins.base import plugins
from lemur.sources import service as source_service
@ -87,6 +87,7 @@ def sync(source_strings):
)
metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label})
sentry.captureException()
@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
@ -117,6 +118,7 @@ def clean(source_strings, commit):
except Exception as e:
current_app.logger.exception(e)
metrics.send('clean_failed', 'counter', 1, metric_tags={'source': source.label})
sentry.captureException()
current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
certificate.name,