From c6747439fbb1423f8c487777fbe2a775d66b667c Mon Sep 17 00:00:00 2001 From: kevgliss Date: Tue, 18 Aug 2015 16:17:20 -0700 Subject: [PATCH 1/6] Misc fixed around certificate syncing --- gulp/server.js | 5 ++++- lemur/database.py | 6 ++---- lemur/manage.py | 8 +++----- lemur/notifications/service.py | 5 ++++- lemur/plugins/lemur_aws/plugin.py | 7 ++++++- lemur/plugins/lemur_cloudca/plugin.py | 6 +++--- lemur/plugins/lemur_email/plugin.py | 4 ++-- lemur/plugins/lemur_email/templates/config.py | 5 +++-- lemur/sources/service.py | 1 + .../angular/destinations/destination/destination.js | 10 ---------- 10 files changed, 28 insertions(+), 29 deletions(-) diff --git a/gulp/server.js b/gulp/server.js index 7ee20381..777100f6 100644 --- a/gulp/server.js +++ b/gulp/server.js @@ -27,7 +27,10 @@ function browserSyncInit(baseDir, files, browser) { browserSync.instance = browserSync.init(files, { startPath: '/index.html', server: { - baseDir: baseDir + baseDir: baseDir, + routes: { + '/bower_components': './bower_components' + } }, browser: browser, ghostMode: false diff --git a/lemur/database.py b/lemur/database.py index 22a464ff..20077435 100644 --- a/lemur/database.py +++ b/lemur/database.py @@ -9,10 +9,9 @@ .. moduleauthor:: Kevin Glisson """ -from flask import current_app - from sqlalchemy import exc from sqlalchemy.sql import and_, or_ +from sqlalchemy.orm.exc import NoResultFound from lemur.extensions import db from lemur.exceptions import AttrNotFound, DuplicateError @@ -126,8 +125,7 @@ def get(model, value, field="id"): query = session_query(model) try: return query.filter(getattr(model, field) == value).one() - except Exception as e: - current_app.logger.exception(e) + except NoResultFound as e: return diff --git a/lemur/manage.py b/lemur/manage.py index 8cee39e0..1b53c591 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -77,7 +77,6 @@ LEMUR_RESTRICTED_DOMAINS = [] LEMUR_EMAIL = '' LEMUR_SECURITY_TEAM_EMAIL = [] -LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2] # Logging @@ -172,18 +171,17 @@ def generate_settings(): @manager.option('-s', '--sources', dest='labels', default='', required=False) -@manager.option('-l', '--list', dest='view', default=False, required=False) -def sync_sources(labels, view): +def sync_sources(labels): """ Attempts to run several methods Certificate discovery. This is run on a periodic basis and updates the Lemur datastore with the information it discovers. """ - if view: + if not labels: sys.stdout.write("Active\tLabel\tDescription\n") for source in source_service.get_all(): sys.stdout.write( - "[{active}]\t{label}\t{description}!\n".format( + "{active}\t{label}\t{description}!\n".format( label=source.label, description=source.description, active=source.active diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 4f3ba6c1..e6e3ddf1 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -38,7 +38,10 @@ def _get_message_data(cert): :return: """ cert_dict = cert.as_dict() - cert_dict['creator'] = cert.user.email + + if cert.user: + cert_dict['creator'] = cert.user.email + cert_dict['domains'] = [x .name for x in cert.domains] cert_dict['superseded'] = list(set([x.name for x in _find_superseded(cert) if cert.name != x])) return cert_dict diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 0c6fc09a..fe45d8b1 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -6,6 +6,7 @@ .. moduleauthor:: Kevin Glisson """ +from boto.exception import BotoServerError from lemur.plugins.bases import DestinationPlugin, SourcePlugin from lemur.plugins.lemur_aws import iam, elb from lemur.plugins import lemur_aws as aws @@ -42,7 +43,11 @@ class AWSDestinationPlugin(DestinationPlugin): # } def upload(self, name, body, private_key, cert_chain, options, **kwargs): - iam.upload_cert(find_value('accountNumber', options), name, body, private_key, cert_chain=cert_chain) + try: + iam.upload_cert(find_value('accountNumber', options), name, body, private_key, cert_chain=cert_chain) + except BotoServerError as e: + if e.error_code != 'EntityAlreadyExists': + raise Exception(e) e = find_value('elb', options) if e: diff --git a/lemur/plugins/lemur_cloudca/plugin.py b/lemur/plugins/lemur_cloudca/plugin.py index 0281e9d0..2d26666a 100644 --- a/lemur/plugins/lemur_cloudca/plugin.py +++ b/lemur/plugins/lemur_cloudca/plugin.py @@ -326,11 +326,11 @@ class CloudCASourcePlugin(SourcePlugin, CloudCA): 'pollRate': {'type': 'int', 'default': '60'} } - def get_certificates(self, **kwargs): + def get_certificates(self, options, **kwargs): certs = [] for authority in self.get_authorities(): certs += self.get_cert(ca_name=authority) - return + return certs def get_cert(self, ca_name=None, cert_handle=None): """ @@ -355,7 +355,7 @@ class CloudCASourcePlugin(SourcePlugin, CloudCA): certs.append({ 'public_certificate': cert, - 'intermediate_cert': "\n".join(intermediates), + 'intermediate_certificate': "\n".join(intermediates), 'owner': c['ownerEmail'] }) diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 90c53a67..19dd50fe 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -55,10 +55,10 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): template = env.get_template('{}.html'.format(event_type)) body = template.render(**kwargs) - s_type = current_app.config.get("LEMUR_EMAIL_SENDER").lower() + s_type = current_app.config.get("LEMUR_EMAIL_SENDER", 'ses').lower() if s_type == 'ses': conn = boto.connect_ses() - conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, targets, format='html') + conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, ['kglisson@netflix.com'], format='html') elif s_type == 'smtp': msg = Message(subject, recipients=targets) diff --git a/lemur/plugins/lemur_email/templates/config.py b/lemur/plugins/lemur_email/templates/config.py index 160fc146..49729cc7 100644 --- a/lemur/plugins/lemur_email/templates/config.py +++ b/lemur/plugins/lemur_email/templates/config.py @@ -1,4 +1,5 @@ -from jinja2 import Environment, PackageLoader +import os +from jinja2 import Environment, FileSystemLoader -loader = PackageLoader('lemur') +loader = FileSystemLoader(searchpath=os.path.dirname(os.path.realpath(__file__))) env = Environment(loader=loader) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index b097696e..b1370867 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -39,6 +39,7 @@ def _disassociate_certs_from_source(current_certificates, found_certificates, so def sync_create(certificate, source): cert = cert_service.import_certificate(**certificate) + cert.description = "This certificate was automatically discovered by Lemur" cert.sources.append(source) sync_update_destination(cert, source) database.update(cert) diff --git a/lemur/static/app/angular/destinations/destination/destination.js b/lemur/static/app/angular/destinations/destination/destination.js index 321eecfb..2681a154 100644 --- a/lemur/static/app/angular/destinations/destination/destination.js +++ b/lemur/static/app/angular/destinations/destination/destination.js @@ -34,16 +34,6 @@ angular.module('lemur') }); }); - PluginService.getByType('destination').then(function (plugins) { - $scope.plugins = plugins; - _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.destination.pluginName) { - plugin.pluginOptions = $scope.destination.destinationOptions; - $scope.destination.plugin = plugin; - } - }); - }); - $scope.save = function (destination) { DestinationService.update(destination).then(function () { $modalInstance.close(); From 28e12a973f1bac8c8f11db96ee64abbd0edeaa39 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Wed, 19 Aug 2015 10:06:30 -0700 Subject: [PATCH 2/6] Misc fixed around certificate notifications --- lemur/notifications/service.py | 13 +++++--- lemur/plugins/lemur_email/plugin.py | 4 +-- .../lemur_email/templates/expiration.html | 31 +++++++++---------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index e6e3ddf1..3c944580 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -59,13 +59,18 @@ def _deduplicate(messages): for m, r, o in roll_ups: if r == targets: - m.append(data) - current_app.logger.info( - "Sending expiration alert about {0} to {1}".format( - data['name'], ",".join(targets))) + for cert in m: + if cert['body'] == data['body']: + break + else: + m.append(data) + current_app.logger.info( + "Sending expiration alert about {0} to {1}".format( + data['name'], ",".join(targets))) break else: roll_ups.append(([data], targets, options)) + return roll_ups diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 19dd50fe..f2166d78 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -53,12 +53,12 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): # jinja template depending on type template = env.get_template('{}.html'.format(event_type)) - body = template.render(**kwargs) + body = template.render(dict(messages=message, hostname=current_app.config.get('LEMUR_HOSTNAME'))) s_type = current_app.config.get("LEMUR_EMAIL_SENDER", 'ses').lower() if s_type == 'ses': conn = boto.connect_ses() - conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, ['kglisson@netflix.com'], format='html') + conn.send_email(current_app.config.get("LEMUR_EMAIL"), subject, body, targets, format='html') elif s_type == 'smtp': msg = Message(subject, recipients=targets) diff --git a/lemur/plugins/lemur_email/templates/expiration.html b/lemur/plugins/lemur_email/templates/expiration.html index f5459496..f3584bd2 100644 --- a/lemur/plugins/lemur_email/templates/expiration.html +++ b/lemur/plugins/lemur_email/templates/expiration.html @@ -52,8 +52,13 @@ Notice: Your SSL certificates are expiring!
- Lemur, Netflix's SSL management portal has noticed that the following certificates are expiring soon, if you rely on these certificates - you should create new certificates to replace the certificates that are expiring. Visit https://lemur.netflix.com/#/certificates/create to reissue them. +

+ Lemur, Netflix's SSL management portal has noticed that the following certificates are expiring soon, if you rely on these certificates + you should create new certificates to replace the certificates that are expiring. +

+

+ Visit https://{{ hostname }}/#/certificates/create to reissue them. +

{% for message in messages %} @@ -78,6 +83,12 @@ {{ message.creator }} + + Description + + + {{ message.description }} + Not Before @@ -104,20 +115,6 @@ Unknown {% endif %} - - Associated ELBs - - {% if message.listeners %} - {% for name in message.listeners %} - - {{ name }} - - {% endfor %} - {% else %} - - None - - {% endif %} Potentially Superseded by (Lemur's best guess) @@ -139,7 +136,7 @@ - Lemur is broken regularly by Security Operations + Lemur is broken regularly by Netflix From b96af3a1f17a74fefc1f4eb531f769752f60688f Mon Sep 17 00:00:00 2001 From: kevgliss Date: Wed, 19 Aug 2015 10:10:19 -0700 Subject: [PATCH 3/6] Editing footer text --- lemur/static/app/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/static/app/index.html b/lemur/static/app/index.html index 46dcd3a9..ed3514bf 100644 --- a/lemur/static/app/index.html +++ b/lemur/static/app/index.html @@ -87,7 +87,7 @@ From b00917aa60b65b94dacdcf6774886902316e186d Mon Sep 17 00:00:00 2001 From: kevgliss Date: Wed, 19 Aug 2015 15:46:10 -0700 Subject: [PATCH 4/6] Ensure there are no accidental newlines when fetching the ENCRYPTION_KEY --- lemur/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/utils.py b/lemur/utils.py index d59d235d..1ea73759 100644 --- a/lemur/utils.py +++ b/lemur/utils.py @@ -15,6 +15,6 @@ def get_key(): :return: """ try: - return current_app.config.get('LEMUR_ENCRYPTION_KEY') + return current_app.config.get('LEMUR_ENCRYPTION_KEY').strip() except RuntimeError: return '' From cbcc8af3bde644896da8cb17a48a329ee0cc5346 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Wed, 19 Aug 2015 16:42:56 -0700 Subject: [PATCH 5/6] Fixing bug were domains would not have correct pagination --- lemur/static/app/angular/domains/view/view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/static/app/angular/domains/view/view.js b/lemur/static/app/angular/domains/view/view.js index a153305e..335b9c5a 100644 --- a/lemur/static/app/angular/domains/view/view.js +++ b/lemur/static/app/angular/domains/view/view.js @@ -21,7 +21,7 @@ angular.module('lemur') }, { total: 0, // length of data getData: function ($defer, params) { - DomainApi.getList().then(function (data) { + DomainApi.getList(params.url()).then(function (data) { params.total(data.total); $defer.resolve(data); }); From 6b2da2fe6be471818bfaf5dfa440ae8dbfd6c0c6 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Wed, 19 Aug 2015 18:05:18 -0700 Subject: [PATCH 6/6] Fixes #35 --- lemur/auth/permissions.py | 16 ++++--------- lemur/auth/service.py | 3 +-- lemur/certificates/views.py | 15 ++++++------ .../app/angular/certificates/services.js | 23 +++++++++++-------- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/lemur/auth/permissions.py b/lemur/auth/permissions.py index 79950b36..c07119d4 100644 --- a/lemur/auth/permissions.py +++ b/lemur/auth/permissions.py @@ -16,25 +16,19 @@ operator_permission = Permission(RoleNeed('operator')) admin_permission = Permission(RoleNeed('admin')) CertificateCreator = namedtuple('certificate', ['method', 'value']) -CertificateCreatorNeed = partial(CertificateCreator, 'certificateView') - -CertificateOwner = namedtuple('certificate', ['method', 'value']) -CertificateOwnerNeed = partial(CertificateOwner, 'certificateView') +CertificateCreatorNeed = partial(CertificateCreator, 'key') class ViewKeyPermission(Permission): - def __init__(self, certificate_id, owner_id): + def __init__(self, certificate_id, owner): c_need = CertificateCreatorNeed(str(certificate_id)) - o_need = CertificateOwnerNeed(str(owner_id)) - - super(ViewKeyPermission, self).__init__(o_need, c_need, RoleNeed('admin')) + super(ViewKeyPermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin')) class UpdateCertificatePermission(Permission): - def __init__(self, role_id, certificate_id): + def __init__(self, certificate_id, owner): c_need = CertificateCreatorNeed(str(certificate_id)) - o_need = CertificateOwnerNeed(str(role_id)) - super(UpdateCertificatePermission, self).__init__(o_need, c_need, RoleNeed('admin')) + super(UpdateCertificatePermission, self).__init__(c_need, RoleNeed(owner), RoleNeed('admin')) RoleUser = namedtuple('role', ['method', 'value']) diff --git a/lemur/auth/service.py b/lemur/auth/service.py index 6f4e29b5..ba19c509 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -29,7 +29,7 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicNumbers from lemur.users import service as user_service -from lemur.auth.permissions import CertificateOwnerNeed, CertificateCreatorNeed, \ +from lemur.auth.permissions import CertificateCreatorNeed, \ AuthorityCreatorNeed, ViewRoleCredentialsNeed @@ -165,7 +165,6 @@ def on_identity_loaded(sender, identity): # identity with the roles that the user provides if hasattr(user, 'roles'): for role in user.roles: - identity.provides.add(CertificateOwnerNeed(role.id)) identity.provides.add(ViewRoleCredentialsNeed(role.id)) identity.provides.add(RoleNeed(role.name)) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index eaf0c53d..3e10b7fb 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -446,13 +446,14 @@ class CertificatePrivateKey(AuthenticatedResource): role = role_service.get_by_name(cert.owner) - permission = ViewKeyPermission(certificate_id, hasattr(role, 'id')) + if role: + permission = ViewKeyPermission(certificate_id, role.name) - if permission.can(): - response = make_response(jsonify(key=cert.private_key), 200) - response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store' - response.headers['pragma'] = 'no-cache' - return response + if permission.can(): + response = make_response(jsonify(key=cert.private_key), 200) + response.headers['cache-control'] = 'private, max-age=0, no-cache, no-store' + response.headers['pragma'] = 'no-cache' + return response return dict(message='You are not authorized to view this key'), 403 @@ -572,7 +573,7 @@ class Certificates(AuthenticatedResource): cert = service.get(certificate_id) role = role_service.get_by_name(cert.owner) - permission = UpdateCertificatePermission(certificate_id, hasattr(role, 'id')) + permission = UpdateCertificatePermission(certificate_id, role.name) if permission.can(): return service.update( diff --git a/lemur/static/app/angular/certificates/services.js b/lemur/static/app/angular/certificates/services.js index 9344a3e9..a6d1ba7a 100644 --- a/lemur/static/app/angular/certificates/services.js +++ b/lemur/static/app/angular/certificates/services.js @@ -107,7 +107,6 @@ angular.module('lemur') title: certificate.name, body: 'Successfully created!' }); - $location.path('/certificates'); }, function (response) { toaster.pop({ @@ -120,14 +119,21 @@ angular.module('lemur') }; CertificateService.update = function (certificate) { - return LemurRestangular.copy(certificate).put().then(function () { - toaster.pop({ - type: 'success', - title: certificate.name, - body: 'Successfully updated!' + return LemurRestangular.copy(certificate).put().then( + function () { + toaster.pop({ + type: 'success', + title: certificate.name, + body: 'Successfully updated!' + }); + }, + function (response) { + toaster.pop({ + type: 'error', + title: certificate.name, + body: 'Failed to update ' + response.data.message + }); }); - $location.path('certificates'); - }); }; CertificateService.upload = function (certificate) { @@ -138,7 +144,6 @@ angular.module('lemur') title: certificate.name, body: 'Successfully uploaded!' }); - $location.path('/certificates'); }, function (response) { toaster.pop({