From e247d635fca923355db865382f55d54c2c0ed099 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Sat, 1 Aug 2015 15:29:34 -0700 Subject: [PATCH] Adding backend code for sources models --- lemur/__init__.py | 3 + lemur/certificates/models.py | 5 +- lemur/elbs/models.py | 44 --- lemur/elbs/service.py | 124 ------ lemur/elbs/views.py | 78 ---- lemur/listeners/__init__.py | 0 lemur/listeners/models.py | 42 -- lemur/listeners/service.py | 159 -------- lemur/listeners/views.py | 128 ------- lemur/manage.py | 8 +- lemur/migrations/versions/1ff763f5b80b_.py | 42 ++ lemur/migrations/versions/4c8915e461b3_.py | 41 ++ lemur/migrations/versions/4dc5ddd111b8_.py | 31 ++ lemur/models.py | 19 +- lemur/{elbs => sources}/__init__.py | 0 lemur/sources/models.py | 29 ++ lemur/sources/service.py | 107 ++++++ lemur/{certificates => sources}/sync.py | 2 +- lemur/sources/views.py | 359 ++++++++++++++++++ .../certificates/certificate/edit.tpl.html | 38 ++ .../certificate/notifications.tpl.html | 28 ++ .../notification/notification.js | 56 +++ .../notification/notification.tpl.html | 87 +++++ .../app/angular/notifications/services.js | 106 ++++++ .../app/angular/notifications/view/view.js | 96 +++++ .../angular/notifications/view/view.tpl.html | 52 +++ 26 files changed, 1096 insertions(+), 588 deletions(-) delete mode 100644 lemur/elbs/models.py delete mode 100644 lemur/elbs/service.py delete mode 100644 lemur/elbs/views.py delete mode 100644 lemur/listeners/__init__.py delete mode 100644 lemur/listeners/models.py delete mode 100644 lemur/listeners/service.py delete mode 100644 lemur/listeners/views.py create mode 100644 lemur/migrations/versions/1ff763f5b80b_.py create mode 100644 lemur/migrations/versions/4c8915e461b3_.py create mode 100644 lemur/migrations/versions/4dc5ddd111b8_.py rename lemur/{elbs => sources}/__init__.py (100%) create mode 100644 lemur/sources/models.py create mode 100644 lemur/sources/service.py rename lemur/{certificates => sources}/sync.py (98%) create mode 100644 lemur/sources/views.py create mode 100644 lemur/static/app/angular/certificates/certificate/edit.tpl.html create mode 100644 lemur/static/app/angular/certificates/certificate/notifications.tpl.html create mode 100644 lemur/static/app/angular/notifications/notification/notification.js create mode 100644 lemur/static/app/angular/notifications/notification/notification.tpl.html create mode 100644 lemur/static/app/angular/notifications/services.js create mode 100644 lemur/static/app/angular/notifications/view/view.js create mode 100644 lemur/static/app/angular/notifications/view/view.tpl.html diff --git a/lemur/__init__.py b/lemur/__init__.py index 43c7e0df..32d65e06 100644 --- a/lemur/__init__.py +++ b/lemur/__init__.py @@ -22,6 +22,8 @@ from lemur.certificates.views import mod as certificates_bp from lemur.status.views import mod as status_bp from lemur.plugins.views import mod as plugins_bp from lemur.notifications.views import mod as notifications_bp +from lemur.sources.views import mod as sources_bp + LEMUR_BLUEPRINTS = ( users_bp, @@ -36,6 +38,7 @@ LEMUR_BLUEPRINTS = ( status_bp, plugins_bp, notifications_bp, + sources_bp ) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index d876a3da..012b5566 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -23,7 +23,9 @@ from lemur.plugins.base import plugins from lemur.domains.models import Domain from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE -from lemur.models import certificate_associations, certificate_destination_associations, certificate_notification_associations + +from lemur.models import certificate_associations, certificate_source_associations, \ + certificate_destination_associations, certificate_notification_associations def create_name(issuer, not_before, not_after, subject, san): @@ -222,6 +224,7 @@ class Certificate(db.Model): authority_id = Column(Integer, ForeignKey('authorities.id')) notifications = relationship("Notification", secondary=certificate_notification_associations, backref='certificate') destinations = relationship("Destination", secondary=certificate_destination_associations, backref='certificate') + sources = relationship("Source", secondary=certificate_source_associations, backref='certificate') domains = relationship("Domain", secondary=certificate_associations, backref="certificate") elb_listeners = relationship("Listener", lazy='dynamic', backref='certificate') diff --git a/lemur/elbs/models.py b/lemur/elbs/models.py deleted file mode 100644 index d57a9f1f..00000000 --- a/lemur/elbs/models.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -.. module: lemur.elbs.models - :platform: Unix - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Kevin Glisson -""" -from sqlalchemy import Column, BigInteger, String, DateTime, PassiveDefault, func -from sqlalchemy.orm import relationship - -from lemur.database import db -from lemur.listeners.models import Listener - - -class ELB(db.Model): - __tablename__ = 'elbs' - id = Column(BigInteger, primary_key=True) - # account_id = Column(BigInteger, ForeignKey("accounts.id"), index=True) - region = Column(String(32)) - name = Column(String(128)) - vpc_id = Column(String(128)) - scheme = Column(String(128)) - dns_name = Column(String(128)) - listeners = relationship("Listener", backref='elb', cascade="all, delete, delete-orphan") - date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False) - - def __init__(self, elb_obj=None): - if elb_obj: - self.region = elb_obj.connection.region.name - self.name = elb_obj.name - self.vpc_id = elb_obj.vpc_id - self.scheme = elb_obj.scheme - self.dns_name = elb_obj.dns_name - for listener in elb_obj.listeners: - self.listeners.append(Listener(listener)) - - def as_dict(self): - return {c.name: getattr(self, c.name) for c in self.__table__.columns} - - def serialize(self): - blob = self.as_dict() - del blob['date_created'] - return blob diff --git a/lemur/elbs/service.py b/lemur/elbs/service.py deleted file mode 100644 index d00110bf..00000000 --- a/lemur/elbs/service.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -.. module: lemur.elbs.service - :platform: Unix - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Kevin Glisson - -""" -from sqlalchemy import func -from sqlalchemy.sql import and_ - -from lemur import database -from lemur.elbs.models import ELB -from lemur.listeners.models import Listener - - -def get_all(account_id, elb_name): - """ - Retrieves all ELBs in a given account - - :param account_id: - :param elb_name: - :rtype : Elb - :return: - """ - query = database.session_query(ELB) - return query.filter(and_(ELB.name == elb_name, ELB.account_id == account_id)).all() - - -def get_by_region_and_account(region, account_id): - query = database.session_query(ELB) - return query.filter(and_(ELB.region == region, ELB.account_id == account_id)).all() - - -def get_all_elbs(): - """ - Get all ELBs that Lemur knows about - - :rtype : list - :return: - """ - return ELB.query.all() - - -def get(elb_id): - """ - Retrieve an ELB with a give ID - - :rtype : Elb - :param elb_id: - :return: - """ - return database.get(ELB, elb_id) - - -def create(account, elb): - """ - Create a new ELB - - :param account: - :param elb: - """ - elb = ELB(elb) - account.elbs.append(elb) - database.create(elb) - - -def delete(elb_id): - """ - Delete an ELB - - :param elb_id: - """ - database.delete(get(elb_id)) - - -def render(args): - query = database.session_query(ELB) - - sort_by = args.pop('sort_by') - sort_dir = args.pop('sort_dir') - page = args.pop('page') - count = args.pop('count') - filt = args.pop('filter') - active = args.pop('active') - certificate_id = args.pop('certificate_id') - - if certificate_id: - query.filter(ELB.listeners.any(Listener.certificate_id == certificate_id)) - - if active == 'true': - query = query.filter(ELB.listeners.any()) - - if filt: - terms = filt.split(';') - query = database.filter(query, ELB, terms) - - query = database.find_all(query, ELB, args) - - if sort_by and sort_dir: - query = database.sort(query, ELB, sort_by, sort_dir) - - return database.paginate(query, page, count) - - -def stats(**kwargs): - attr = getattr(ELB, kwargs.get('metric')) - query = database.db.session.query(attr, func.count(attr)) - - if kwargs.get('account_id'): - query = query.filter(ELB.account_id == kwargs.get('account_id')) - - if kwargs.get('active') == 'true': - query = query.join(ELB.listeners) - query = query.filter(Listener.certificate_id != None) # noqa - - items = query.group_by(attr).all() - - results = [] - for key, count in items: - if key: - results.append({"key": key, "y": count}) - return results diff --git a/lemur/elbs/views.py b/lemur/elbs/views.py deleted file mode 100644 index 214d28e2..00000000 --- a/lemur/elbs/views.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -.. module: lemur.elbs.service - :platform: Unix - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Kevin Glisson - -""" -from flask import Blueprint -from flask.ext.restful import reqparse, Api, fields -from lemur.elbs import service -from lemur.auth.service import AuthenticatedResource - -from lemur.common.utils import marshal_items, paginated_parser - - -mod = Blueprint('elbs', __name__) -api = Api(mod) - - -FIELDS = { - 'name': fields.String, - 'id': fields.Integer, - 'region': fields.String, - 'scheme': fields.String, - 'accountId': fields.Integer(attribute='account_id'), - 'vpcId': fields.String(attribute='vpc_id') -} - - -class ELBsList(AuthenticatedResource): - """ Defines the 'elbs' endpoint """ - def __init__(self): - super(ELBsList, self).__init__() - - @marshal_items(FIELDS) - def get(self): - parser = paginated_parser.copy() - parser.add_argument('owner', type=str, location='args') - parser.add_argument('id', type=str, location='args') - parser.add_argument('accountId', type=str, dest='account_id', location='args') - parser.add_argument('certificateId', type=str, dest='certificate_id', location='args') - parser.add_argument('active', type=str, default='true', location='args') - - args = parser.parse_args() - return service.render(args) - - -class ELBsStats(AuthenticatedResource): - def __init__(self): - self.reqparse = reqparse.RequestParser() - super(ELBsStats, self).__init__() - - def get(self): - self.reqparse.add_argument('metric', type=str, location='args') - self.reqparse.add_argument('accountId', dest='account_id', location='args') - self.reqparse.add_argument('active', type=str, default='true', location='args') - - args = self.reqparse.parse_args() - - items = service.stats(**args) - return {"items": items, "total": len(items)} - - -class ELBs(AuthenticatedResource): - def __init__(self): - self.reqparse = reqparse.RequestParser() - super(ELBs, self).__init__() - - @marshal_items(FIELDS) - def get(self, elb_id): - return service.get(elb_id) - - -api.add_resource(ELBsList, '/elbs', endpoint='elbs') -api.add_resource(ELBs, '/elbs/', endpoint='elb') -api.add_resource(ELBsStats, '/elbs/stats', endpoint='elbsStats') diff --git a/lemur/listeners/__init__.py b/lemur/listeners/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/lemur/listeners/models.py b/lemur/listeners/models.py deleted file mode 100644 index b72c9b48..00000000 --- a/lemur/listeners/models.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -.. module: lemur.elbs.models - :platform: Unix - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Kevin Glisson - -""" -from sqlalchemy import Column, Integer, BigInteger, String, ForeignKey, DateTime, PassiveDefault, func - -from lemur.database import db -from lemur.certificates import service as cert_service -from lemur.certificates.models import Certificate, get_name_from_arn - - -class Listener(db.Model): - __tablename__ = 'listeners' - id = Column(BigInteger, primary_key=True) - certificate_id = Column(Integer, ForeignKey(Certificate.id), index=True) - elb_id = Column(BigInteger, ForeignKey("elbs.id"), index=True) - instance_port = Column(Integer) - instance_protocol = Column(String(16)) - load_balancer_port = Column(Integer) - load_balancer_protocol = Column(String(16)) - date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False) - - def __init__(self, listener): - self.load_balancer_port = listener.load_balancer_port - self.load_balancer_protocol = listener.protocol - self.instance_port = listener.instance_port - self.instance_protocol = listener.instance_protocol - if listener.ssl_certificate_id not in ["Invalid-Certificate", None]: - self.certificate_id = cert_service.get_by_name(get_name_from_arn(listener.ssl_certificate_id)).id - - def as_dict(self): - return {c.name: getattr(self, c.name) for c in self.__table__.columns} - - def serialize(self): - blob = self.as_dict() - del blob['date_created'] - return blob diff --git a/lemur/listeners/service.py b/lemur/listeners/service.py deleted file mode 100644 index 6f2ae596..00000000 --- a/lemur/listeners/service.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -.. module: lemur.listeners.service - :platform: Unix - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Kevin Glisson - -""" -from sqlalchemy import func - -from lemur import database - -from lemur.exceptions import CertificateUnavailable - -from lemur.elbs.models import ELB -from lemur.listeners.models import Listener -from lemur.elbs import service as elb_service -from lemur.certificates import service as certificate_service - -# from lemur.common.services.aws.elb import update_listeners, create_new_listeners, delete_listeners - - -def verify_attachment(certificate_id, elb_account_number): - """ - Ensures that the certificate we want ot attach to our listener is - in the same account as our listener. - - :rtype : Certificate - :param certificate_id: - :param elb_account_number: - :return: :raise CertificateUnavailable: - """ - cert = certificate_service.get(certificate_id) - - # we need to ensure that the specified cert is in our account - for account in cert.accounts: - if account.account_number == elb_account_number: - break - else: - raise CertificateUnavailable - return cert - - -def get(listener_id): - return database.get(Listener, listener_id) - - -def create(elb_id, instance_protocol, instance_port, load_balancer_port, load_balancer_protocol, certificate_id=None): - listener = Listener(elb_id, - instance_port, - instance_protocol, - load_balancer_port, - load_balancer_protocol - ) - - elb = elb_service.get(elb_id) - elb.listeners.append(listener) - account_number = elb.account.account_number - - cert = verify_attachment(certificate_id, account_number) - listener_tuple = (load_balancer_port, instance_port, load_balancer_protocol, cert.get_art(account_number),) - # create_new_listeners(account_number, elb.region, elb.name, [listener_tuple]) - - return {'message': 'Listener has been created'} - - -def update(listener_id, **kwargs): - listener = get(listener_id) - - # if the lb_port has changed we need to make sure we are deleting - # the listener on the old port to avoid listener duplication - ports = [] - if listener.load_balancer_port != kwargs.get('load_balancer_port'): - ports.append(listener.load_balancer_port) - else: - ports.append(kwargs.get('load_balancer_port')) - - certificate_id = kwargs.get('certificate_id') - - listener.instance_port = kwargs.get('instance_port') - listener.instance_protocol = kwargs.get('instance_protocol') - listener.load_balancer_port = kwargs.get('load_balancer_port') - listener.load_balancer_protocol = kwargs.get('load_balancer_protocol') - - elb = listener.elb - account_number = listener.elb.account.account_number - - arn = None - if certificate_id: - cert = verify_attachment(certificate_id, account_number) - cert.elb_listeners.append(listener) - arn = cert.get_arn(account_number) - - # remove certificate that is no longer wanted - if listener.certificate and not certificate_id: - listener.certificate.remove() - - database.update(listener) - listener_tuple = (listener.load_balancer_port, listener.instance_port, listener.load_balancer_protocol, arn,) - # update_listeners(account_number, elb.region, elb.name, [listener_tuple], ports) - - return {'message': 'Listener has been updated'} - - -def delete(listener_id): - # first try to delete the listener in aws - listener = get(listener_id) - # delete_listeners(listener.elb.account.account_number, listener.elb.region, listener.elb.name, [listener.load_balancer_port]) - # cleanup operation in lemur - database.delete(listener) - - -def render(args): - query = database.session_query(Listener) - - sort_by = args.pop('sort_by') - sort_dir = args.pop('sort_dir') - page = args.pop('page') - count = args.pop('count') - filt = args.pop('filter') - certificate_id = args.pop('certificate_id', None) - elb_id = args.pop('elb_id', None) - - if certificate_id: - query = database.get_all(Listener, certificate_id, field='certificate_id') - - if elb_id: - query = query.filter(Listener.elb_id == elb_id) - - if filt: - terms = filt.split(';') - query = database.filter(query, Listener, terms) - - query = database.find_all(query, Listener, args) - - if sort_by and sort_dir: - query = database.sort(query, Listener, sort_by, sort_dir) - - return database.paginate(query, page, count) - - -def stats(**kwargs): - attr = getattr(Listener, kwargs.get('metric')) - query = database.db.session.query(attr, func.count(attr)) - query = query.join(Listener.elb) - - if kwargs.get('account_id'): - query = query.filter(ELB.account_id == kwargs.get('account_id')) - - if kwargs.get('active') == 'true': - query = query.filter(Listener.certificate_id != None) # noqa - - items = query.group_by(attr).all() - results = [] - for key, count in items: - if key: - results.append({"key": key, "y": count}) - return results diff --git a/lemur/listeners/views.py b/lemur/listeners/views.py deleted file mode 100644 index b603d827..00000000 --- a/lemur/listeners/views.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -.. module: lemur.listeners.service - :platform: Unix - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. - -.. moduleauthor:: Kevin Glisson - -""" -from flask import Blueprint -from flask.ext.restful import reqparse, Api, fields - -from lemur.listeners import service -from lemur.auth.service import AuthenticatedResource -from lemur.auth.permissions import admin_permission -from lemur.common.utils import marshal_items, paginated_parser - - -mod = Blueprint('listeners', __name__) -api = Api(mod) - - -FIELDS = { - 'id': fields.Integer, - 'elbId': fields.Integer(attribute="elb_id"), - 'certificateId': fields.Integer(attribute="certificate_id"), - 'instancePort': fields.Integer(attribute="instance_port"), - 'instanceProtocol': fields.String(attribute="instance_protocol"), - 'loadBalancerPort': fields.Integer(attribute="load_balancer_port"), - 'loadBalancerProtocol': fields.String(attribute="load_balancer_protocol") -} - - -class ListenersList(AuthenticatedResource): - def __init__(self): - super(ListenersList, self).__init__() - - @marshal_items(FIELDS) - def get(self): - parser = paginated_parser.copy() - parser.add_argument('certificateId', type=int, dest='certificate_id', location='args') - args = parser.parse_args() - return service.render(args) - - -class ListenersCertificateList(AuthenticatedResource): - def __init__(self): - super(ListenersCertificateList, self).__init__() - - @marshal_items(FIELDS) - def get(self, certificate_id): - parser = paginated_parser.copy() - args = parser.parse_args() - args['certificate_id'] = certificate_id - return service.render(args) - - -class ListenersELBList(AuthenticatedResource): - def __init__(self): - super(ListenersELBList, self).__init__() - - @marshal_items(FIELDS) - def get(self, elb_id): - parser = paginated_parser.copy() - args = parser.parse_args() - args['elb_id'] = elb_id - return service.render(args) - - -class ListenersStats(AuthenticatedResource): - def __init__(self): - self.reqparse = reqparse.RequestParser() - super(ListenersStats, self).__init__() - - def get(self): - self.reqparse.add_argument('metric', type=str, location='args') - self.reqparse.add_argument('accountId', dest='account_id', location='args') - self.reqparse.add_argument('active', type=str, default='true', location='args') - - args = self.reqparse.parse_args() - - items = service.stats(**args) - return {"items": items, "total": len(items)} - - -class Listeners(AuthenticatedResource): - def __init__(self): - super(Listeners, self).__init__() - - @marshal_items(FIELDS) - def get(self, listener_id): - return service.get(listener_id) - - @admin_permission.require(http_exception=403) - @marshal_items(FIELDS) - def post(self): - self.reqparse.add_argument('elbId', type=str, dest='elb_id', required=True, location='json') - self.reqparse.add_argument('instanceProtocol', type=str, dest='instance_protocol', required=True, location='json') - self.reqparse.add_argument('instancePort', type=int, dest='instance_port', required=True, location='json') - self.reqparse.add_argument('loadBalancerProtocol', type=str, dest='load_balancer_protocol', required=True, location='json') - self.reqparse.add_argument('loadBalancerPort', type=int, dest='load_balancer_port', required=True, location='json') - self.reqparse.add_argument('certificateId', type=int, dest='certificate_id', location='json') - - args = self.reqparse.parse_args() - return service.create(**args) - - @admin_permission.require(http_exception=403) - @marshal_items(FIELDS) - def put(self, listener_id): - self.reqparse.add_argument('instanceProtocol', type=str, dest='instance_protocol', required=True, location='json') - self.reqparse.add_argument('instancePort', type=int, dest='instance_port', required=True, location='json') - self.reqparse.add_argument('loadBalancerProtocol', type=str, dest='load_balancer_protocol', required=True, location='json') - self.reqparse.add_argument('loadBalancerPort', type=int, dest='load_balancer_port', required=True, location='json') - self.reqparse.add_argument('certificateId', type=int, dest='certificate_id', location='json') - - args = self.reqparse.parse_args() - return service.update(listener_id, **args) - - @admin_permission.require(http_exception=403) - def delete(self, listener_id): - return service.delete(listener_id) - - -api.add_resource(ListenersList, '/listeners', endpoint='listeners') -api.add_resource(Listeners, '/listeners/', endpoint='listener') -api.add_resource(ListenersStats, '/listeners/stats', endpoint='listenersStats') -api.add_resource(ListenersCertificateList, '/certificates//listeners', endpoint='listenersCertificates') -api.add_resource(ListenersELBList, '/elbs//listeners', endpoint='elbListeners') diff --git a/lemur/manage.py b/lemur/manage.py index 21929980..df2dd120 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -19,7 +19,7 @@ from lemur.certificates import service as cert_service from lemur.plugins.base import plugins from lemur.certificates.verify import verify_string -from lemur.certificates import sync +from lemur.sources import sync from lemur import create_app @@ -33,6 +33,7 @@ from lemur.domains.models import Domain # noqa from lemur.elbs.models import ELB # noqa from lemur.listeners.models import Listener # noqa from lemur.notifications.models import Notification # noqa +from lemur.sources.models import Source # noqa manager = Manager(create_app) @@ -183,12 +184,11 @@ class Sync(Command): run on a periodic basis and updates the Lemur datastore with the information it discovers. """ + + # TODO create these commands dynamically option_list = [ Group( Option('-a', '--all', action="store_true"), - Option('-b', '--aws', action="store_true"), - Option('-d', '--cloudca', action="store_true"), - Option('-s', '--source', action="store_true"), exclusive=True, required=True ) ] diff --git a/lemur/migrations/versions/1ff763f5b80b_.py b/lemur/migrations/versions/1ff763f5b80b_.py new file mode 100644 index 00000000..e4a9af22 --- /dev/null +++ b/lemur/migrations/versions/1ff763f5b80b_.py @@ -0,0 +1,42 @@ +"""Adding in models for certificate sources + +Revision ID: 1ff763f5b80b +Revises: 4dc5ddd111b8 +Create Date: 2015-08-01 15:24:20.412725 + +""" + +# revision identifiers, used by Alembic. +revision = '1ff763f5b80b' +down_revision = '4dc5ddd111b8' + +from alembic import op +import sqlalchemy as sa + +import sqlalchemy_utils + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('sources', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('label', sa.String(length=32), nullable=True), + sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('plugin_name', sa.String(length=32), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('certificate_source_associations', + sa.Column('source_id', sa.Integer(), nullable=True), + sa.Column('certificate_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'), + sa.ForeignKeyConstraint(['source_id'], ['destinations.id'], ondelete='cascade') + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('certificate_source_associations') + op.drop_table('sources') + ### end Alembic commands ### diff --git a/lemur/migrations/versions/4c8915e461b3_.py b/lemur/migrations/versions/4c8915e461b3_.py new file mode 100644 index 00000000..f67d837f --- /dev/null +++ b/lemur/migrations/versions/4c8915e461b3_.py @@ -0,0 +1,41 @@ +"""Adding notifications + +Revision ID: 4c8915e461b3 +Revises: 3b718f59b8ce +Create Date: 2015-07-24 14:34:57.316273 + +""" + +# revision identifiers, used by Alembic. +revision = '4c8915e461b3' +down_revision = '3b718f59b8ce' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +import sqlalchemy_utils + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('notifications', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('label', sa.String(length=128), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True), + sa.Column('active', sa.Boolean(), nullable=True), + sa.Column('plugin_name', sa.String(length=32), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.drop_column(u'certificates', 'challenge') + op.drop_column(u'certificates', 'csr_config') + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column(u'certificates', sa.Column('csr_config', sa.TEXT(), autoincrement=False, nullable=True)) + op.add_column(u'certificates', sa.Column('challenge', postgresql.BYTEA(), autoincrement=False, nullable=True)) + op.drop_table('notifications') + ### end Alembic commands ### diff --git a/lemur/migrations/versions/4dc5ddd111b8_.py b/lemur/migrations/versions/4dc5ddd111b8_.py new file mode 100644 index 00000000..8330744d --- /dev/null +++ b/lemur/migrations/versions/4dc5ddd111b8_.py @@ -0,0 +1,31 @@ +"""Creating a one-to-many relationship for notifications + +Revision ID: 4dc5ddd111b8 +Revises: 4c8915e461b3 +Create Date: 2015-07-24 15:02:04.398262 + +""" + +# revision identifiers, used by Alembic. +revision = '4dc5ddd111b8' +down_revision = '4c8915e461b3' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('certificate_notification_associations', + sa.Column('notification_id', sa.Integer(), nullable=True), + sa.Column('certificate_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'), + sa.ForeignKeyConstraint(['notification_id'], ['notifications.id'], ondelete='cascade') + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('certificate_notification_associations') + ### end Alembic commands ### diff --git a/lemur/models.py b/lemur/models.py index 10ee07be..761c2da9 100644 --- a/lemur/models.py +++ b/lemur/models.py @@ -8,9 +8,7 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ - from sqlalchemy import Column, Integer, ForeignKey - from lemur.database import db certificate_associations = db.Table('certificate_associations', @@ -25,12 +23,19 @@ certificate_destination_associations = db.Table('certificate_destination_associa ForeignKey('certificates.id', ondelete='cascade')) ) +certificate_source_associations = db.Table('certificate_source_associations', + Column('source_id', Integer, + ForeignKey('destinations.id', ondelete='cascade')), + Column('certificate_id', Integer, + ForeignKey('certificates.id', ondelete='cascade')) + ) + certificate_notification_associations = db.Table('certificate_notification_associations', - Column('notification_id', Integer, - ForeignKey('notifications.id', ondelete='cascade')), - Column('certificate_id', Integer, - ForeignKey('certificates.id', ondelete='cascade')) - ) + Column('notification_id', Integer, + ForeignKey('notifications.id', ondelete='cascade')), + Column('certificate_id', Integer, + ForeignKey('certificates.id', ondelete='cascade')) + ) roles_users = db.Table('roles_users', Column('user_id', Integer, ForeignKey('users.id')), Column('role_id', Integer, ForeignKey('roles.id')) diff --git a/lemur/elbs/__init__.py b/lemur/sources/__init__.py similarity index 100% rename from lemur/elbs/__init__.py rename to lemur/sources/__init__.py diff --git a/lemur/sources/models.py b/lemur/sources/models.py new file mode 100644 index 00000000..85beaeea --- /dev/null +++ b/lemur/sources/models.py @@ -0,0 +1,29 @@ +""" +.. module: lemur.sources.models + :platform: unix + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +import copy +from sqlalchemy import Column, Integer, String, Text +from sqlalchemy_utils import JSONType +from lemur.database import db + +from lemur.plugins.base import plugins + + +class Source(db.Model): + __tablename__ = 'sources' + id = Column(Integer, primary_key=True) + label = Column(String(32)) + options = Column(JSONType) + description = Column(Text()) + plugin_name = Column(String(32)) + + @property + def plugin(self): + p = plugins.get(self.plugin_name) + c = copy.deepcopy(p) + c.options = self.options + return c diff --git a/lemur/sources/service.py b/lemur/sources/service.py new file mode 100644 index 00000000..dd7eaa1a --- /dev/null +++ b/lemur/sources/service.py @@ -0,0 +1,107 @@ +""" +.. module: lemur.sources.service + :platform: Unix + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from lemur import database +from lemur.sources.models import Source +from lemur.certificates.models import Certificate + + +def create(label, plugin_name, options, description=None): + """ + Creates a new source, that can then be used as a source for certificates. + + :param label: Source common name + :param description: + :rtype : Source + :return: New source + """ + source = Source(label=label, options=options, plugin_name=plugin_name, description=description) + return database.create(source) + + +def update(source_id, label, options, description): + """ + Updates an existing source. + + :param source_id: Lemur assigned ID + :param label: Source common name + :rtype : Source + :return: + """ + source = get(source_id) + + source.label = label + source.options = options + source.description = description + + return database.update(source) + + +def delete(source_id): + """ + Deletes an source. + + :param source_id: Lemur assigned ID + """ + database.delete(get(source_id)) + + +def get(source_id): + """ + Retrieves an source by it's lemur assigned ID. + + :param source_id: Lemur assigned ID + :rtype : Source + :return: + """ + return database.get(Source, source_id) + + +def get_by_label(label): + """ + Retrieves a source by it's label + + :param label: + :return: + """ + return database.get(Source, label, field='label') + + +def get_all(): + """ + Retrieves all source currently known by Lemur. + + :return: + """ + query = database.session_query(Source) + return database.find_all(query, Source, {}).all() + + +def render(args): + sort_by = args.pop('sort_by') + sort_dir = args.pop('sort_dir') + page = args.pop('page') + count = args.pop('count') + filt = args.pop('filter') + certificate_id = args.pop('certificate_id', None) + + if certificate_id: + query = database.session_query(Source).join(Certificate, Source.certificate) + query = query.filter(Certificate.id == certificate_id) + else: + query = database.session_query(Source) + + if filt: + terms = filt.split(';') + query = database.filter(query, Source, terms) + + query = database.find_all(query, Source, args) + + if sort_by and sort_dir: + query = database.sort(query, Source, sort_by, sort_dir) + + return database.paginate(query, page, count) diff --git a/lemur/certificates/sync.py b/lemur/sources/sync.py similarity index 98% rename from lemur/certificates/sync.py rename to lemur/sources/sync.py index b91af6b1..5b37897f 100644 --- a/lemur/certificates/sync.py +++ b/lemur/sources/sync.py @@ -1,5 +1,5 @@ """ -.. module: sync +.. module: lemur.sources.sync :platform: Unix :synopsis: This module contains various certificate syncing operations. Because of the nature of the SSL environment there are multiple ways diff --git a/lemur/sources/views.py b/lemur/sources/views.py new file mode 100644 index 00000000..807054cc --- /dev/null +++ b/lemur/sources/views.py @@ -0,0 +1,359 @@ +""" +.. module: lemur.sources.views + :platform: Unix + :synopsis: This module contains all of the accounts view code. + :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from flask import Blueprint +from flask.ext.restful import Api, reqparse, fields +from lemur.sources import service + +from lemur.auth.service import AuthenticatedResource +from lemur.auth.permissions import admin_permission +from lemur.common.utils import paginated_parser, marshal_items + + +mod = Blueprint('sources', __name__) +api = Api(mod) + + +FIELDS = { + 'description': fields.String, + 'sourceOptions': fields.Raw(attribute='options'), + 'pluginName': fields.String(attribute='plugin_name'), + 'label': fields.String, + 'id': fields.Integer, +} + + +class SourcesList(AuthenticatedResource): + """ Defines the 'sources' endpoint """ + def __init__(self): + self.reqparse = reqparse.RequestParser() + super(SourcesList, self).__init__() + + @marshal_items(FIELDS) + def get(self): + """ + .. http:get:: /sources + + The current account list + + **Example request**: + + .. sourcecode:: http + + GET /sources HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + { + "items": [ + { + "sourceOptions": [ + { + "name": "accountNumber", + "required": true, + "value": 111111111112, + "helpMessage": "Must be a valid AWS account number!", + "validation": "/^[0-9]{12,12}$/", + "type": "int" + } + ], + "pluginName": "aws-source", + "id": 3, + "description": "test", + "label": "test" + } + ], + "total": 1 + } + + :query sortBy: field to sort on + :query sortDir: acs or desc + :query page: int. default is 1 + :query filter: key value pair. format is k=v; + :query limit: limit number. default is 10 + :reqheader Authorization: OAuth token to authenticate + :statuscode 200: no error + """ + parser = paginated_parser.copy() + args = parser.parse_args() + return service.render(args) + + @admin_permission.require(http_exception=403) + @marshal_items(FIELDS) + def post(self): + """ + .. http:post:: /sources + + Creates a new account + + **Example request**: + + .. sourcecode:: http + + POST /sources HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + { + "sourceOptions": [ + { + "name": "accountNumber", + "required": true, + "value": 111111111112, + "helpMessage": "Must be a valid AWS account number!", + "validation": "/^[0-9]{12,12}$/", + "type": "int" + } + ], + "pluginName": "aws-source", + "id": 3, + "description": "test", + "label": "test" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + { + "sourceOptions": [ + { + "name": "accountNumber", + "required": true, + "value": 111111111112, + "helpMessage": "Must be a valid AWS account number!", + "validation": "/^[0-9]{12,12}$/", + "type": "int" + } + ], + "pluginName": "aws-source", + "id": 3, + "description": "test", + "label": "test" + } + + :arg label: human readable account label + :arg description: some description about the account + :reqheader Authorization: OAuth token to authenticate + :statuscode 200: no error + """ + self.reqparse.add_argument('label', type=str, location='json', required=True) + self.reqparse.add_argument('plugin', type=dict, location='json', required=True) + self.reqparse.add_argument('description', type=str, location='json') + + args = self.reqparse.parse_args() + return service.create(args['label'], args['plugin']['slug'], args['plugin']['pluginOptions'], args['description']) + + +class Sources(AuthenticatedResource): + def __init__(self): + self.reqparse = reqparse.RequestParser() + super(Sources, self).__init__() + + @marshal_items(FIELDS) + def get(self, source_id): + """ + .. http:get:: /sources/1 + + Get a specific account + + **Example request**: + + .. sourcecode:: http + + GET /sources/1 HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + { + "sourceOptions": [ + { + "name": "accountNumber", + "required": true, + "value": 111111111112, + "helpMessage": "Must be a valid AWS account number!", + "validation": "/^[0-9]{12,12}$/", + "type": "int" + } + ], + "pluginName": "aws-source", + "id": 3, + "description": "test", + "label": "test" + } + + :reqheader Authorization: OAuth token to authenticate + :statuscode 200: no error + """ + return service.get(source_id) + + @admin_permission.require(http_exception=403) + @marshal_items(FIELDS) + def put(self, source_id): + """ + .. http:put:: /sources/1 + + Updates an account + + **Example request**: + + .. sourcecode:: http + + POST /sources/1 HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + { + "sourceOptions": [ + { + "name": "accountNumber", + "required": true, + "value": 111111111112, + "helpMessage": "Must be a valid AWS account number!", + "validation": "/^[0-9]{12,12}$/", + "type": "int" + } + ], + "pluginName": "aws-source", + "id": 3, + "description": "test", + "label": "test" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + { + "sourceOptions": [ + { + "name": "accountNumber", + "required": true, + "value": 111111111112, + "helpMessage": "Must be a valid AWS account number!", + "validation": "/^[0-9]{12,12}$/", + "type": "int" + } + ], + "pluginName": "aws-source", + "id": 3, + "description": "test", + "label": "test" + } + + :arg accountNumber: aws account number + :arg label: human readable account label + :arg description: some description about the account + :reqheader Authorization: OAuth token to authenticate + :statuscode 200: no error + """ + self.reqparse.add_argument('label', type=str, location='json', required=True) + self.reqparse.add_argument('plugin', type=dict, location='json', required=True) + self.reqparse.add_argument('description', type=str, location='json') + + args = self.reqparse.parse_args() + return service.update(source_id, args['label'], args['plugin']['pluginOptions'], args['description']) + + @admin_permission.require(http_exception=403) + def delete(self, source_id): + service.delete(source_id) + return {'result': True} + + +class CertificateSources(AuthenticatedResource): + """ Defines the 'certificate/', endpoint='account') +api.add_resource(CertificateSources, '/certificates//sources', + endpoint='certificateSources') diff --git a/lemur/static/app/angular/certificates/certificate/edit.tpl.html b/lemur/static/app/angular/certificates/certificate/edit.tpl.html new file mode 100644 index 00000000..a3cf3888 --- /dev/null +++ b/lemur/static/app/angular/certificates/certificate/edit.tpl.html @@ -0,0 +1,38 @@ + diff --git a/lemur/static/app/angular/certificates/certificate/notifications.tpl.html b/lemur/static/app/angular/certificates/certificate/notifications.tpl.html new file mode 100644 index 00000000..5f8136ac --- /dev/null +++ b/lemur/static/app/angular/certificates/certificate/notifications.tpl.html @@ -0,0 +1,28 @@ +
+ +
+
+ + + + +
+ + + + + + +
{{ notification.label }}{{ notification.description }} + +
+
+
diff --git a/lemur/static/app/angular/notifications/notification/notification.js b/lemur/static/app/angular/notifications/notification/notification.js new file mode 100644 index 00000000..a8135c22 --- /dev/null +++ b/lemur/static/app/angular/notifications/notification/notification.js @@ -0,0 +1,56 @@ +'use strict'; + +angular.module('lemur') + + .controller('NotificationsCreateController', function ($scope, $modalInstance, PluginService, NotificationService, CertificateService, LemurRestangular){ + $scope.notification = LemurRestangular.restangularizeElement(null, {}, 'notifications'); + + PluginService.getByType('notification').then(function (plugins) { + $scope.plugins = plugins; + }); + $scope.save = function (notification) { + NotificationService.create(notification).then( + function () { + $modalInstance.close(); + }, + function () { + + } + ); + }; + + $scope.cancel = function () { + $modalInstance.dismiss('cancel'); + }; + + $scope.certificateService = CertificateService; + }) + + .controller('NotificationsEditController', function ($scope, $modalInstance, NotificationService, NotificationApi, PluginService, CertificateService, editId) { + NotificationApi.get(editId).then(function (notification) { + $scope.notification = notification; + NotificationService.getCertificates(notification); + }); + + PluginService.getByType('notification').then(function (plugins) { + $scope.plugins = plugins; + _.each($scope.plugins, function (plugin) { + if (plugin.slug == $scope.notification.pluginName) { + plugin.pluginOptions = $scope.notification.notificationOptions; + $scope.notification.plugin = plugin; + }; + }); + }); + + $scope.save = function (notification) { + NotificationService.update(notification).then(function () { + $modalInstance.close(); + }); + }; + + $scope.cancel = function () { + $modalInstance.dismiss('cancel'); + }; + + $scope.certificateService = CertificateService; + }); diff --git a/lemur/static/app/angular/notifications/notification/notification.tpl.html b/lemur/static/app/angular/notifications/notification/notification.tpl.html new file mode 100644 index 00000000..8289ae5b --- /dev/null +++ b/lemur/static/app/angular/notifications/notification/notification.tpl.html @@ -0,0 +1,87 @@ + + diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js new file mode 100644 index 00000000..9bb6affa --- /dev/null +++ b/lemur/static/app/angular/notifications/services.js @@ -0,0 +1,106 @@ +'use strict'; +angular.module('lemur') + .service('NotificationApi', function (LemurRestangular) { + LemurRestangular.extendModel('notifications', function (obj) { + return angular.extend(obj, { + attachCertificate: function (certificate) { + this.selectedCertificate = null; + if (this.certificates === undefined) { + this.certificates = []; + } + this.certificates.push(certificate); + }, + removeCertificate: function (index) { + this.certificate.splice(index, 1); + } + }); + }); + return LemurRestangular.all('notifications'); + }) + .service('NotificationService', function ($location, NotificationApi, PluginService, toaster) { + var NotificationService = this; + NotificationService.findNotificationsByName = function (filterValue) { + return NotificationApi.getList({'filter[label]': filterValue}) + .then(function (notifications) { + return notifications; + }); + }; + + NotificationService.getCertificates = function (notification) { + notification.getList('certificates').then(function (certificates) { + notification.certificates = certificates; + }); + }; + + NotificationService.getPlugin = function (notification) { + return PluginService.getByName(notification.pluginName).then(function (plugin) { + notification.plugin = plugin; + }); + }; + + + NotificationService.loadMoreCertificates = function (notification, page) { + notification.getList('certificates', {page: page}).then(function (certificates) { + _.each(certificates, function (certificate) { + notification.roles.push(certificate); + }); + }); + }; + + NotificationService.create = function (notification) { + return NotificationApi.post(notification).then( + function () { + toaster.pop({ + type: 'success', + title: notification.label, + body: 'Successfully created!' + }); + $location.path('notifications'); + }, + function (response) { + toaster.pop({ + type: 'error', + title: notification.label, + body: 'Was not created! ' + response.data.message + }); + }); + }; + + NotificationService.update = function (notification) { + return notification.put().then( + function () { + toaster.pop({ + type: 'success', + title: notification.label, + body: 'Successfully updated!' + }); + $location.path('notifications'); + }, + function (response) { + toaster.pop({ + type: 'error', + title: notification.label, + body: 'Was not updated! ' + response.data.message + }); + }); + }; + + NotificationService.updateActive = function (notification) { + notification.put().then( + function () { + toaster.pop({ + type: 'success', + title: notification.name, + body: 'Successfully updated!' + }); + }, + function (response) { + toaster.pop({ + type: 'error', + title: notification.name, + body: 'Was not updated! ' + response.data.message + }); + }); + }; + return NotificationService; + }); diff --git a/lemur/static/app/angular/notifications/view/view.js b/lemur/static/app/angular/notifications/view/view.js new file mode 100644 index 00000000..ae269e67 --- /dev/null +++ b/lemur/static/app/angular/notifications/view/view.js @@ -0,0 +1,96 @@ +'use strict'; + +angular.module('lemur') + + .config(function config($routeProvider) { + $routeProvider.when('/notifications', { + templateUrl: '/angular/notifications/view/view.tpl.html', + controller: 'NotificationsViewController' + }); + }) + + .controller('NotificationsViewController', function ($q, $scope, $modal, NotificationApi, NotificationService, ngTableParams, toaster) { + $scope.filter = {}; + $scope.notificationsTable = new ngTableParams({ + page: 1, // show first page + count: 10, // count per page + sorting: { + id: 'desc' // initial sorting + }, + filter: $scope.filter + }, { + total: 0, // length of data + getData: function ($defer, params) { + NotificationApi.getList(params.url()).then( + function (data) { + _.each(data, function (notification) { + NotificationService.getPlugin(notification); + }); + params.total(data.total); + $defer.resolve(data); + } + ); + } + }); + + $scope.getNotificationStatus = function () { + var def = $q.defer(); + def.resolve([{'title': 'Active', 'id': true}, {'title': 'Inactive', 'id': false}]); + return def; + }; + + $scope.remove = function (notification) { + notification.remove().then( + function () { + $scope.notificationsTable.reload(); + }, + function (response) { + toaster.pop({ + type: 'error', + title: 'Opps', + body: 'I see what you did there' + response.data.message + }); + } + ); + }; + + $scope.edit = function (notificationId) { + var modalInstance = $modal.open({ + animation: true, + templateUrl: '/angular/notifications/notification/notification.tpl.html', + controller: 'NotificationsEditController', + size: 'lg', + resolve: { + editId: function () { + return notificationId; + } + } + }); + + modalInstance.result.then(function () { + $scope.notificationsTable.reload(); + }); + + }; + + $scope.create = function () { + var modalInstance = $modal.open({ + animation: true, + controller: 'NotificationsCreateController', + templateUrl: '/angular/notifications/notification/notification.tpl.html', + size: 'lg' + }); + + modalInstance.result.then(function () { + $scope.notificationsTable.reload(); + }); + + }; + + $scope.toggleFilter = function (params) { + params.settings().$scope.show_filter = !params.settings().$scope.show_filter; + }; + + $scope.notificationService = NotificationService; + + }); diff --git a/lemur/static/app/angular/notifications/view/view.tpl.html b/lemur/static/app/angular/notifications/view/view.tpl.html new file mode 100644 index 00000000..1335e5e3 --- /dev/null +++ b/lemur/static/app/angular/notifications/view/view.tpl.html @@ -0,0 +1,52 @@ +
+
+

Notifications + you have to speak up son!

+
+
+
+ +
+
+ +
+
+
+
+ + + + + + + + + +
+
    +
  • {{ notification.label }}
  • +
  • {{ notification.description }}
  • +
+
+
    +
  • {{ notification.plugin.title }}
  • +
  • {{ notification.plugin.description }}
  • +
+
+
+ +
+
+
+ + +
+
+
+
+
+