Marsmallowing sources (#310)

lemur/sources/schemas.py

@@ -0,0 +1,32 @@
+"""
+.. module: lemur.sources.schemas
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+
+from lemur.schemas import PluginSchema
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+
+
+class SourceInputSchema(LemurInputSchema):
+    label = fields.String(required=True)
+    options = fields.Dict(load_from='sourceOptions', required=True)
+    description = fields.String()
+    plugin = fields.Nested(PluginSchema)
+    active = fields.Boolean()
+
+
+class SourceOutputSchema(LemurOutputSchema):
+    label = fields.String()
+    options = fields.Dict(dump_to='sourceOptions')
+    description = fields.String()
+    plugin = fields.Nested(PluginSchema)
+    fields.Boolean()
+
+
+source_input_schema = SourceInputSchema()
+sources_output_schema = SourceOutputSchema(many=True)
+source_output_schema = SourceOutputSchema()

lemur/sources/service.py

@@ -174,10 +174,6 @@ def get_all():
 
 
 def render(args):
-    sort_by = args.pop('sort_by')
-    sort_dir = args.pop('sort_dir')
-    page = args.pop('page')
-    count = args.pop('count')
     filt = args.pop('filter')
     certificate_id = args.pop('certificate_id', None)
 
@@ -191,9 +187,4 @@ def render(args):
         terms = filt.split(';')
         query = database.filter(query, Source, terms)
 
-    query = database.find_all(query, Source, args)
+    return database.sort_and_page(query, Source, args)
-
-    if sort_by and sort_dir:
-        query = database.sort(query, Source, sort_by, sort_dir)
-
-    return database.paginate(query, page, count)

lemur/sources/views.py

@@ -7,35 +7,28 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import Blueprint
-from flask.ext.restful import Api, reqparse, fields
+from flask.ext.restful import Api, reqparse
 from lemur.sources import service
 
+from lemur.common.schema import validate_schema
+from lemur.sources.schemas import source_input_schema, source_output_schema, sources_output_schema
+
 from lemur.auth.service import AuthenticatedResource
 from lemur.auth.permissions import admin_permission
-from lemur.common.utils import paginated_parser, marshal_items
+from lemur.common.utils import paginated_parser
 
 
 mod = Blueprint('sources', __name__)
 api = Api(mod)
 
 
-FIELDS = {
-    'description': fields.String,
-    'sourceOptions': fields.Raw(attribute='options'),
-    'pluginName': fields.String(attribute='plugin_name'),
-    'lastRun': fields.DateTime(attribute='last_run', dt_format='iso8061'),
-    'label': fields.String,
-    'id': fields.Integer,
-}
-
-
 class SourcesList(AuthenticatedResource):
     """ Defines the 'sources' endpoint """
     def __init__(self):
         self.reqparse = reqparse.RequestParser()
         super(SourcesList, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, sources_output_schema)
     def get(self):
         """
         .. http:get:: /sources
@@ -94,8 +87,8 @@ class SourcesList(AuthenticatedResource):
         return service.render(args)
 
     @admin_permission.require(http_exception=403)
-    @marshal_items(FIELDS)
+    @validate_schema(source_input_schema, source_output_schema)
-    def post(self):
+    def post(self, data=None):
         """
         .. http:post:: /sources
 
@@ -158,12 +151,7 @@ class SourcesList(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
+        return service.create(data['label'], data['plugin']['slug'], data['plugin']['pluginOptions'], data['description'])
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-
-        args = self.reqparse.parse_args()
-        return service.create(args['label'], args['plugin']['slug'], args['plugin']['pluginOptions'], args['description'])
 
 
 class Sources(AuthenticatedResource):
@@ -171,7 +159,7 @@ class Sources(AuthenticatedResource):
         self.reqparse = reqparse.RequestParser()
         super(Sources, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, source_output_schema)
     def get(self, source_id):
         """
         .. http:get:: /sources/1
@@ -218,8 +206,8 @@ class Sources(AuthenticatedResource):
         return service.get(source_id)
 
     @admin_permission.require(http_exception=403)
-    @marshal_items(FIELDS)
+    @validate_schema(source_input_schema, source_output_schema)
-    def put(self, source_id):
+    def put(self, source_id, data=None):
         """
         .. http:put:: /sources/1
 
@@ -283,12 +271,7 @@ class Sources(AuthenticatedResource):
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
         """
-        self.reqparse.add_argument('label', type=str, location='json', required=True)
+        return service.update(source_id, data['label'], data['plugin']['pluginOptions'], data['description'])
-        self.reqparse.add_argument('plugin', type=dict, location='json', required=True)
-        self.reqparse.add_argument('description', type=str, location='json')
-
-        args = self.reqparse.parse_args()
-        return service.update(source_id, args['label'], args['plugin']['pluginOptions'], args['description'])
 
     @admin_permission.require(http_exception=403)
     def delete(self, source_id):
@@ -301,7 +284,7 @@ class CertificateSources(AuthenticatedResource):
     def __init__(self):
         super(CertificateSources, self).__init__()
 
-    @marshal_items(FIELDS)
+    @validate_schema(None, sources_output_schema)
     def get(self, certificate_id):
         """
         .. http:get:: /certificates/1/sources

lemur/static/app/index.html

@@ -56,8 +56,8 @@
                     <li><a ui-sref="destinations">Destinations</a></li>
                     <li><a ui-sref="sources">Sources</a></li>
                     <li></li>
-                    <li class="dropdown" dropdown on-toggle="toggled(open)">
+                    <li class="dropdown" uib-dropdown on-toggle="toggled(open)">
-                        <a href class="dropdown-toggle" dropdown-toggle>Settings <span class="caret"></span></a>
+                        <a href class="dropdown-toggle" uib-dropdown-toggle>Settings <span class="caret"></span></a>
                         <ul class="dropdown-menu">
                             <li><a ui-sref="roles">Roles</a></li>
                             <li><a ui-sref="users">Users</a></li>

lemur/tests/test_sources.py

@@ -1,134 +1,108 @@
-from lemur.sources.service import *  # noqa
+import pytest
+
 from lemur.sources.views import *  # noqa
 
-from json import dumps
+from .vectors import VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN
 
 
-def test_crud(session):
+def validate_source_schema(client):
-    source = create('testdest', 'aws-source', {}, description='source1')
+    from lemur.sources.schemas import SourceInputSchema
-    assert source.id > 0
 
-    source = update(source.id, 'testdest2', {}, 'source2')
+    input_data = {
-    assert source.label == 'testdest2'
+        'label': 'exampleSource',
+        'options': {},
+        'plugin': {'slug': 'aws-source'}
+    }
 
-    assert len(get_all()) == 1
+    data, errors = SourceInputSchema().load(input_data)
-
+    assert not errors
-    delete(1)
-    assert len(get_all()) == 0
 
 
-def test_source_get(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.get(api.url_for(Sources, source_id=1)).status_code == 401
+    (VALID_USER_HEADER_TOKEN, 404),
+    (VALID_ADMIN_HEADER_TOKEN, 404),
+    ('', 401)
+])
+def test_source_get(client, token, status):
+    assert client.get(api.url_for(Sources, source_id=1), headers=token).status_code == status
 
 
-def test_source_post(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.post(api.url_for(Sources, source_id=1), data={}).status_code == 405
+    (VALID_USER_HEADER_TOKEN, 405),
+    (VALID_ADMIN_HEADER_TOKEN, 405),
+    ('', 405)
+])
+def test_source_post_(client, token, status):
+    assert client.post(api.url_for(Sources, source_id=1), data={}, headers=token).status_code == status
 
 
-def test_source_put(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.put(api.url_for(Sources, source_id=1), data={}).status_code == 401
+    (VALID_USER_HEADER_TOKEN, 403),
+    (VALID_ADMIN_HEADER_TOKEN, 400),
+    ('', 401)
+])
+def test_source_put(client, token, status):
+    assert client.put(api.url_for(Sources, source_id=1), data={}, headers=token).status_code == status
 
 
-def test_source_delete(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.delete(api.url_for(Sources, source_id=1)).status_code == 401
+    (VALID_USER_HEADER_TOKEN, 403),
+    (VALID_ADMIN_HEADER_TOKEN, 200),
+    ('', 401)
+])
+def test_source_delete(client, token, status):
+    assert client.delete(api.url_for(Sources, source_id=1), headers=token).status_code == status
 
 
-def test_source_patch(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.patch(api.url_for(Sources, source_id=1), data={}).status_code == 405
+    (VALID_USER_HEADER_TOKEN, 405),
+    (VALID_ADMIN_HEADER_TOKEN, 405),
+    ('', 405)
+])
+def test_source_patch(client, token, status):
+    assert client.patch(api.url_for(Sources, source_id=1), data={}, headers=token).status_code == status
 
 
-VALID_USER_HEADER_TOKEN = {
+@pytest.mark.parametrize("token,status", [
-    'Authorization': 'Basic ' + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE0MzUyMzMzNjksInN1YiI6MSwiZXhwIjoxNTIxNTQ2OTY5fQ.1qCi0Ip7mzKbjNh0tVd3_eJOrae3rNa_9MCVdA4WtQI'}
+    (VALID_USER_HEADER_TOKEN, 200),
+    (VALID_ADMIN_HEADER_TOKEN, 200),
+    ('', 401)
+])
+def test_sources_list_get(client, token, status):
+    assert client.get(api.url_for(SourcesList), headers=token).status_code == status
 
 
-def test_auth_source_get(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.get(api.url_for(Sources, source_id=1), headers=VALID_USER_HEADER_TOKEN).status_code == 200
+    (VALID_USER_HEADER_TOKEN, 403),
+    (VALID_ADMIN_HEADER_TOKEN, 400),
+    ('', 401)
+])
+def test_sources_list_post(client, token, status):
+    assert client.post(api.url_for(SourcesList), data={}, headers=token).status_code == status
 
 
-def test_auth_source_post_(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.post(api.url_for(Sources, source_id=1), data={}, headers=VALID_USER_HEADER_TOKEN).status_code == 405
+    (VALID_USER_HEADER_TOKEN, 405),
+    (VALID_ADMIN_HEADER_TOKEN, 405),
+    ('', 405)
+])
+def test_sources_list_put(client, token, status):
+    assert client.put(api.url_for(SourcesList), data={}, headers=token).status_code == status
 
 
-def test_auth_source_put(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.put(api.url_for(Sources, source_id=1), data={}, headers=VALID_USER_HEADER_TOKEN).status_code == 403
+    (VALID_USER_HEADER_TOKEN, 405),
+    (VALID_ADMIN_HEADER_TOKEN, 405),
+    ('', 405)
+])
+def test_sources_list_delete(client, token, status):
+    assert client.delete(api.url_for(SourcesList), headers=token).status_code == status
 
 
-def test_auth_source_delete(client):
+@pytest.mark.parametrize("token,status", [
-    assert client.delete(api.url_for(Sources, source_id=1), headers=VALID_USER_HEADER_TOKEN).status_code == 403
+    (VALID_USER_HEADER_TOKEN, 405),
-
+    (VALID_ADMIN_HEADER_TOKEN, 405),
-
+    ('', 405)
-def test_auth_source_patch(client):
+])
-    assert client.patch(api.url_for(Sources, source_id=1), data={}, headers=VALID_USER_HEADER_TOKEN).status_code == 405
+def test_sources_list_patch(client, token, status):
-
+    assert client.patch(api.url_for(SourcesList), data={}, headers=token).status_code == status
-
-VALID_ADMIN_HEADER_TOKEN = {
-    'Authorization': 'Basic ' + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE0MzUyNTAyMTgsInN1YiI6MiwiZXhwIjoxNTIxNTYzODE4fQ.6mbq4-Ro6K5MmuNiTJBB153RDhlM5LGJBjI7GBKkfqA'}
-
-
-def test_admin_source_get(client):
-    assert client.get(api.url_for(Sources, source_id=1), headers=VALID_ADMIN_HEADER_TOKEN).status_code == 200
-
-
-def test_admin_source_post(client):
-    assert client.post(api.url_for(Sources, source_id=1), data={}, headers=VALID_ADMIN_HEADER_TOKEN).status_code == 405
-
-
-def test_admin_source_put(client):
-    assert client.put(api.url_for(Sources, source_id=1), data={}, headers=VALID_ADMIN_HEADER_TOKEN).status_code == 400
-
-
-def test_admin_source_delete(client):
-    assert client.delete(api.url_for(Sources, source_id=1), headers=VALID_ADMIN_HEADER_TOKEN).status_code == 200
-
-
-def test_admin_source_patch(client):
-    assert client.patch(api.url_for(Sources, source_id=1), data={}, headers=VALID_ADMIN_HEADER_TOKEN).status_code == 405
-
-
-def test_sources_get(client):
-    assert client.get(api.url_for(SourcesList)).status_code == 401
-
-
-def test_sources_post(client):
-    assert client.post(api.url_for(SourcesList), data={}).status_code == 401
-
-
-def test_sources_put(client):
-    assert client.put(api.url_for(SourcesList), data={}).status_code == 405
-
-
-def test_sources_delete(client):
-    assert client.delete(api.url_for(SourcesList)).status_code == 405
-
-
-def test_sources_patch(client):
-    assert client.patch(api.url_for(SourcesList), data={}).status_code == 405
-
-
-def test_auth_sources_get(client):
-    assert client.get(api.url_for(SourcesList), headers=VALID_USER_HEADER_TOKEN).status_code == 200
-
-
-def test_auth_sources_post(client):
-    assert client.post(api.url_for(SourcesList), data={}, headers=VALID_USER_HEADER_TOKEN).status_code == 403
-
-
-def test_admin_sources_get(client):
-    resp = client.get(api.url_for(SourcesList), headers=VALID_ADMIN_HEADER_TOKEN)
-    assert resp.status_code == 200
-    assert resp.json == {'items': [], 'total': 0}
-
-
-def test_admin_sources_crud(client):
-    assert client.post(api.url_for(SourcesList), headers=VALID_ADMIN_HEADER_TOKEN).status_code == 400
-    data = {'plugin': {'slug': 'aws-source', 'pluginOptions': {}}, 'label': 'test', 'description': 'test'}
-    resp = client.post(api.url_for(SourcesList), data=dumps(data), content_type='application/json', headers=VALID_ADMIN_HEADER_TOKEN)
-    assert resp.status_code == 200
-    assert client.get(api.url_for(Sources, source_id=resp.json['id']), headers=VALID_ADMIN_HEADER_TOKEN).status_code == 200
-    resp = client.get(api.url_for(SourcesList), headers=VALID_ADMIN_HEADER_TOKEN)
-    assert resp.status_code == 200
-    assert resp.json['items'][0]['description'] == 'test'
-    assert client.delete(api.url_for(Sources, source_id=2), headers=VALID_ADMIN_HEADER_TOKEN).status_code == 200
-    resp = client.get(api.url_for(SourcesList), headers=VALID_ADMIN_HEADER_TOKEN)
-    assert resp.status_code == 200
-    assert resp.json == {'items': [], 'total': 0}