Adds the ability for destination plugins to be sub-classed from Expor… (#839)

* Adds the ability for destination plugins to be sub-classed from ExportDestination. These plugins have the extra option of specifying an export plugin before the destination receives the data. Closes #807.

* fixing tests

lemur/destinations/service.py

@@ -22,6 +22,11 @@ def create(label, plugin_name, options, description=None):
     :rtype : Destination
     :return: New destination
     """
+    # remove any sub-plugin objects before try to save the json options
+    for option in options:
+        if 'plugin' in option['type']:
+            del option['value']['plugin_object']
+
     destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description)
     return database.create(destination)
 

lemur/plugins/bases/__init__.py

@@ -1,4 +1,4 @@
-from .destination import DestinationPlugin  # noqa
+from .destination import DestinationPlugin, ExportDestinationPlugin  # noqa
 from .issuer import IssuerPlugin  # noqa
 from .source import SourcePlugin  # noqa
 from .notification import NotificationPlugin, ExpirationNotificationPlugin  # noqa

lemur/plugins/bases/destination.py

@@ -6,7 +6,7 @@
 
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from lemur.plugins.base import Plugin
+from lemur.plugins.base import Plugin, plugins
 
 
 class DestinationPlugin(Plugin):
@@ -15,3 +15,32 @@ class DestinationPlugin(Plugin):
 
     def upload(self, name, body, private_key, cert_chain, options, **kwargs):
         raise NotImplementedError
+
+
+class ExportDestinationPlugin(DestinationPlugin):
+    default_options = [
+        {
+            'name': 'exportPlugin',
+            'type': 'export-plugin',
+            'required': True,
+            'helpMessage': 'Export plugin to use before sending data to destination.'
+        }
+    ]
+
+    @property
+    def options(self):
+        return list(self.default_options) + self.additional_options
+
+    def export(self, body, private_key, cert_chain, options):
+        export_plugin = self.get_option('exportPlugin', options)
+
+        if export_plugin:
+            plugin = plugins.get(export_plugin['slug'])
+            extension, passphrase, data = plugin.export(body, cert_chain, private_key, export_plugin['plugin_options'])
+            return [(extension, passphrase, data)]
+
+        data = body + '\n' + cert_chain + '\n' + private_key
+        return [('.pem', '', data)]
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        raise NotImplementedError

lemur/plugins/lemur_aws/plugin.py

@@ -34,9 +34,9 @@
 """
 from flask import current_app
 
-from lemur.plugins.bases import DestinationPlugin, SourcePlugin
-from lemur.plugins.lemur_aws import iam, s3, elb, ec2
 from lemur.plugins import lemur_aws as aws
+from lemur.plugins.lemur_aws import iam, s3, elb, ec2
+from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
 
 
 def get_region_from_dns(dns):
@@ -264,7 +264,7 @@ class AWSSourcePlugin(SourcePlugin):
         iam.delete_cert(certificate.name, account_number=account_number)
 
 
-class S3DestinationPlugin(DestinationPlugin):
+class S3DestinationPlugin(ExportDestinationPlugin):
     title = 'AWS-S3'
     slug = 'aws-s3'
     description = 'Allow the uploading of certificates to Amazon S3'
@@ -272,7 +272,7 @@ class S3DestinationPlugin(DestinationPlugin):
     author = 'Mikhail Khodorovskiy, Harm Weites <harm@weites.com>'
     author_url = 'https://github.com/Netflix/lemur'
 
-    options = [
+    additional_options = [
         {
             'name': 'bucket',
             'type': 'str',
@@ -308,11 +308,6 @@ class S3DestinationPlugin(DestinationPlugin):
             'required': False,
             'validation': '/^$|\s+/',
             'helpMessage': 'Must be a valid S3 object prefix!',
-        },
-        {
-            'name': 'export-plugin',
-            'type': 'str',
-            'required': False
         }
     ]
 
@@ -320,24 +315,17 @@ class S3DestinationPlugin(DestinationPlugin):
         super(S3DestinationPlugin, self).__init__(*args, **kwargs)
 
     def upload(self, name, body, private_key, chain, options, **kwargs):
-        # ensure our data is in the right format
+        files = self.export(body, private_key, chain, options)
-        if self.get_option('export-plugin', options):
-            pass
 
-        # assume we want standard pem file
+        for ext, passphrase, data in files:
-        else:
-            # s3 doesn't require private key we write whatever we have
-            files = [(body, '.pem'), (private_key, '.key.pem'), (chain, '.chain.pem')]
-
-            for data, ext in files:
             s3.put(
-                    account_number=self.get_option('account_number', options),
+                self.get_option('region', options),
-                    region=self.get_option('region', options),
+                self.get_option('bucket', options),
-                    bucket_name=self.get_option('bucket', options),
+                '{prefix}/{name}{extension}'.format(
-                    prefix='{prefix}/{name}{extension}'.format(
                     prefix=self.get_option('prefix', options),
                     name=name,
                     extension=ext),
-                    data=data,
+                self.get_option('encrypt', options),
-                    encrypt=self.get_option('encrypt', options)
+                data,
+                account_number=self.get_option('accountNumber', options)
             )

lemur/plugins/lemur_aws/s3.py

@@ -7,7 +7,6 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import current_app
-
 from .sts import sts_client
 
 

lemur/plugins/lemur_kubernetes/plugin.py

@@ -31,12 +31,16 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data):
 
     if 200 <= create_resp.status_code <= 299:
         return None
+
     elif create_resp.json()['reason'] != 'AlreadyExists':
         return create_resp.content
+
     update_resp = k8s_api.put(_resolve_uri(k8s_base_uri, namespace, kind, name), json=data)
+
     if not 200 <= update_resp.status_code <= 299:
         return update_resp.content
-    return None
+
+    return
 
 
 def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,):
@@ -49,6 +53,7 @@ def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,):
 def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_VERSION):
     if not namespace:
         namespace = 'default'
+
     return "/".join(itertools.chain.from_iterable([
         (_resolve_ns(k8s_base_uri, namespace, api_ver=api_ver),),
         ((kind + 's').lower(),),

lemur/schemas.py

@@ -159,9 +159,16 @@ class PluginInputSchema(LemurInputSchema):
     def get_object(self, data, many=False):
         try:
             data['plugin_object'] = plugins.get(data['slug'])
+
+            # parse any sub-plugins
+            for option in data.get('plugin_options', []):
+                if 'plugin' in option.get('type', []):
+                    sub_data, errors = PluginInputSchema().load(option['value'])
+                    option['value'] = sub_data
+
             return data
-        except Exception:
+        except Exception as e:
-            raise ValidationError('Unable to find plugin: {0}'.format(data['slug']))
+            raise ValidationError('Unable to find plugin. Slug: {0} Reason: {1}'.format(data['slug'], e))
 
 
 class PluginOutputSchema(LemurOutputSchema):

lemur/static/app/angular/destinations/destination/destination.js

@@ -2,13 +2,17 @@
 
 angular.module('lemur')
 
-  .controller('DestinationsCreateController', function ($scope, $uibModalInstance, PluginService, DestinationService, LemurRestangular, toaster){
+  .controller('DestinationsCreateController', function ($scope, $uibModalInstance, PluginService, DestinationService, LemurRestangular, toaster) {
     $scope.destination = LemurRestangular.restangularizeElement(null, {}, 'destinations');
 
     PluginService.getByType('destination').then(function (plugins) {
       $scope.plugins = plugins;
     });
 
+    PluginService.getByType('export').then(function (plugins) {
+      $scope.exportPlugins = plugins;
+    });
+
     $scope.save = function (destination) {
       DestinationService.create(destination).then(
         function () {
@@ -36,13 +40,32 @@ angular.module('lemur')
   })
 
   .controller('DestinationsEditController', function ($scope, $uibModalInstance, DestinationService, DestinationApi, PluginService, toaster, editId) {
+
+
     DestinationApi.get(editId).then(function (destination) {
       $scope.destination = destination;
+
       PluginService.getByType('destination').then(function (plugins) {
         $scope.plugins = plugins;
+
         _.each($scope.plugins, function (plugin) {
-          if (plugin.slug === $scope.destination.pluginName) {
+          if (plugin.slug === $scope.destination.plugin.slug) {
+            plugin.pluginOptions = $scope.destination.plugin.pluginOptions;
             $scope.destination.plugin = plugin;
+            _.each($scope.destination.plugin.pluginOptions, function (option) {
+              if (option.type === 'export-plugin') {
+                PluginService.getByType('export').then(function (plugins) {
+                  $scope.exportPlugins = plugins;
+
+                  _.each($scope.exportPlugins, function (plugin) {
+                    if (plugin.slug === option.value.slug) {
+                      plugin.pluginOptions = option.value.pluginOptions;
+                      option.value = plugin;
+                    }
+                  });
+                });
+              }
+            });
           }
         });
       });

lemur/static/app/angular/destinations/destination/destination.tpl.html

@@ -1,6 +1,8 @@
 <div class="modal-header">
-    <button type="button" class="close" ng-click="cancel()" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+  <button type="button" class="close" ng-click="cancel()" aria-label="Close"><span aria-hidden="true">&times;</span>
-    <h3><span ng-show="!destination.fromServer">Create</span><span ng-show="destination.fromServer">Edit</span> Destination <span class="text-muted"><small>oh the places you will go!</small></span></h3>
+  </button>
+  <h3><span ng-show="!destination.fromServer">Create</span><span ng-show="destination.fromServer">Edit</span>
+    Destination <span class="text-muted"><small>oh the places you will go!</small></span></h3>
 </div>
 <div class="modal-body">
   <form name="createForm" class="form-horizontal" role="form" novalidate>
@@ -11,7 +13,8 @@
       </label>
       <div class="col-sm-10">
         <input name="label" ng-model="destination.label" placeholder="Label" class="form-control" required/>
-                <p ng-show="createForm.label.$invalid && !createForm.label.$pristine" class="help-block">You must enter an destination label</p>
+        <p ng-show="createForm.label.$invalid && !createForm.label.$pristine" class="help-block">You must enter an
+          destination label</p>
       </div>
     </div>
     <div class="form-group">
@@ -19,7 +22,8 @@
         Description
       </label>
       <div class="col-sm-10">
-                <textarea name="comments" ng-model="destination.description" placeholder="Something elegant" class="form-control" ></textarea>
+        <textarea name="comments" ng-model="destination.description" placeholder="Something elegant"
+                  class="form-control"></textarea>
       </div>
     </div>
     <div class="form-group">
@@ -27,7 +31,8 @@
         Plugin
       </label>
       <div class="col-sm-10">
-                <select class="form-control" ng-model="destination.plugin" ng-options="plugin.title for plugin in plugins" required></select>
+        <select class="form-control" ng-model="destination.plugin" ng-options="plugin.title for plugin in plugins"
+                required></select>
       </div>
     </div>
     <div class="form-group" ng-repeat="item in destination.plugin.pluginOptions">
@@ -37,10 +42,42 @@
             {{ item.name | titleCase }}
           </label>
           <div class="col-sm-10">
-                        <input name="sub" ng-if="item.type == 'int'" type="number" ng-pattern="/^[0-9]{12,12}$/" class="form-control" ng-model="item.value"/>
+            <input name="sub" ng-if="item.type == 'int'" type="number" ng-pattern="/^[0-9]{12,12}$/"
-                        <select name="sub" ng-if="item.type == 'select'" class="form-control" ng-options="i for i in item.available" ng-model="item.value"></select>
+                   class="form-control" ng-model="item.value"/>
+            <select name="sub" ng-if="item.type == 'select'" class="form-control" ng-options="i for i in item.available"
+                    ng-model="item.value"></select>
             <input name="sub" ng-if="item.type == 'bool'" class="form-control" type="checkbox" ng-model="item.value">
             <input name="sub" ng-if="item.type == 'str'" type="text" class="form-control" ng-model="item.value"/>
+            <div ng-if="item.type == 'export-plugin'">
+              <form name="exportForm" class="form-horizontal" role="form" novalidate>
+                <select class="form-control" ng-model="item.value"
+                        ng-options="plugin.title for plugin in exportPlugins" required></select>
+                <div class="col-sm-10">
+                  <div class="form-group" ng-repeat="item in item.value.pluginOptions">
+                    <ng-form name="subForm" class="form-horizontal" role="form" novalidate>
+                      <div
+                        ng-class="{'has-error': subForm.sub.$invalid, 'has-success': !subForm.sub.$invalid&&subForm.sub.$dirty}">
+                        <label class="control-label col-sm-2">
+                          {{ item.name | titleCase }}
+                        </label>
+                        <div class="col-sm-10">
+                          <input name="sub" ng-if="item.type == 'int'" type="number" ng-pattern="/^[0-9]{12,12}$/"
+                                 class="form-control" ng-model="item.value"/>
+                          <select name="sub" ng-if="item.type == 'select'" class="form-control"
+                                  ng-options="i for i in item.available" ng-model="item.value"></select>
+                          <input name="sub" ng-if="item.type == 'bool'" class="form-control" type="checkbox"
+                                 ng-model="item.value">
+                          <input name="sub" ng-if="item.type == 'str'" type="text" class="form-control"
+                                 ng-model="item.value" ng-pattern="item.validation"/>
+                          <p ng-show="subForm.sub.$invalid && !subForm.sub.$pristine"
+                             class="help-block">{{ item.helpMessage }}</p>
+                        </div>
+                      </div>
+                    </ng-form>
+                  </div>
+                </div>
+              </form>
+            </div>
             <p ng-show="subForm.sub.$invalid && !subForm.sub.$pristine" class="help-block">{{ item.helpMessage }}</p>
           </div>
         </div>
@@ -49,7 +86,8 @@
   </form>
 </div>
 <div class="modal-footer">
-    <button ng-click="save(destination)" type="submit" ng-disabled="createForm.$invalid" class="btn btn-primary">Save</button>
+  <button ng-click="save(destination)" type="submit" ng-disabled="createForm.$invalid" class="btn btn-primary">Save
+  </button>
   <button ng-click="cancel()" class="btn btn-danger">Cancel</button>
 </div>
 

lemur/tests/test_authorities.py

@@ -5,7 +5,7 @@ from lemur.authorities.views import *  # noqa
 from lemur.tests.vectors import VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN
 
 
-def test_authority_input_schema(client, role):
+def test_authority_input_schema(client, role, issuer_plugin):
     from lemur.authorities.schemas import AuthorityInputSchema
 
     input_data = {
@@ -13,7 +13,7 @@ def test_authority_input_schema(client, role):
         'owner': 'jim@example.com',
         'description': 'An example authority.',
         'commonName': 'AnExampleAuthority',
-        'plugin': {'slug': 'verisign-issuer', 'plugin_options': [{'name': 'test', 'value': 'blah'}]},
+        'plugin': {'slug': 'test-issuer', 'plugin_options': [{'name': 'test', 'value': 'blah'}]},
         'type': 'root',
         'signingAlgorithm': 'sha256WithRSA',
         'keyType': 'RSA2048',

setup.py

@@ -43,7 +43,7 @@ install_requires = [
     'Flask-Principal==0.4.0',
     'Flask-Mail==0.9.1',
     'SQLAlchemy-Utils==0.32.14',
-    'requests==2.11.1',
+    'requests==2.18.1',
     'ndg-httpsclient==0.4.2',
     'psycopg2==2.7.1',
     'arrow==0.10.0',