From c05343d58e180dc084bdeeb9852eeab5715d2075 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Mon, 26 Jun 2017 12:03:24 -0700 Subject: [PATCH] =?UTF-8?q?Adds=20the=20ability=20for=20destination=20plug?= =?UTF-8?q?ins=20to=20be=20sub-classed=20from=20Expor=E2=80=A6=20(#839)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds the ability for destination plugins to be sub-classed from ExportDestination. These plugins have the extra option of specifying an export plugin before the destination receives the data. Closes #807. * fixing tests --- lemur/destinations/service.py | 5 + lemur/plugins/bases/__init__.py | 2 +- lemur/plugins/bases/destination.py | 31 ++++- lemur/plugins/lemur_aws/plugin.py | 46 +++---- lemur/plugins/lemur_aws/s3.py | 1 - lemur/plugins/lemur_kubernetes/plugin.py | 7 +- lemur/schemas.py | 11 +- .../destinations/destination/destination.js | 33 ++++- .../destination/destination.tpl.html | 130 +++++++++++------- lemur/tests/test_authorities.py | 4 +- setup.py | 2 +- 11 files changed, 183 insertions(+), 89 deletions(-) diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index d3c9e0dc..5cccb8a4 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -22,6 +22,11 @@ def create(label, plugin_name, options, description=None): :rtype : Destination :return: New destination """ + # remove any sub-plugin objects before try to save the json options + for option in options: + if 'plugin' in option['type']: + del option['value']['plugin_object'] + destination = Destination(label=label, options=options, plugin_name=plugin_name, description=description) return database.create(destination) diff --git a/lemur/plugins/bases/__init__.py b/lemur/plugins/bases/__init__.py index 96833e7b..9ce6289b 100644 --- a/lemur/plugins/bases/__init__.py +++ b/lemur/plugins/bases/__init__.py @@ -1,4 +1,4 @@ -from .destination import DestinationPlugin # noqa +from .destination import DestinationPlugin, ExportDestinationPlugin # noqa from .issuer import IssuerPlugin # noqa from .source import SourcePlugin # noqa from .notification import NotificationPlugin, ExpirationNotificationPlugin # noqa diff --git a/lemur/plugins/bases/destination.py b/lemur/plugins/bases/destination.py index d4172b8a..486a656d 100644 --- a/lemur/plugins/bases/destination.py +++ b/lemur/plugins/bases/destination.py @@ -6,7 +6,7 @@ .. moduleauthor:: Kevin Glisson """ -from lemur.plugins.base import Plugin +from lemur.plugins.base import Plugin, plugins class DestinationPlugin(Plugin): @@ -15,3 +15,32 @@ class DestinationPlugin(Plugin): def upload(self, name, body, private_key, cert_chain, options, **kwargs): raise NotImplementedError + + +class ExportDestinationPlugin(DestinationPlugin): + default_options = [ + { + 'name': 'exportPlugin', + 'type': 'export-plugin', + 'required': True, + 'helpMessage': 'Export plugin to use before sending data to destination.' + } + ] + + @property + def options(self): + return list(self.default_options) + self.additional_options + + def export(self, body, private_key, cert_chain, options): + export_plugin = self.get_option('exportPlugin', options) + + if export_plugin: + plugin = plugins.get(export_plugin['slug']) + extension, passphrase, data = plugin.export(body, cert_chain, private_key, export_plugin['plugin_options']) + return [(extension, passphrase, data)] + + data = body + '\n' + cert_chain + '\n' + private_key + return [('.pem', '', data)] + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + raise NotImplementedError diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 4c34bf24..28c34acd 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -34,9 +34,9 @@ """ from flask import current_app -from lemur.plugins.bases import DestinationPlugin, SourcePlugin -from lemur.plugins.lemur_aws import iam, s3, elb, ec2 from lemur.plugins import lemur_aws as aws +from lemur.plugins.lemur_aws import iam, s3, elb, ec2 +from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin def get_region_from_dns(dns): @@ -264,7 +264,7 @@ class AWSSourcePlugin(SourcePlugin): iam.delete_cert(certificate.name, account_number=account_number) -class S3DestinationPlugin(DestinationPlugin): +class S3DestinationPlugin(ExportDestinationPlugin): title = 'AWS-S3' slug = 'aws-s3' description = 'Allow the uploading of certificates to Amazon S3' @@ -272,7 +272,7 @@ class S3DestinationPlugin(DestinationPlugin): author = 'Mikhail Khodorovskiy, Harm Weites ' author_url = 'https://github.com/Netflix/lemur' - options = [ + additional_options = [ { 'name': 'bucket', 'type': 'str', @@ -308,11 +308,6 @@ class S3DestinationPlugin(DestinationPlugin): 'required': False, 'validation': '/^$|\s+/', 'helpMessage': 'Must be a valid S3 object prefix!', - }, - { - 'name': 'export-plugin', - 'type': 'str', - 'required': False } ] @@ -320,24 +315,17 @@ class S3DestinationPlugin(DestinationPlugin): super(S3DestinationPlugin, self).__init__(*args, **kwargs) def upload(self, name, body, private_key, chain, options, **kwargs): - # ensure our data is in the right format - if self.get_option('export-plugin', options): - pass + files = self.export(body, private_key, chain, options) - # assume we want standard pem file - else: - # s3 doesn't require private key we write whatever we have - files = [(body, '.pem'), (private_key, '.key.pem'), (chain, '.chain.pem')] - - for data, ext in files: - s3.put( - account_number=self.get_option('account_number', options), - region=self.get_option('region', options), - bucket_name=self.get_option('bucket', options), - prefix='{prefix}/{name}{extension}'.format( - prefix=self.get_option('prefix', options), - name=name, - extension=ext), - data=data, - encrypt=self.get_option('encrypt', options) - ) + for ext, passphrase, data in files: + s3.put( + self.get_option('region', options), + self.get_option('bucket', options), + '{prefix}/{name}{extension}'.format( + prefix=self.get_option('prefix', options), + name=name, + extension=ext), + self.get_option('encrypt', options), + data, + account_number=self.get_option('accountNumber', options) + ) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index cdf5a6c4..0ba7aa4b 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -7,7 +7,6 @@ .. moduleauthor:: Kevin Glisson """ from flask import current_app - from .sts import sts_client diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index 6c873e7e..d1d47051 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -31,12 +31,16 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): if 200 <= create_resp.status_code <= 299: return None + elif create_resp.json()['reason'] != 'AlreadyExists': return create_resp.content + update_resp = k8s_api.put(_resolve_uri(k8s_base_uri, namespace, kind, name), json=data) + if not 200 <= update_resp.status_code <= 299: return update_resp.content - return None + + return def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,): @@ -49,6 +53,7 @@ def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,): def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_VERSION): if not namespace: namespace = 'default' + return "/".join(itertools.chain.from_iterable([ (_resolve_ns(k8s_base_uri, namespace, api_ver=api_ver),), ((kind + 's').lower(),), diff --git a/lemur/schemas.py b/lemur/schemas.py index 46e02b3f..e50ac0a0 100644 --- a/lemur/schemas.py +++ b/lemur/schemas.py @@ -159,9 +159,16 @@ class PluginInputSchema(LemurInputSchema): def get_object(self, data, many=False): try: data['plugin_object'] = plugins.get(data['slug']) + + # parse any sub-plugins + for option in data.get('plugin_options', []): + if 'plugin' in option.get('type', []): + sub_data, errors = PluginInputSchema().load(option['value']) + option['value'] = sub_data + return data - except Exception: - raise ValidationError('Unable to find plugin: {0}'.format(data['slug'])) + except Exception as e: + raise ValidationError('Unable to find plugin. Slug: {0} Reason: {1}'.format(data['slug'], e)) class PluginOutputSchema(LemurOutputSchema): diff --git a/lemur/static/app/angular/destinations/destination/destination.js b/lemur/static/app/angular/destinations/destination/destination.js index dcca9962..21f624c8 100644 --- a/lemur/static/app/angular/destinations/destination/destination.js +++ b/lemur/static/app/angular/destinations/destination/destination.js @@ -2,16 +2,20 @@ angular.module('lemur') - .controller('DestinationsCreateController', function ($scope, $uibModalInstance, PluginService, DestinationService, LemurRestangular, toaster){ + .controller('DestinationsCreateController', function ($scope, $uibModalInstance, PluginService, DestinationService, LemurRestangular, toaster) { $scope.destination = LemurRestangular.restangularizeElement(null, {}, 'destinations'); PluginService.getByType('destination').then(function (plugins) { - $scope.plugins = plugins; + $scope.plugins = plugins; + }); + + PluginService.getByType('export').then(function (plugins) { + $scope.exportPlugins = plugins; }); $scope.save = function (destination) { DestinationService.create(destination).then( - function () { + function () { toaster.pop({ type: 'success', title: destination.label, @@ -27,7 +31,7 @@ angular.module('lemur') directiveData: response.data, timeout: 100000 }); - }); + }); }; $scope.cancel = function () { @@ -36,13 +40,32 @@ angular.module('lemur') }) .controller('DestinationsEditController', function ($scope, $uibModalInstance, DestinationService, DestinationApi, PluginService, toaster, editId) { + + DestinationApi.get(editId).then(function (destination) { $scope.destination = destination; + PluginService.getByType('destination').then(function (plugins) { $scope.plugins = plugins; + _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.destination.pluginName) { + if (plugin.slug === $scope.destination.plugin.slug) { + plugin.pluginOptions = $scope.destination.plugin.pluginOptions; $scope.destination.plugin = plugin; + _.each($scope.destination.plugin.pluginOptions, function (option) { + if (option.type === 'export-plugin') { + PluginService.getByType('export').then(function (plugins) { + $scope.exportPlugins = plugins; + + _.each($scope.exportPlugins, function (plugin) { + if (plugin.slug === option.value.slug) { + plugin.pluginOptions = option.value.pluginOptions; + option.value = plugin; + } + }); + }); + } + }); } }); }); diff --git a/lemur/static/app/angular/destinations/destination/destination.tpl.html b/lemur/static/app/angular/destinations/destination/destination.tpl.html index 15609c69..6ab9d78b 100644 --- a/lemur/static/app/angular/destinations/destination/destination.tpl.html +++ b/lemur/static/app/angular/destinations/destination/destination.tpl.html @@ -1,55 +1,93 @@
- -

CreateEdit Destination oh the places you will go!

+ +

CreateEdit + Destination oh the places you will go!

-
-
- -
- -

You must enter an destination label

-
-
-
- -
- -
-
-
- -
- -
-
-
- -
- -
- - - - -

{{ item.helpMessage }}

-
+ +
+ +
+ +

You must enter an + destination label

+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+ + + + +
+ + +
+
+ +
+ +
+ + + + +

{{ item.helpMessage }}

+
+
+ +
- + +
+

{{ item.helpMessage }}

+
- + +
+
- - + +
diff --git a/lemur/tests/test_authorities.py b/lemur/tests/test_authorities.py index 6b2f86b2..fef5cbcd 100644 --- a/lemur/tests/test_authorities.py +++ b/lemur/tests/test_authorities.py @@ -5,7 +5,7 @@ from lemur.authorities.views import * # noqa from lemur.tests.vectors import VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN -def test_authority_input_schema(client, role): +def test_authority_input_schema(client, role, issuer_plugin): from lemur.authorities.schemas import AuthorityInputSchema input_data = { @@ -13,7 +13,7 @@ def test_authority_input_schema(client, role): 'owner': 'jim@example.com', 'description': 'An example authority.', 'commonName': 'AnExampleAuthority', - 'plugin': {'slug': 'verisign-issuer', 'plugin_options': [{'name': 'test', 'value': 'blah'}]}, + 'plugin': {'slug': 'test-issuer', 'plugin_options': [{'name': 'test', 'value': 'blah'}]}, 'type': 'root', 'signingAlgorithm': 'sha256WithRSA', 'keyType': 'RSA2048', diff --git a/setup.py b/setup.py index 7e5da70d..c69ffc8e 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ 'Flask-Principal==0.4.0', 'Flask-Mail==0.9.1', 'SQLAlchemy-Utils==0.32.14', - 'requests==2.11.1', + 'requests==2.18.1', 'ndg-httpsclient==0.4.2', 'psycopg2==2.7.1', 'arrow==0.10.0',