Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Starting to move to new plugin architecture.

Browse Source
This commit is contained in:
kevgliss 2015-07-04 12:47:57 -07:00
parent eadfaaeed0
commit 3f49bb95ff
24 changed files with 327 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lemur/authorities/service.py
View File

@ -17,7 +17,7 @@ from lemur.roles import service as role_service
from lemur.roles.models import Role
import lemur.certificates.service as cert_service
from lemur.common.services.issuers.manager import get_plugin_by_name
from lemur.plugins.base import plugins
def update(authority_id, active=None, roles=None):
"""
@ -49,7 +49,7 @@ def create(kwargs):
:return:
"""
issuer = get_plugin_by_name(kwargs.get('pluginName'))
issuer = plugins.get(kwargs.get('pluginName'))
kwargs['creator'] = g.current_user.email
cert_body, intermediate, issuer_roles = issuer.create_authority(kwargs)

5
lemur/certificates/service.py
View File

@ -18,8 +18,7 @@ from flask import g, current_app
from lemur import database
from lemur.common.services.aws import iam
from lemur.common.services.issuers.manager import get_plugin_by_name
from lemur.plugins.base import plugins
from lemur.certificates.models import Certificate
from lemur.certificates.exceptions import UnableToCreateCSR, \
UnableToCreatePrivateKey, MissingFiles
@ -127,7 +126,7 @@ def mint(issuer_options):
"""
authority = issuer_options['authority']
issuer = get_plugin_by_name(authority.plugin_name)
issuer = plugins.get(authority.plugin_name)
# NOTE if we wanted to support more issuers it might make sense to
# push CSR creation down to the plugin
path = create_csr(issuer.get_csr_config(issuer_options))

5
lemur/certificates/sync.py
View File

@ -30,8 +30,7 @@ from lemur.certificates.models import Certificate, get_name_from_arn
from lemur.common.services.aws.iam import get_all_server_certs
from lemur.common.services.aws.iam import get_cert_from_arn
from lemur.common.services.issuers.manager import get_plugin_by_name
from lemur.plugins.base import plugins
def aws():
"""
@ -101,7 +100,7 @@ def cloudca():
"""
user = user_service.get_by_email('lemur@nobody')
# sync all new certificates/authorities not created through lemur
issuer = get_plugin_by_name('cloudca')
issuer = plugins.get('cloudca')
authorities = issuer.get_authorities()
total = 0
new = 1

64
lemur/common/managers.py Normal file
View File

@ -0,0 +1,64 @@
"""
.. module: lemur.common.managers
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import current_app
# inspired by https://github.com/getsentry/sentry
class InstanceManager(object):
def __init__(self, class_list=None, instances=True):
if class_list is None:
class_list = []
self.instances = instances
self.update(class_list)
def get_class_list(self):
return self.class_list
def add(self, class_path):
self.cache = None
self.class_list.append(class_path)
def remove(self, class_path):
self.cache = None
self.class_list.remove(class_path)
def update(self, class_list):
"""
Updates the class list and wipes the cache.
"""
self.cache = None
self.class_list = class_list
def all(self):
"""
Returns a list of cached instances.
"""
class_list = list(self.get_class_list())
if not class_list:
self.cache = []
return []
if self.cache is not None:
return self.cache
results = []
for cls_path in class_list:
module_name, class_name = cls_path.rsplit('.', 1)
try:
module = __import__(module_name, {}, {}, class_name)
cls = getattr(module, class_name)
if self.instances:
results.append(cls())
else:
results.append(cls)
except Exception:
current_app.logger.exception('Unable to import %s', cls_path)
continue
self.cache = results
return results

37
lemur/common/services/issuers/manager.py
View File

@ -1,37 +0,0 @@
"""
.. module: lemur.common.services.issuers.manager
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson (kglisson@netflix.com)
"""
import pkgutil
from importlib import import_module
from flask import current_app
from lemur.common.services.issuers import plugins
# TODO make the plugin dir configurable
def get_plugin_by_name(plugin_name):
"""
Fetches a given plugin by it's name. We use a known location for issuer plugins and attempt
to load it such that it can be used for issuing certificates.
:param plugin_name:
:return: a plugin `class` :raise Exception: Generic error whenever the plugin specified can not be found.
"""
for importer, modname, ispkg in pkgutil.iter_modules(plugins.__path__):
try:
issuer = import_module('lemur.common.services.issuers.plugins.{0}.{0}'.format(modname))
if issuer.__name__ == plugin_name:
# we shouldn't return bad issuers
issuer_obj = issuer.init()
return issuer_obj
except Exception as e:
current_app.logger.warn("Issuer {0} was unable to be imported: {1}".format(modname, e))
else:
raise Exception("Could not find the specified plugin: {0}".format(plugin_name))

27
lemur/common/services/issuers/plugins/cloudca/constants.py
View File

@ -1,27 +0,0 @@
CSR_CONFIG = """
# Configuration for standard CSR generation for Netflix
# Used for procuring CloudCA certificates
# Author: kglisson
# Contact: secops@netflix.com
[ req ]
# Use a 2048 bit private key
default_bits = 2048
default_keyfile = key.pem
prompt = no
encrypt_key = no
# base request
distinguished_name = req_distinguished_name
# distinguished_name
[ req_distinguished_name ]
countryName = "{country}" # C=
stateOrProvinceName = "{state}" # ST=
localityName = "{location}" # L=
organizationName = "{organization}" # O=
organizationalUnitName = "{organizationalUnit}" # OU=
# This is the hostname/subject name on the certificate
commonName = "{commonName}" # CN=
"""

27
lemur/factory.py
View File

@ -12,6 +12,7 @@
import os
import imp
import errno
import pkg_resources
from logging import Formatter
from logging.handlers import RotatingFileHandler
@ -51,6 +52,7 @@ def create_app(app_name=None, blueprints=None, config=None):
configure_blueprints(app, blueprints)
configure_extensions(app)
configure_logging(app)
install_plugins(app)
return app
@ -91,7 +93,7 @@ def configure_app(app, config=None):
elif os.path.isfile(os.path.expanduser("~/.lemur/lemur.conf.py")):
app.config.from_object(from_file(os.path.expanduser("~/.lemur/lemur.conf.py")))
else:
app.config.from_object(from_file(os.path.join(os.getcwd(), 'default.conf.py')))
app.config.from_object(from_file(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'default.conf.py')))
@ -136,3 +138,26 @@ def configure_logging(app):
app.logger.setLevel(app.config.get('LOG_LEVEL', 'DEBUG'))
app.logger.addHandler(handler)
def install_plugins(app):
"""
Installs new issuers that are not currently bundled with Lemur.
:param settings:
:return:
"""
from lemur.plugins.base import register
# entry_points={
# 'lemur.plugins': [
# 'verisign = lemur_verisign.plugin:VerisignPlugin'
# ],
# },
for ep in pkg_resources.iter_entry_points('lemur.plugins'):
try:
plugin = ep.load()
except Exception:
import sys
import traceback
app.logger.error("Failed to load plugin %r:\n%s\n" % (ep.name, traceback.format_exc()))
else:
register(plugin)

41
lemur/manage.py
View File

@ -333,7 +333,7 @@ class InitializeApp(Command):
else:
sys.stdout.write("[-] Default user has already been created, skipping...!\n")
if current_app.app.config.get('AWS_ACCOUNT_MAPPINGS'):
if current_app.config.get('AWS_ACCOUNT_MAPPINGS'):
for account_name, account_number in current_app.config.get('AWS_ACCOUNT_MAPPINGS').items():
account = account_service.get_by_account_number(account_number)
@ -346,45 +346,6 @@ class InitializeApp(Command):
sys.stdout.write("[/] Done!\n")
#def install_issuers(settings):
# """
# Installs new issuers that are not currently bundled with Lemur.
#
# :param settings:
# :return:
# """
# from lemur.issuers import register
# # entry_points={
# # 'lemur.issuers': [
# # 'verisign = lemur_issuers.issuers:VerisignPlugin'
# # ],
# # },
# installed_apps = list(settings.INSTALLED_APPS)
# for ep in pkg_resources.iter_entry_points('lemur.apps'):
# try:
# issuer = ep.load()
# except Exception:
# import sys
# import traceback
#
# sys.stderr.write("Failed to load app %r:\n%s\n" % (ep.name, traceback.format_exc()))
# else:
# installed_apps.append(ep.module_name)
# settings.INSTALLED_APPS = tuple(installed_apps)
#
# for ep in pkg_resources.iter_entry_points('lemur.issuers'):
# try:
# issuer = ep.load()
# except Exception:
# import sys
# import traceback
#
# sys.stderr.write("Failed to load issuer %r:\n%s\n" % (ep.name, traceback.format_exc()))
# else:
# register(issuer)
class CreateUser(Command):
"""
This command allows for the creation of a new user within Lemur

4
lemur/plugins/__init__.py Normal file
View File

@ -0,0 +1,4 @@
from __future__ import absolute_import
from lemur.plugins.base import * # NOQA
from lemur.plugins.bases import * # NOQA

16
lemur/plugins/base/__init__.py Normal file
View File

@ -0,0 +1,16 @@
"""
.. module: lemur.plugins.base
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from __future__ import absolute_import, print_function
from lemur.plugins.base.manager import PluginManager
from lemur.plugins.base.v1 import * # NOQA
plugins = PluginManager()
register = plugins.register
unregister = plugins.unregister

59
lemur/plugins/base/manager.py Normal file
View File

@ -0,0 +1,59 @@
"""
.. module: lemur.plugins.base.manager
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson (kglisson@netflix.com)
"""
from flask import current_app
from lemur.common.managers import InstanceManager
# inspired by https://github.com/getsentry/sentry
class PluginManager(InstanceManager):
def __iter__(self):
return iter(self.all())
def __len__(self):
return sum(1 for i in self.all())
def all(self, version=1):
for plugin in sorted(super(PluginManager, self).all(), key=lambda x: x.get_title()):
if not plugin.is_enabled():
continue
if version is not None and plugin.__version__ != version:
continue
yield plugin
def get(self, slug):
for plugin in self.all(version=1):
if plugin.slug == slug:
return plugin
for plugin in self.all(version=2):
if plugin.slug == slug:
return plugin
raise KeyError(slug)
def first(self, func_name, *args, **kwargs):
version = kwargs.pop('version', 1)
for plugin in self.all(version=version):
try:
result = getattr(plugin, func_name)(*args, **kwargs)
except Exception as e:
current_app.logger.error('Error processing %s() on %r: %s', func_name, plugin.__class__, e, extra={
'func_arg': args,
'func_kwargs': kwargs,
}, exc_info=True)
continue
if result is not None:
return result
def register(self, cls):
self.add('%s.%s' % (cls.__module__, cls.__name__))
return cls
def unregister(self, cls):
self.remove('%s.%s' % (cls.__module__, cls.__name__))
return cls

117
lemur/plugins/base/v1.py Normal file
View File

@ -0,0 +1,117 @@
"""
.. module: lemur.plugins.base.v1
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from threading import local
# stolen from https://github.com/getsentry/sentry/
class PluginMount(type):
def __new__(cls, name, bases, attrs):
new_cls = type.__new__(cls, name, bases, attrs)
if IPlugin in bases:
return new_cls
if new_cls.title is None:
new_cls.title = new_cls.__name__
if not new_cls.slug:
new_cls.slug = new_cls.title.replace(' ', '-').lower()
return new_cls
class IPlugin(local):
"""
Plugin interface. Should not be inherited from directly.
A plugin should be treated as if it were a singleton. The owner does not
control when or how the plugin gets instantiated, nor is it guaranteed that
it will happen, or happen more than once.
>>> from lemur.plugins import Plugin
>>>
>>> class MyPlugin(Plugin):
>>> def get_title(self):
>>> return 'My Plugin'
As a general rule all inherited methods should allow ``**kwargs`` to ensure
ease of future compatibility.
"""
# Generic plugin information
title = None
slug = None
description = None
version = None
author = None
author_url = None
resource_links = ()
# Configuration specifics
conf_key = None
conf_title = None
# Global enabled state
enabled = True
can_disable = True
def is_enabled(self, project=None):
"""
Returns a boolean representing if this plugin is enabled.
If ``project`` is passed, it will limit the scope to that project.
>>> plugin.is_enabled()
"""
if not self.enabled:
return False
if not self.can_disable:
return True
return True
def get_conf_key(self):
"""
Returns a string representing the configuration keyspace prefix for this plugin.
"""
if not self.conf_key:
self.conf_key = self.get_conf_title().lower().replace(' ', '_')
return self.conf_key
def get_conf_title(self):
"""
Returns a string representing the title to be shown on the configuration page.
"""
return self.conf_title or self.get_title()
def get_title(self):
"""
Returns the general title for this plugin.
>>> plugin.get_title()
"""
return self.title
def get_description(self):
"""
Returns the description for this plugin. This is shown on the plugin configuration
page.
>>> plugin.get_description()
"""
return self.description
def get_resource_links(self):
"""
Returns a list of tuples pointing to various resources for this plugin.
>>> def get_resource_links(self):
>>> return [
>>> ('Documentation', 'http://sentry.readthedocs.org'),
>>> ('Bug Tracker', 'https://github.com/getsentry/sentry/issues'),
>>> ('Source', 'https://github.com/getsentry/sentry'),
>>> ]
"""
return self.resource_links
class Plugin(IPlugin):
"""
A plugin should be treated as if it were a singleton. The owner does not
control when or how the plugin gets instantiated, nor is it guaranteed that
it will happen, or happen more than once.
"""
__version__ = 1
__metaclass__ = PluginMount

2
lemur/plugins/bases/__init__.py Normal file
View File

@ -0,0 +1,2 @@
from .destination import DestinationPlugin # NOQA
from .issuer import IssuerPlugin # NOQA

13
lemur/plugins/bases/destination.py Normal file
View File

@ -0,0 +1,13 @@
"""
.. module: lemur.bases.destination
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from lemur.plugins.base import Plugin
class DestinationPlugin(Plugin):
pass

11
lemur/common/services/issuers/issuer.py → lemur/plugins/bases/issuer.py
View File

@ -1,23 +1,18 @@
"""
.. module: authority
.. module: lemur.bases.issuer
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import current_app
from lemur.plugins.base import Plugin
class Issuer(object):
class IssuerPlugin(Plugin):
"""
This is the base class from which all of the supported
issuers will inherit from.
"""
def __init__(self):
self.dry_run = current_app.config.get('DRY_RUN')
def create_certificate(self):
raise NotImplementedError

0
lemur/common/services/issuers/__init__.py → lemur/plugins/bases/source.py
View File

0
lemur/common/services/issuers/plugins/__init__.py → lemur/plugins/lemur_aws/__init__.py
View File

0
lemur/plugins/lemur_aws/aws.py Normal file
View File

0
lemur/common/services/issuers/plugins/cloudca/__init__.py → lemur/plugins/lemur_cloudca/__init__.py
View File

23
lemur/common/services/issuers/plugins/cloudca/cloudca.py → lemur/plugins/lemur_cloudca/plugin.py
View File

@ -18,10 +18,8 @@ from requests.adapters import HTTPAdapter
from flask import current_app
from lemur.exceptions import LemurException
from lemur.common.services.issuers.issuer import Issuer
from lemur.common.services.issuers.plugins import cloudca
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins import lemur_cloudca as cloudca
from lemur.authorities import service as authority_service
@ -144,7 +142,7 @@ def get_auth_data(ca_name):
raise CloudCAException("You do not have the required role to issue certificates from {0}".format(ca_name))
class CloudCA(Issuer):
class CloudCAPlugin(IssuerPlugin):
title = 'CloudCA'
slug = 'cloudca'
description = 'Enables the creation of certificates from the cloudca API.'
@ -164,7 +162,7 @@ class CloudCA(Issuer):
else:
current_app.logger.warning("No CLOUDCA credentials found, lemur will be unable to request certificates from CLOUDCA")
super(CloudCA, self).__init__(*args, **kwargs)
super(CloudCAPlugin, self).__init__(*args, **kwargs)
def create_authority(self, options):
"""
@ -261,15 +259,6 @@ class CloudCA(Issuer):
return cert, "".join(intermediates),
def get_csr_config(self, issuer_options):
"""
Get a valid CSR for use with CloudCA
:param issuer_options:
:return:
"""
return cloudca.constants.CSR_CONFIG.format(**issuer_options)
def random(self, length=10):
"""
Uses CloudCA as a decent source of randomness.
@ -340,7 +329,3 @@ class CloudCA(Issuer):
response = self.session.get(self.url + endpoint, timeout=10, verify=self.ca_bundle)
return process_response(response)
def init():
return CloudCA()

0
lemur/common/services/issuers/plugins/verisign/__init__.py → lemur/plugins/lemur_verisign/__init__.py
View File

40
lemur/common/services/issuers/plugins/verisign/constants.py → lemur/plugins/lemur_verisign/constants.py
View File

@ -1,42 +1,3 @@
CSR_CONFIG = """
# Configuration for standard CSR generation for Netflix
# Used for procuring VeriSign certificates
# Author: jachan
# Contact: cloudsecurity@netflix.com
[ req ]
# Use a 2048 bit private key
default_bits = 2048
default_keyfile = key.pem
prompt = no
encrypt_key = no
# base request
distinguished_name = req_distinguished_name
# extensions
# Uncomment the following line if you are requesting a SAN cert
{is_san_comment}req_extensions = req_ext
# distinguished_name
[ req_distinguished_name ]
countryName = "US" # C=
stateOrProvinceName = "CALIFORNIA" # ST=
localityName = "Los Gatos" # L=
organizationName = "Netflix, Inc." # O=
organizationalUnitName = "{OU}" # OU=
# This is the hostname/subject name on the certificate
commonName = "{DNS[0]}" # CN=
[ req_ext ]
# Uncomment the following line if you are requesting a SAN cert
{is_san_comment}subjectAltName = @alt_names
[alt_names]
# Put your SANs here
{DNS_LINES}
"""
VERISIGN_INTERMEDIATE = """
-----BEGIN CERTIFICATE-----
MIIFFTCCA/2gAwIBAgIQKC4nkXkzkuQo8iGnTsk3rjANBgkqhkiG9w0BAQsFADCB
@ -70,7 +31,6 @@ J+71/xuzAYN6
-----END CERTIFICATE-----
"""
VERISIGN_ROOT = """
-----BEGIN CERTIFICATE-----
MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw

54
lemur/common/services/issuers/plugins/verisign/verisign.py → lemur/plugins/lemur_verisign/plugin.py
View File

@ -1,5 +1,5 @@
"""
.. module: lemur.common.services.issuers.plugins.verisign.verisign
.. module: lemur.plugins.lemur_verisign.verisign
:platform: Unix
:synopsis: This module is responsible for communicating with the VeriSign VICE 2.0 API.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
@ -13,10 +13,9 @@ import xmltodict
from flask import current_app
from lemur.common.services.issuers.issuer import Issuer
from lemur.common.services.issuers.plugins import verisign
from lemur.certificates.exceptions import InsufficientDomains
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins import lemur_verisign as verisign
from lemur.plugins.lemur_verisign import constants
# https://support.venafi.com/entries/66445046-Info-VeriSign-Error-Codes
@ -58,7 +57,7 @@ VERISIGN_ERRORS = {
}
class Verisign(Issuer):
class VerisignPlugin(IssuerPlugin):
title = 'VeriSign'
slug = 'verisign'
description = 'Enables the creation of certificates by the VICE2.0 verisign API.'
@ -70,7 +69,7 @@ class Verisign(Issuer):
def __init__(self, *args, **kwargs):
self.session = requests.Session()
self.session.cert = current_app.config.get('VERISIGN_PEM_PATH')
super(Verisign, self).__init__(*args, **kwargs)
super(VerisignPlugin, self).__init__(*args, **kwargs)
@staticmethod
def handle_response(content):
@ -127,41 +126,7 @@ class Verisign(Issuer):
response = self.session.post(url, data=data)
cert = self.handle_response(response.content)['Response']['Certificate']
return cert, verisign.constants.VERISIGN_INTERMEDIATE,
def get_csr_config(self, issuer_options):
"""
Used to generate a valid CSR for the given Certificate Authority.
:param issuer_options:
:return: :raise InsufficientDomains:
"""
domains = []
if issuer_options.get('commonName'):
domains.append(issuer_options.get('commonName'))
if issuer_options.get('extensions'):
for n in issuer_options['extensions']['subAltNames']['names']:
if n['value']:
domains.append(n['value'])
is_san_comment = "#"
dns_lines = []
if len(domains) < 1:
raise InsufficientDomains
elif len(domains) > 1:
is_san_comment = ""
for domain_line in list(set(domains)):
dns_lines.append("DNS.{} = {}".format(len(dns_lines) + 1, domain_line))
return verisign.constants.CSR_CONFIG.format(
is_san_comment=is_san_comment,
OU=issuer_options.get('organizationalUnit', 'Operations'),
DNS=domains,
DNS_LINES="\n".join(dns_lines))
return cert, constants.VERISIGN_INTERMEDIATE,
@staticmethod
def create_authority(options):
@ -173,7 +138,7 @@ class Verisign(Issuer):
:return:
"""
role = {'username': '', 'password': '', 'name': 'verisign'}
return verisign.constants.VERISIGN_ROOT, "", [role]
return constants.VERISIGN_ROOT, "", [role]
def get_available_units(self):
"""
@ -189,6 +154,3 @@ class Verisign(Issuer):
def get_authorities(self):
pass
def init():
return Verisign()

4
setup.py
View File

@ -103,6 +103,10 @@ setup(
'console_scripts': [
'lemur = lemur.manage:main',
],
'lemur.plugins': [
'verisign = lemur.plugins.lemur_verisign.plugin:VerisignPlugin',
'cloudca = lemur.plugins.lemur_cloudca.plugin:CloudCAPlugin',
],
},
classifiers=[
'Framework :: Flask',