Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

adding regex filtering

Browse Source
This commit is contained in:
alwaysjolley
2019-04-18 13:52:40 -04:00
parent 9ecc19c481 81e2ff99ff
commit fb3b0e8cd7
17 changed files with 243 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
lemur/certificates/schemas.py
View File

@ -10,6 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load
from marshmallow.exceptions import ValidationError
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.certificates import utils as cert_utils
from lemur.common import missing, utils, validators
from lemur.common.fields import ArrowDateTime, Hex
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
@ -96,6 +97,9 @@ class CertificateInputSchema(CertificateCreationSchema):
@validates_schema
def validate_authority(self, data):
if isinstance(data['authority'], str):
raise ValidationError("Authority not found.")
if not data['authority'].active:
raise ValidationError("The authority is inactive.", ['authority'])
@ -107,6 +111,11 @@ class CertificateInputSchema(CertificateCreationSchema):
def load_data(self, data):
if data.get('replacements'):
data['replaces'] = data['replacements'] # TODO remove when field is deprecated
if data.get('csr'):
dns_names = cert_utils.get_dns_names_from_csr(data['csr'])
if not data['extensions']['subAltNames']['names']:
data['extensions']['subAltNames']['names'] = []
data['extensions']['subAltNames']['names'] += dns_names
return missing.convert_validity_years(data)

42
lemur/certificates/utils.py Normal file
View File

@ -0,0 +1,42 @@
"""
Utils to parse certificate data.
.. module: lemur.certificates.hooks
:platform: Unix
:copyright: (c) 2019 by Javier Ramos, see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Javier Ramos <javier.ramos@booking.com>
"""
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from marshmallow.exceptions import ValidationError
def get_dns_names_from_csr(data):
"""
Fetches DNSNames from CSR.
Potentially extendable to any kind of SubjectAlternativeName
:param data: PEM-encoded string with CSR
:return:
"""
dns_names = []
try:
request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
except Exception:
raise ValidationError('CSR presented is not valid.')
try:
alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
for name in alt_names.value.get_values_for_type(x509.DNSName):
dns_name = {
'nameType': 'DNSName',
'value': name
}
dns_names.append(dns_name)
except x509.ExtensionNotFound:
pass
return dns_names

37
lemur/common/celery.py
View File

@ -18,8 +18,11 @@ from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.plugins.base import plugins, IPlugin
from lemur.sources.cli import clean, sync, validate_sources
from lemur.destinations import service as destinations_service
from lemur.sources import service as sources_service
if current_app:
flask_app = current_app
@ -255,3 +258,35 @@ def sync_source(source):
sync([source])
log_data["message"] = "Done syncing source"
current_app.logger.debug(log_data)
@celery.task()
def sync_source_destination():
"""
This celery task will sync destination and source, to make sure all new destinations are also present as source.
Some destinations do not qualify as sources, and hence should be excluded from being added as sources
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reviels the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
"""
current_app.logger.debug("Syncing source and destination")
# a set of all accounts numbers available as sources
src_accounts = set()
sources = validate_sources("all")
for src in sources:
src_accounts.add(IPlugin.get_option('accountNumber', src.options))
for dst in destinations_service.get_all():
destination_plugin = plugins.get(dst.plugin_name)
account_number = IPlugin.get_option('accountNumber', dst.options)
if destination_plugin.sync_as_source and (account_number not in src_accounts):
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
for o in src_options:
if o.get('name') == 'accountNumber':
o.update({'value': account_number})
sources_service.create(label=dst.label,
plugin_name=destination_plugin.sync_as_source_name,
options=src_options,
description=dst.description)
current_app.logger.info("Source: %s added", dst.label)

3
lemur/manage.py
View File

@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy # noqa
from lemur.pending_certificates.models import PendingCertificate # noqa
from lemur.dns_providers.models import DnsProvider # noqa
from sqlalchemy.sql import text
manager = Manager(create_app)
manager.add_option('-c', '--config', dest='config_path', required=False)
@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
@MigrateCommand.command
def create():
database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
database.db.create_all()
stamp(revision='head')

2
lemur/plugins/bases/destination.py
View File

@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
class DestinationPlugin(Plugin):
type = 'destination'
requires_key = True
sync_as_source = False
sync_as_source_name = ''
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
raise NotImplementedError

5
lemur/plugins/lemur_acme/plugin.py
View File

@ -459,7 +459,10 @@ class ACMEIssuerPlugin(IssuerPlugin):
"pending_cert": entry["pending_cert"],
})
except (PollError, AcmeError, Exception) as e:
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
order_url = order.uri
current_app.logger.error(
"Unable to resolve pending cert: {}. "
"Check out {} for more information.".format(pending_cert, order_url), exc_info=True)
certs.append({
"cert": False,
"pending_cert": entry["pending_cert"],

78
lemur/plugins/lemur_aws/plugin.py
View File

@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
return endpoints
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
# 'elb': {
# 'name': {'type': 'name'},
# 'region': {'type': 'str'},
# 'port': {'type': 'int'}
# }
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class AWSSourcePlugin(SourcePlugin):
title = 'AWS'
slug = 'aws-source'
@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
iam.delete_cert(certificate.name, account_number=account_number)
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
sync_as_source = True
sync_as_source_name = AWSSourcePlugin.slug
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class S3DestinationPlugin(ExportDestinationPlugin):
title = 'AWS-S3'
slug = 'aws-s3'

53
lemur/plugins/lemur_vault_dest/plugin.py
View File

@ -9,6 +9,7 @@
.. moduleauthor:: Christopher Jolley <chris@alwaysjolley.com>
"""
import os
import re
import hvac
from flask import current_app
@ -20,6 +21,14 @@ from lemur.plugins.bases import DestinationPlugin
from cryptography import x509
from cryptography.hazmat.backends import default_backend
class Error(Exception):
"""Base exception class"""
pass
class InvalidSanError(Error):
"""Invlied SAN in SAN list as defined by regex in destination"""
pass
class VaultDestinationPlugin(DestinationPlugin):
"""Hashicorp Vault Destination plugin for Lemur"""
title = 'Vault'
@ -37,6 +46,17 @@ class VaultDestinationPlugin(DestinationPlugin):
'validation': '^https?://[a-zA-Z0-9.:-]+$',
'helpMessage': 'Valid URL to Hashi Vault instance'
},
{
'name': 'vaultKvApiVersion',
'type': 'select',
'value': '2',
'available': [
'1',
'2'
],
'required': True,
'helpMessage': 'Version of the Vault KV API to use'
},
{
'name': 'vaultAuthTokenFile',
'type': 'str',
@ -80,8 +100,9 @@ class VaultDestinationPlugin(DestinationPlugin):
{
'name': 'sanFilter',
'type': 'str',
'value': '.*',
'required': False,
'validation': '^[0-9a-zA-Z\\\?\[\](){}^$+._-]+$',
'validation': '^[0-9a-zA-Z\\\?\[\](){}|^$+*,._-]+$',
'helpMessage': 'Valid regex filter'
}
]
@ -105,25 +126,30 @@ class VaultDestinationPlugin(DestinationPlugin):
path = self.get_option('vaultPath', options)
bundle = self.get_option('bundleChain', options)
obj_name = self.get_option('objectName', options)
api_version = self.get_option('vaultKvApiVersion', options)
san_filter = self.get_option('sanFilter', options)
san_list = get_san_list(body)
for san in san_list:
if not re.match(san_filter, san):
current_app.logger.exception(
"Exception uploading secret to vault: invalid SAN in certificate",
exc_info=True)
if san_filter:
for san in san_list:
if not re.match(san_filter, san, flags=re.IGNORECASE):
current_app.logger.exception(
"Exception uploading secret to vault: invalid SAN: {}".format(san),
exc_info=True)
os._exit(1)
with open(token_file, 'r') as file:
token = file.readline().rstrip('\n')
client = hvac.Client(url=url, token=token)
client.secrets.kv.default_kv_version = api_version
if obj_name:
path = '{0}/{1}'.format(path, obj_name)
else:
path = '{0}/{1}'.format(path, cname)
secret = get_secret(url, token, mount, path)
secret = get_secret(client, mount, path)
secret['data'][cname] = {}
if bundle == 'Nginx' and cert_chain:
@ -137,8 +163,9 @@ class VaultDestinationPlugin(DestinationPlugin):
if isinstance(san_list, list):
secret['data'][cname]['san'] = san_list
try:
client.secrets.kv.v1.create_or_update_secret(
path=path, mount_point=mount, secret=secret['data'])
client.secrets.kv.create_or_update_secret(
path=path, mount_point=mount, secret=secret['data']
)
except ConnectionError as err:
current_app.logger.exception(
"Exception uploading secret to vault: {0}".format(err), exc_info=True)
@ -158,12 +185,14 @@ def get_san_list(body):
return san_list
def get_secret(url, token, mount, path):
def get_secret(client, mount, path):
""" retreiive existing data from mount path and return dictionary """
result = {'data': {}}
try:
client = hvac.Client(url=url, token=token)
result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount)
if client.secrets.kv.default_kv_version == '1':
result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount)
else:
result = client.secrets.kv.v2.read_secret_version(path=path, mount_point=mount)
except ConnectionError:
pass
finally:

2
lemur/tests/conftest.py
View File

@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from flask import current_app
from flask_principal import identity_changed, Identity
from sqlalchemy.sql import text
from lemur import create_app
from lemur.common.utils import parse_private_key
@ -55,6 +56,7 @@ def app(request):
@pytest.yield_fixture(scope="session")
def db(app, request):
_db.drop_all()
_db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
_db.create_all()
_db.app = app