adding regex filtering

This commit is contained in:
alwaysjolley 2019-04-18 13:52:40 -04:00
commit fb3b0e8cd7
17 changed files with 243 additions and 114 deletions

View File

@ -1,6 +1,6 @@
language: python
sudo: required
dist: trusty
dist: xenial
node_js:
- "6.2.0"
@ -10,8 +10,8 @@ addons:
matrix:
include:
- python: "3.5"
env: TOXENV=py35
- python: "3.7"
env: TOXENV=py37
cache:
directories:

View File

@ -13,10 +13,13 @@ services:
VIRTUAL_ENV: 'true'
postgres:
image: postgres:9.4
image: postgres
restart: always
environment:
POSTGRES_USER: lemur
POSTGRES_PASSWORD: lemur
ports:
- "5432:5432"
redis:
image: "redis:alpine"

View File

@ -10,6 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load
from marshmallow.exceptions import ValidationError
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.certificates import utils as cert_utils
from lemur.common import missing, utils, validators
from lemur.common.fields import ArrowDateTime, Hex
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
@ -96,6 +97,9 @@ class CertificateInputSchema(CertificateCreationSchema):
@validates_schema
def validate_authority(self, data):
if isinstance(data['authority'], str):
raise ValidationError("Authority not found.")
if not data['authority'].active:
raise ValidationError("The authority is inactive.", ['authority'])
@ -107,6 +111,11 @@ class CertificateInputSchema(CertificateCreationSchema):
def load_data(self, data):
if data.get('replacements'):
data['replaces'] = data['replacements'] # TODO remove when field is deprecated
if data.get('csr'):
dns_names = cert_utils.get_dns_names_from_csr(data['csr'])
if not data['extensions']['subAltNames']['names']:
data['extensions']['subAltNames']['names'] = []
data['extensions']['subAltNames']['names'] += dns_names
return missing.convert_validity_years(data)

View File

@ -0,0 +1,42 @@
"""
Utils to parse certificate data.
.. module: lemur.certificates.hooks
:platform: Unix
:copyright: (c) 2019 by Javier Ramos, see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Javier Ramos <javier.ramos@booking.com>
"""
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from marshmallow.exceptions import ValidationError
def get_dns_names_from_csr(data):
"""
Fetches DNSNames from CSR.
Potentially extendable to any kind of SubjectAlternativeName
:param data: PEM-encoded string with CSR
:return:
"""
dns_names = []
try:
request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend())
except Exception:
raise ValidationError('CSR presented is not valid.')
try:
alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName)
for name in alt_names.value.get_values_for_type(x509.DNSName):
dns_name = {
'nameType': 'DNSName',
'value': name
}
dns_names.append(dns_name)
except x509.ExtensionNotFound:
pass
return dns_names

View File

@ -18,8 +18,11 @@ from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.plugins.base import plugins, IPlugin
from lemur.sources.cli import clean, sync, validate_sources
from lemur.destinations import service as destinations_service
from lemur.sources import service as sources_service
if current_app:
flask_app = current_app
@ -255,3 +258,35 @@ def sync_source(source):
sync([source])
log_data["message"] = "Done syncing source"
current_app.logger.debug(log_data)
@celery.task()
def sync_source_destination():
"""
This celery task will sync destination and source, to make sure all new destinations are also present as source.
Some destinations do not qualify as sources, and hence should be excluded from being added as sources
We identify qualified destinations based on the sync_as_source attributed of the plugin.
The destination sync_as_source_name reviels the name of the suitable source-plugin.
We rely on account numbers to avoid duplicates.
"""
current_app.logger.debug("Syncing source and destination")
# a set of all accounts numbers available as sources
src_accounts = set()
sources = validate_sources("all")
for src in sources:
src_accounts.add(IPlugin.get_option('accountNumber', src.options))
for dst in destinations_service.get_all():
destination_plugin = plugins.get(dst.plugin_name)
account_number = IPlugin.get_option('accountNumber', dst.options)
if destination_plugin.sync_as_source and (account_number not in src_accounts):
src_options = copy.deepcopy(plugins.get(destination_plugin.sync_as_source_name).options)
for o in src_options:
if o.get('name') == 'accountNumber':
o.update({'value': account_number})
sources_service.create(label=dst.label,
plugin_name=destination_plugin.sync_as_source_name,
options=src_options,
description=dst.description)
current_app.logger.info("Source: %s added", dst.label)

View File

@ -49,6 +49,8 @@ from lemur.policies.models import RotationPolicy # noqa
from lemur.pending_certificates.models import PendingCertificate # noqa
from lemur.dns_providers.models import DnsProvider # noqa
from sqlalchemy.sql import text
manager = Manager(create_app)
manager.add_option('-c', '--config', dest='config_path', required=False)
@ -142,6 +144,7 @@ SQLALCHEMY_DATABASE_URI = 'postgresql://lemur:lemur@localhost:5432/lemur'
@MigrateCommand.command
def create():
database.db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
database.db.create_all()
stamp(revision='head')

View File

@ -12,6 +12,8 @@ from lemur.plugins.base import Plugin, plugins
class DestinationPlugin(Plugin):
type = 'destination'
requires_key = True
sync_as_source = False
sync_as_source_name = ''
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
raise NotImplementedError

View File

@ -459,7 +459,10 @@ class ACMEIssuerPlugin(IssuerPlugin):
"pending_cert": entry["pending_cert"],
})
except (PollError, AcmeError, Exception) as e:
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
order_url = order.uri
current_app.logger.error(
"Unable to resolve pending cert: {}. "
"Check out {} for more information.".format(pending_cert, order_url), exc_info=True)
certs.append({
"cert": False,
"pending_cert": entry["pending_cert"],

View File

@ -149,47 +149,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict):
return endpoints
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
# 'elb': {
# 'name': {'type': 'name'},
# 'region': {'type': 'str'},
# 'port': {'type': 'int'}
# }
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class AWSSourcePlugin(SourcePlugin):
title = 'AWS'
slug = 'aws-source'
@ -266,6 +225,43 @@ class AWSSourcePlugin(SourcePlugin):
iam.delete_cert(certificate.name, account_number=account_number)
class AWSDestinationPlugin(DestinationPlugin):
title = 'AWS'
slug = 'aws-destination'
description = 'Allow the uploading of certificates to AWS IAM'
version = aws.VERSION
sync_as_source = True
sync_as_source_name = AWSSourcePlugin.slug
author = 'Kevin Glisson'
author_url = 'https://github.com/netflix/lemur'
options = [
{
'name': 'accountNumber',
'type': 'str',
'required': True,
'validation': '[0-9]{12}',
'helpMessage': 'Must be a valid AWS account number!',
},
{
'name': 'path',
'type': 'str',
'default': '/',
'helpMessage': 'Path to upload certificate.'
}
]
def upload(self, name, body, private_key, cert_chain, options, **kwargs):
iam.upload_cert(name, body, private_key,
self.get_option('path', options),
cert_chain=cert_chain,
account_number=self.get_option('accountNumber', options))
def deploy(self, elb_name, account, region, certificate):
pass
class S3DestinationPlugin(ExportDestinationPlugin):
title = 'AWS-S3'
slug = 'aws-s3'

View File

@ -9,6 +9,7 @@
.. moduleauthor:: Christopher Jolley <chris@alwaysjolley.com>
"""
import os
import re
import hvac
from flask import current_app
@ -20,6 +21,14 @@ from lemur.plugins.bases import DestinationPlugin
from cryptography import x509
from cryptography.hazmat.backends import default_backend
class Error(Exception):
"""Base exception class"""
pass
class InvalidSanError(Error):
"""Invlied SAN in SAN list as defined by regex in destination"""
pass
class VaultDestinationPlugin(DestinationPlugin):
"""Hashicorp Vault Destination plugin for Lemur"""
title = 'Vault'
@ -37,6 +46,17 @@ class VaultDestinationPlugin(DestinationPlugin):
'validation': '^https?://[a-zA-Z0-9.:-]+$',
'helpMessage': 'Valid URL to Hashi Vault instance'
},
{
'name': 'vaultKvApiVersion',
'type': 'select',
'value': '2',
'available': [
'1',
'2'
],
'required': True,
'helpMessage': 'Version of the Vault KV API to use'
},
{
'name': 'vaultAuthTokenFile',
'type': 'str',
@ -80,8 +100,9 @@ class VaultDestinationPlugin(DestinationPlugin):
{
'name': 'sanFilter',
'type': 'str',
'value': '.*',
'required': False,
'validation': '^[0-9a-zA-Z\\\?\[\](){}^$+._-]+$',
'validation': '^[0-9a-zA-Z\\\?\[\](){}|^$+*,._-]+$',
'helpMessage': 'Valid regex filter'
}
]
@ -105,25 +126,30 @@ class VaultDestinationPlugin(DestinationPlugin):
path = self.get_option('vaultPath', options)
bundle = self.get_option('bundleChain', options)
obj_name = self.get_option('objectName', options)
api_version = self.get_option('vaultKvApiVersion', options)
san_filter = self.get_option('sanFilter', options)
san_list = get_san_list(body)
for san in san_list:
if not re.match(san_filter, san):
current_app.logger.exception(
"Exception uploading secret to vault: invalid SAN in certificate",
exc_info=True)
if san_filter:
for san in san_list:
if not re.match(san_filter, san, flags=re.IGNORECASE):
current_app.logger.exception(
"Exception uploading secret to vault: invalid SAN: {}".format(san),
exc_info=True)
os._exit(1)
with open(token_file, 'r') as file:
token = file.readline().rstrip('\n')
client = hvac.Client(url=url, token=token)
client.secrets.kv.default_kv_version = api_version
if obj_name:
path = '{0}/{1}'.format(path, obj_name)
else:
path = '{0}/{1}'.format(path, cname)
secret = get_secret(url, token, mount, path)
secret = get_secret(client, mount, path)
secret['data'][cname] = {}
if bundle == 'Nginx' and cert_chain:
@ -137,8 +163,9 @@ class VaultDestinationPlugin(DestinationPlugin):
if isinstance(san_list, list):
secret['data'][cname]['san'] = san_list
try:
client.secrets.kv.v1.create_or_update_secret(
path=path, mount_point=mount, secret=secret['data'])
client.secrets.kv.create_or_update_secret(
path=path, mount_point=mount, secret=secret['data']
)
except ConnectionError as err:
current_app.logger.exception(
"Exception uploading secret to vault: {0}".format(err), exc_info=True)
@ -158,12 +185,14 @@ def get_san_list(body):
return san_list
def get_secret(url, token, mount, path):
def get_secret(client, mount, path):
""" retreiive existing data from mount path and return dictionary """
result = {'data': {}}
try:
client = hvac.Client(url=url, token=token)
result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount)
if client.secrets.kv.default_kv_version == '1':
result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount)
else:
result = client.secrets.kv.v2.read_secret_version(path=path, mount_point=mount)
except ConnectionError:
pass
finally:

View File

@ -7,6 +7,7 @@ from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from flask import current_app
from flask_principal import identity_changed, Identity
from sqlalchemy.sql import text
from lemur import create_app
from lemur.common.utils import parse_private_key
@ -55,6 +56,7 @@ def app(request):
@pytest.yield_fixture(scope="session")
def db(app, request):
_db.drop_all()
_db.engine.execute(text('CREATE EXTENSION IF NOT EXISTS pg_trgm'))
_db.create_all()
_db.app = app

View File

@ -7,18 +7,18 @@
aspy.yaml==1.2.0 # via pre-commit
bleach==3.1.0 # via readme-renderer
certifi==2019.3.9 # via requests
cfgv==1.5.0 # via pre-commit
cfgv==1.6.0 # via pre-commit
chardet==3.0.4 # via requests
docutils==0.14 # via readme-renderer
flake8==3.5.0
identify==1.4.0 # via pre-commit
identify==1.4.1 # via pre-commit
idna==2.8 # via requests
importlib-metadata==0.8 # via pre-commit
importlib-metadata==0.9 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.3
pkginfo==1.5.0.1 # via twine
pre-commit==1.14.4
pre-commit==1.15.1
pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8
pygments==2.3.1 # via readme-renderer

View File

@ -4,7 +4,7 @@
#
# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
#
acme==0.32.0
acme==0.33.1
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.8
@ -15,11 +15,11 @@ asn1crypto==0.24.0
asyncpool==1.0
babel==2.6.0 # via sphinx
bcrypt==3.1.6
billiard==3.5.0.5
billiard==3.6.0.0
blinker==1.4
boto3==1.9.116
botocore==1.12.116
celery[redis]==4.2.1
boto3==1.9.130
botocore==1.12.130
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
cffi==1.12.2
@ -42,28 +42,28 @@ flask-sqlalchemy==2.3.2
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
hvac==0.7.2
hvac==0.8.2
idna==2.8
imagesize==1.1.0 # via sphinx
inflection==0.3.1
itsdangerous==1.1.0
jinja2==2.10
jinja2==2.10.1
jmespath==0.9.4
josepy==1.1.0
jsonlines==1.2.0
kombu==4.3.0
kombu==4.5.0
lockfile==0.12.2
mako==1.0.7
mako==1.0.8
markupsafe==1.1.1
marshmallow-sqlalchemy==0.16.1
marshmallow==2.19.1
marshmallow==2.19.2
mock==2.0.0
ndg-httpsclient==0.5.1
packaging==19.0 # via sphinx
paramiko==2.4.2
pbr==5.1.3
pem==18.2.0
psycopg2==2.7.7
pem==19.1.0
psycopg2==2.8.1
pyasn1-modules==0.2.4
pyasn1==0.4.5
pycparser==2.19
@ -71,14 +71,14 @@ pygments==2.3.1 # via sphinx
pyjwt==1.7.1
pynacl==1.3.0
pyopenssl==19.0.0
pyparsing==2.3.1 # via packaging
pyparsing==2.4.0 # via packaging
pyrfc3339==1.1
python-dateutil==2.8.0
python-editor==1.0.4
pytz==2018.9
pytz==2019.1
pyyaml==5.1
raven[flask]==6.10.0
redis==2.10.6
redis==3.2.1
requests-toolbelt==0.9.1
requests[security]==2.21.0
retrying==1.3.3
@ -86,13 +86,18 @@ s3transfer==0.2.0
six==1.12.0
snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.4.3
sphinx==1.8.5
sphinx==2.0.1
sphinxcontrib-applehelp==1.0.1 # via sphinx
sphinxcontrib-devhelp==1.0.1 # via sphinx
sphinxcontrib-htmlhelp==1.0.1 # via sphinx
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx
sphinxcontrib-jsmath==1.0.1 # via sphinx
sphinxcontrib-qthelp==1.0.2 # via sphinx
sphinxcontrib-serializinghtml==1.1.3 # via sphinx
sqlalchemy-utils==0.33.11
sqlalchemy==1.3.1
sqlalchemy==1.3.2
tabulate==0.8.3
urllib3==1.24.1
vine==1.2.0
werkzeug==0.14.1
vine==1.3.0
werkzeug==0.15.2
xmltodict==0.12.0

View File

@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.3.0 # via pytest
attrs==19.1.0 # via pytest
aws-xray-sdk==0.95 # via moto
boto3==1.9.116 # via moto
boto3==1.9.130 # via moto
boto==2.49.0 # via moto
botocore==1.12.116 # via boto3, moto, s3transfer
botocore==1.12.130 # via boto3, moto, s3transfer
certifi==2019.3.9 # via requests
cffi==1.12.2 # via cryptography
chardet==3.0.4 # via requests
@ -18,7 +18,7 @@ click==7.0 # via flask
coverage==4.5.3
cryptography==2.6.1 # via moto
docker-pycreds==0.4.0 # via docker
docker==3.7.0 # via moto
docker==3.7.2 # via moto
docutils==0.14 # via botocore
ecdsa==0.13 # via python-jose
factory-boy==2.11.1
@ -28,13 +28,13 @@ freezegun==0.3.11
future==0.17.1 # via python-jose
idna==2.8 # via requests
itsdangerous==1.1.0 # via flask
jinja2==2.10 # via flask, moto
jinja2==2.10.1 # via flask, moto
jmespath==0.9.4 # via boto3, botocore
jsondiff==1.1.1 # via moto
jsonpickle==1.1 # via aws-xray-sdk
markupsafe==1.1.1 # via jinja2
mock==2.0.0 # via moto
more-itertools==6.0.0 # via pytest
more-itertools==7.0.0 # via pytest
moto==1.3.7
nose==1.3.7
pbr==5.1.3 # via mock
@ -42,14 +42,14 @@ pluggy==0.9.0 # via pytest
py==1.8.0 # via pytest
pyaml==18.11.0 # via moto
pycparser==2.19 # via cffi
pycryptodome==3.7.3 # via python-jose
pycryptodome==3.8.1 # via python-jose
pyflakes==2.1.1
pytest-flask==0.14.0
pytest-mock==1.10.1
pytest==4.3.1
pytest-mock==1.10.3
pytest==4.4.0
python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.9 # via moto
pytz==2019.1 # via moto
pyyaml==5.1
requests-mock==1.5.2
requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses
@ -58,7 +58,7 @@ s3transfer==0.2.0 # via boto3
six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker
urllib3==1.24.1 # via botocore, requests
websocket-client==0.55.0 # via docker
werkzeug==0.14.1 # via flask, moto, pytest-flask
websocket-client==0.56.0 # via docker
werkzeug==0.15.2 # via flask, moto, pytest-flask
wrapt==1.11.1 # via aws-xray-sdk
xmltodict==0.12.0 # via moto

View File

@ -27,7 +27,7 @@ gunicorn
hvac # required for the vault destination plugin
inflection
jinja2
kombu==4.3.0 # kombu 4.4.0 requires redis 3
kombu
lockfile
marshmallow-sqlalchemy
marshmallow
@ -39,7 +39,7 @@ pyjwt
pyOpenSSL
python_ldap
raven[flask]
redis<3 # redis>=3 is not compatible with celery
redis
requests
retrying
six

View File

@ -4,7 +4,7 @@
#
# pip-compile --output-file requirements.txt requirements.in -U --no-index
#
acme==0.32.0
acme==0.33.1
alembic-autogenerate-enums==0.0.2
alembic==1.0.8 # via flask-migrate
amqp==2.4.2 # via kombu
@ -13,11 +13,11 @@ arrow==0.13.1
asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
bcrypt==3.1.6 # via flask-bcrypt, paramiko
billiard==3.5.0.5 # via celery
billiard==3.6.0.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.9.116
botocore==1.12.116
celery[redis]==4.2.1
boto3==1.9.130
botocore==1.12.130
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
cffi==1.12.2 # via bcrypt, cryptography, pynacl
@ -40,26 +40,26 @@ flask-sqlalchemy==2.3.2
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
hvac==0.7.2
hvac==0.8.2
idna==2.8 # via requests
inflection==0.3.1
itsdangerous==1.1.0 # via flask
jinja2==2.10
jinja2==2.10.1
jmespath==0.9.4 # via boto3, botocore
josepy==1.1.0 # via acme
jsonlines==1.2.0 # via cloudflare
kombu==4.3.0
kombu==4.5.0
lockfile==0.12.2
mako==1.0.7 # via alembic
mako==1.0.8 # via alembic
markupsafe==1.1.1 # via jinja2, mako
marshmallow-sqlalchemy==0.16.1
marshmallow==2.19.1
marshmallow==2.19.2
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.1.3 # via mock
pem==18.2.0
psycopg2==2.7.7
pem==19.1.0
psycopg2==2.8.1
pyasn1-modules==0.2.4 # via python-ldap
pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
pycparser==2.19 # via cffi
@ -70,19 +70,19 @@ pyrfc3339==1.1 # via acme
python-dateutil==2.8.0 # via alembic, arrow, botocore
python-editor==1.0.4 # via alembic
python-ldap==3.2.0
pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339
pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339
pyyaml==5.1
raven[flask]==6.10.0
redis==2.10.6
redis==3.2.1
requests-toolbelt==0.9.1 # via acme
requests[security]==2.21.0
retrying==1.3.3
s3transfer==0.2.0 # via boto3
six==1.12.0
sqlalchemy-utils==0.33.11
sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlalchemy==1.3.2 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.3
urllib3==1.24.1 # via botocore, requests
vine==1.2.0 # via amqp
werkzeug==0.14.1 # via flask
vine==1.3.0 # via amqp, celery
werkzeug==0.15.2 # via flask
xmltodict==0.12.0

View File

@ -1,2 +1,2 @@
[tox]
envlist = py35
envlist = py37