From eaa73998a0b17858d83fe21080f78f2ec4d2c1f3 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 19 Feb 2019 15:03:15 -0500 Subject: [PATCH 01/12] adding lemur_vault destination plugin --- lemur/plugins/lemur_vault/__init__.py | 5 ++ lemur/plugins/lemur_vault/plugin.py | 85 +++++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 lemur/plugins/lemur_vault/__init__.py create mode 100644 lemur/plugins/lemur_vault/plugin.py diff --git a/lemur/plugins/lemur_vault/__init__.py b/lemur/plugins/lemur_vault/__init__.py new file mode 100644 index 00000000..8ce5a7f3 --- /dev/null +++ b/lemur/plugins/lemur_vault/__init__.py @@ -0,0 +1,5 @@ +try: + VERSION = __import__('pkg_resources') \ + .get_distribution(__name__).version +except Exception as e: + VERSION = 'unknown' diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault/plugin.py new file mode 100644 index 00000000..505170ad --- /dev/null +++ b/lemur/plugins/lemur_vault/plugin.py @@ -0,0 +1,85 @@ +""" +.. module: lemur.plugins.lemur_vault.plugin + :platform: Unix + :copyright: (c) 2019 + :license: Apache, see LICENCE for more details. + + Plugin for uploading certificates and private key as secret to hashi vault + that can be pulled down by end point nodes. + +.. moduleauthor:: Christopher Jolley +""" +import hvac + +#import lemur_vault +from flask import current_app + +from lemur.common.defaults import common_name +from lemur.common.utils import parse_certificate +from lemur.plugins.bases import DestinationPlugin + +class VaultDestinationPlugin(DestinationPlugin): + """Hashicorp Vault Destination plugin for Lemur""" + title = 'Vault' + slug = 'hashi-vault-destination' + description = 'Allow the uploading of certificates to Hashi Vault as secret' + + author = 'Christopher Jolley' + author_url = 'https://github.com/alwaysjolley/lemur' + + options = [ + { + 'name': 'vaultMount', + 'type': 'str', + 'required': True, + 'validation': '^[a-zA-Z0-9]+$', + 'helpMessage': 'Must be a valid Vault secrets mount name!' + }, + { + 'name': 'vaultPath', + 'type': 'str', + 'required': True, + 'validation': '^([a-zA-Z0-9_-]+/?)+$', + 'helpMessage': 'Must be a valid Vault secrets path' + }, + { + 'name': 'vaultUrl', + 'type': 'str', + 'required': True, + 'validation': '^https?://[a-zA-Z0-9.-]+(?::[0-9]+)?$', + 'helpMessage': 'Must be a valid Vault server url' + } + ] + + def __init__(self, *args, **kwargs): + super(VaultDestinationPlugin, self).__init__(*args, **kwargs) + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + """ + Upload certificate and private key + + :param private_key: + :param cert_chain: + :return: + """ + cn = common_name(parse_certificate(body)) + data = {} + #current_app.logger.warning("Cert body content: {0}".format(body)) + + token = current_app.config.get('VAULT_TOKEN') + + mount = self.get_option('vaultMount', options) + path = '{0}/{1}'.format(self.get_option('vaultPath', options),cn) + url = self.get_option('vaultUrl', options) + + client = hvac.Client(url=url, token=token) + + data['cert'] = cert_chain + data['key'] = private_key + + ## upload certificate and key + try: + client.secrets.kv.v1.create_or_update_secret(path=path, mount_point=mount, secret=data) + except Exception as err: + current_app.logger.exception( + "Exception uploading secret to vault: {0}".format(err), exc_info=True) diff --git a/setup.py b/setup.py index 1511b013..b5dcdb3b 100644 --- a/setup.py +++ b/setup.py @@ -154,7 +154,8 @@ setup( 'digicert_cis_issuer = lemur.plugins.lemur_digicert.plugin:DigiCertCISIssuerPlugin', 'digicert_cis_source = lemur.plugins.lemur_digicert.plugin:DigiCertCISSourcePlugin', 'csr_export = lemur.plugins.lemur_csr.plugin:CSRExportPlugin', - 'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin' + 'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin', + 'vault_desination = lemur.plugins.lemur_vault.plugin:VaultDestinationPlugin' ], }, classifiers=[ From a0ca486f0f9975eeffe6dddb13ed3fe60eee9661 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 19 Feb 2019 15:22:11 -0500 Subject: [PATCH 02/12] adding hvac and updating requrements --- requirements-dev.txt | 12 ++++++------ requirements-docs.txt | 39 ++++++++++++++++++++------------------- requirements-tests.txt | 24 ++++++++++++------------ requirements.in | 3 ++- requirements.txt | 35 ++++++++++++++++++----------------- 5 files changed, 58 insertions(+), 55 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ac35f3e9..6e2a3fb9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.2.1 # via pre-commit +identify==1.2.2 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit importlib-resources==1.0.2 # via pre-commit @@ -19,19 +19,19 @@ invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.2 +pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine -requests-toolbelt==0.9.0 # via twine +requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.30.0 # via twine -twine==1.12.1 +tqdm==4.31.1 # via twine +twine==1.13.0 urllib3==1.24.1 # via requests -virtualenv==16.3.0 # via pre-commit +virtualenv==16.4.0 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index 15085766..e9dd92cb 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,24 +4,24 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.30.2 +acme==0.31.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.7 -amqp==2.4.0 +amqp==2.4.1 aniso8601==4.1.0 -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.86 -botocore==1.12.86 +boto3==1.9.98 +botocore==1.12.98 celery[redis]==4.2.1 certifi==2018.11.29 -cffi==1.11.5 +cffi==1.12.1 chardet==3.0.4 click==7.0 cloudflare==2.1.0 @@ -33,7 +33,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -41,6 +41,7 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 +hvac==0.7.2 idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 @@ -49,17 +50,17 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.2.post1 +kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 -marshmallow-sqlalchemy==0.15.0 -marshmallow==2.18.0 +marshmallow-sqlalchemy==0.16.0 +marshmallow==2.18.1 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 -pbr==5.1.1 +pbr==5.1.2 pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 @@ -71,26 +72,26 @@ pynacl==1.3.0 pyopenssl==19.0.0 pyparsing==2.3.1 # via packaging pyrfc3339==1.1 -python-dateutil==2.7.5 -python-editor==1.0.3 +python-dateutil==2.8.0 +python-editor==1.0.4 pytz==2018.9 pyyaml==3.13 raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.9.0 +requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 -s3transfer==0.1.13 +s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.4.2 -sphinx==1.8.3 +sphinx-rtd-theme==0.4.3 +sphinx==1.8.4 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 +sqlalchemy==1.2.18 tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 werkzeug==0.14.1 -xmltodict==0.11.0 +xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index c326e951..1bb8ba03 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,14 +5,14 @@ # pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography -atomicwrites==1.2.1 # via pytest +atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.86 # via moto +boto3==1.9.98 # via moto boto==2.49.0 # via moto -botocore==1.12.86 # via boto3, moto, s3transfer +botocore==1.12.98 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests -cffi==1.11.5 # via cryptography +cffi==1.12.1 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 @@ -34,10 +34,10 @@ jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==5.0.0 # via pytest +more-itertools==6.0.0 # via pytest moto==1.3.7 nose==1.3.7 -pbr==5.1.1 # via mock +pbr==5.1.2 # via mock pluggy==0.8.1 # via pytest py==1.7.0 # via pytest pyaml==18.11.0 # via moto @@ -45,20 +45,20 @@ pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose pyflakes==2.1.0 pytest-flask==0.14.0 -pytest-mock==1.10.0 -pytest==4.1.1 -python-dateutil==2.7.5 # via botocore, faker, freezegun, moto +pytest-mock==1.10.1 +pytest==4.3.0 +python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.10.5 # via moto -s3transfer==0.1.13 # via boto3 -six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client +s3transfer==0.2.0 # via boto3 +six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk -xmltodict==0.11.0 # via moto +xmltodict==0.12.0 # via moto diff --git a/requirements.in b/requirements.in index 9824650b..1147cc8d 100644 --- a/requirements.in +++ b/requirements.in @@ -23,6 +23,7 @@ Flask Flask-Cors future gunicorn +hvac # required for the vault destination plugin inflection jinja2 lockfile @@ -42,4 +43,4 @@ retrying six SQLAlchemy-Utils tabulate -xmltodict \ No newline at end of file +xmltodict diff --git a/requirements.txt b/requirements.txt index c595e509..edd56b09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,22 +4,22 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.30.2 +acme==0.31.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.7 # via flask-migrate -amqp==2.4.0 # via kombu +amqp==2.4.1 # via kombu aniso8601==4.1.0 # via flask-restful -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.86 -botocore==1.12.86 +boto3==1.9.98 +botocore==1.12.98 celery[redis]==4.2.1 certifi==2018.11.29 -cffi==1.11.5 # via bcrypt, cryptography, pynacl +cffi==1.12.1 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 @@ -31,7 +31,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -39,6 +39,7 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 +hvac==0.7.2 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask @@ -46,16 +47,16 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.2.post1 # via celery +kombu==4.3.0 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako -marshmallow-sqlalchemy==0.15.0 -marshmallow==2.18.0 +marshmallow-sqlalchemy==0.16.0 +marshmallow==2.18.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.1.1 # via mock +pbr==5.1.2 # via mock pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 # via python-ldap @@ -65,22 +66,22 @@ pyjwt==1.7.1 pynacl==1.3.0 # via paramiko pyopenssl==19.0.0 pyrfc3339==1.1 # via acme -python-dateutil==2.7.5 # via alembic, arrow, botocore -python-editor==1.0.3 # via alembic +python-dateutil==2.8.0 # via alembic, arrow, botocore +python-editor==1.0.4 # via alembic python-ldap==3.1.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.9.0 # via acme +requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 -s3transfer==0.1.13 # via boto3 +s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.18 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask -xmltodict==0.11.0 +xmltodict==0.12.0 From cd65a36437e05d6cd4f5e29b76c6f5e76567beb1 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Mon, 25 Feb 2019 09:42:07 -0500 Subject: [PATCH 03/12] - support multiple bundle configuration, nginx, apache, cert only - update vault destination to support multi cert under one object - added san list as key value - read and update object with new keys, keeping other keys, allowing us to keep an iterable list of keys in an object for deploying multiple certs to a single node --- .gitignore | 5 ++ lemur/plugins/lemur_vault/plugin.py | 81 +++++++++++++++++++++++++---- requirements-dev.txt | 6 +-- requirements-docs.txt | 10 ++-- requirements-tests.txt | 16 +++--- requirements.txt | 10 ++-- 6 files changed, 98 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 97af00ca..72e85f26 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,11 @@ package-lock.json /lemur/static/dist/ /lemur/static/app/vendor/ /wheelhouse +/lemur/lib +/lemur/bin +/lemur/lib64 +/lemur/include + docs/_build .editorconfig .idea diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault/plugin.py index 505170ad..58a9e601 100644 --- a/lemur/plugins/lemur_vault/plugin.py +++ b/lemur/plugins/lemur_vault/plugin.py @@ -18,6 +18,10 @@ from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate from lemur.plugins.bases import DestinationPlugin +from cryptography import x509 +from cryptography.hazmat.backends import default_backend + + class VaultDestinationPlugin(DestinationPlugin): """Hashicorp Vault Destination plugin for Lemur""" title = 'Vault' @@ -48,6 +52,25 @@ class VaultDestinationPlugin(DestinationPlugin): 'required': True, 'validation': '^https?://[a-zA-Z0-9.-]+(?::[0-9]+)?$', 'helpMessage': 'Must be a valid Vault server url' + }, + { + 'name': 'bundleChain', + 'type': 'select', + 'value': 'cert only', + 'available': [ + 'Nginx', + 'Apache', + 'no chain' + ], + 'required': True, + 'helpMessage': 'Bundle the chain into the certificate' + }, + { + 'name': 'objectName', + 'type': 'str', + 'required': False, + 'validation': '[0-9a-zA-Z:_-]+', + 'helpMessage': 'Name to bundle certs under, if blank use cn' } ] @@ -62,24 +85,64 @@ class VaultDestinationPlugin(DestinationPlugin): :param cert_chain: :return: """ - cn = common_name(parse_certificate(body)) - data = {} - #current_app.logger.warning("Cert body content: {0}".format(body)) + cname = common_name(parse_certificate(body)) + secret = {'data':{}} + key_name = '{0}.key'.format(cname) + cert_name = '{0}.crt'.format(cname) + chain_name = '{0}.chain'.format(cname) + sans_name = '{0}.san'.format(cname) token = current_app.config.get('VAULT_TOKEN') mount = self.get_option('vaultMount', options) - path = '{0}/{1}'.format(self.get_option('vaultPath', options),cn) + path = self.get_option('vaultPath', options) url = self.get_option('vaultUrl', options) + bundle = self.get_option('bundleChain', options) + obj_name = self.get_option('objectName', options) client = hvac.Client(url=url, token=token) + if obj_name: + path = '{0}/{1}'.format(path, obj_name) + else: + path = '{0}/{1}'.format(path, cname) - data['cert'] = cert_chain - data['key'] = private_key + secret = get_secret(url, token, mount, path) + - ## upload certificate and key + if bundle == 'Nginx' and cert_chain: + secret['data'][cert_name] = '{0}\n{1}'.format(body, cert_chain) + elif bundle == 'Apache' and cert_chain: + secret['data'][cert_name] = body + secret['data'][chain_name] = cert_chain + else: + secret['data'][cert_name] = body + secret['data'][key_name] = private_key + san_list = get_san_list(body) + if isinstance(san_list, list): + secret['data'][sans_name] = san_list try: - client.secrets.kv.v1.create_or_update_secret(path=path, mount_point=mount, secret=data) - except Exception as err: + client.secrets.kv.v1.create_or_update_secret( + path=path, mount_point=mount, secret=secret['data']) + except ConnectionError as err: current_app.logger.exception( "Exception uploading secret to vault: {0}".format(err), exc_info=True) + +def get_san_list(body): + """ parse certificate for SAN names and return list, return empty list on error """ + try: + byte_body = body.encode('utf-8') + cert = x509.load_pem_x509_certificate(byte_body, default_backend()) + ext = cert.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) + return ext.value.get_values_for_type(x509.DNSName) + except: + pass + return [] + +def get_secret(url, token, mount, path): + result = {'data': {}} + try: + client = hvac.Client(url=url, token=token) + result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) + except: + pass + return result diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e2a3fb9..fd487bd7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in +# pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index # aspy.yaml==1.1.2 # via pre-commit bleach==3.1.0 # via readme-renderer @@ -11,7 +11,7 @@ cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.2.2 # via pre-commit +identify==1.3.0 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit importlib-resources==1.0.2 # via pre-commit @@ -32,6 +32,6 @@ toml==0.10.0 # via pre-commit tqdm==4.31.1 # via twine twine==1.13.0 urllib3==1.24.1 # via requests -virtualenv==16.4.0 # via pre-commit +virtualenv==16.4.1 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index e9dd92cb..8b9c3f2b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in +# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # acme==0.31.0 alabaster==0.7.12 # via sphinx @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.98 -botocore==1.12.98 +boto3==1.9.101 +botocore==1.12.101 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.12.1 @@ -47,13 +47,13 @@ imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 jinja2==2.10 -jmespath==0.9.3 +jmespath==0.9.4 josepy==1.1.0 jsonlines==1.2.0 kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 -markupsafe==1.1.0 +markupsafe==1.1.1 marshmallow-sqlalchemy==0.16.0 marshmallow==2.18.1 mock==2.0.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 1bb8ba03..1c3a4969 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,15 +2,15 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in +# pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.98 # via moto +boto3==1.9.101 # via moto boto==2.49.0 # via moto -botocore==1.12.98 # via boto3, moto, s3transfer +botocore==1.12.101 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.12.1 # via cryptography chardet==3.0.4 # via requests @@ -29,17 +29,17 @@ future==0.17.1 # via python-jose idna==2.8 # via requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto -jmespath==0.9.3 # via boto3, botocore +jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk -markupsafe==1.1.0 # via jinja2 +markupsafe==1.1.1 # via jinja2 mock==2.0.0 # via moto more-itertools==6.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.2 # via mock -pluggy==0.8.1 # via pytest -py==1.7.0 # via pytest +pluggy==0.9.0 # via pytest +py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose @@ -58,7 +58,7 @@ s3transfer==0.2.0 # via boto3 six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests -websocket-client==0.54.0 # via docker +websocket-client==0.55.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements.txt b/requirements.txt index edd56b09..a8615094 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements.txt requirements.in +# pip-compile --output-file requirements.txt requirements.in -U --no-index # acme==0.31.0 alembic-autogenerate-enums==0.0.2 @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.98 -botocore==1.12.98 +boto3==1.9.101 +botocore==1.12.101 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.12.1 # via bcrypt, cryptography, pynacl @@ -44,13 +44,13 @@ idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask jinja2==2.10 -jmespath==0.9.3 # via boto3, botocore +jmespath==0.9.4 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.3.0 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic -markupsafe==1.1.0 # via jinja2, mako +markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.16.0 marshmallow==2.18.1 mock==2.0.0 # via acme From 53301728fa9e052214f7c2f8211a693ff5313ac9 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 26 Feb 2019 09:15:12 -0500 Subject: [PATCH 04/12] Moved url to config file instead of plugin option. One one url can be supported unless both the token and url are moved to the plugin options. --- lemur/plugins/lemur_vault/plugin.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault/plugin.py index 58a9e601..2e46b155 100644 --- a/lemur/plugins/lemur_vault/plugin.py +++ b/lemur/plugins/lemur_vault/plugin.py @@ -47,11 +47,11 @@ class VaultDestinationPlugin(DestinationPlugin): 'helpMessage': 'Must be a valid Vault secrets path' }, { - 'name': 'vaultUrl', + 'name': 'objectName', 'type': 'str', - 'required': True, - 'validation': '^https?://[a-zA-Z0-9.-]+(?::[0-9]+)?$', - 'helpMessage': 'Must be a valid Vault server url' + 'required': False, + 'validation': '[0-9a-zA-Z:_-]+', + 'helpMessage': 'Name to bundle certs under, if blank use cn' }, { 'name': 'bundleChain', @@ -64,13 +64,6 @@ class VaultDestinationPlugin(DestinationPlugin): ], 'required': True, 'helpMessage': 'Bundle the chain into the certificate' - }, - { - 'name': 'objectName', - 'type': 'str', - 'required': False, - 'validation': '[0-9a-zA-Z:_-]+', - 'helpMessage': 'Name to bundle certs under, if blank use cn' } ] @@ -93,10 +86,10 @@ class VaultDestinationPlugin(DestinationPlugin): sans_name = '{0}.san'.format(cname) token = current_app.config.get('VAULT_TOKEN') + url = current_app.config.get('VAULT_URL') mount = self.get_option('vaultMount', options) path = self.get_option('vaultPath', options) - url = self.get_option('vaultUrl', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) From 5d2f603c847771ef0b9bd1651e5c751bc55043f2 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Fri, 1 Mar 2019 09:49:52 -0500 Subject: [PATCH 05/12] renamed vault destination plugin to avoid conflict with vault pki plugin --- lemur/plugins/{lemur_vault => lemur_vault_dest}/__init__.py | 0 lemur/plugins/{lemur_vault => lemur_vault_dest}/plugin.py | 2 +- setup.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename lemur/plugins/{lemur_vault => lemur_vault_dest}/__init__.py (100%) rename lemur/plugins/{lemur_vault => lemur_vault_dest}/plugin.py (98%) diff --git a/lemur/plugins/lemur_vault/__init__.py b/lemur/plugins/lemur_vault_dest/__init__.py similarity index 100% rename from lemur/plugins/lemur_vault/__init__.py rename to lemur/plugins/lemur_vault_dest/__init__.py diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py similarity index 98% rename from lemur/plugins/lemur_vault/plugin.py rename to lemur/plugins/lemur_vault_dest/plugin.py index 2e46b155..a11c92ba 100644 --- a/lemur/plugins/lemur_vault/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -1,5 +1,5 @@ """ -.. module: lemur.plugins.lemur_vault.plugin +.. module: lemur.plugins.lemur_vault_dest.plugin :platform: Unix :copyright: (c) 2019 :license: Apache, see LICENCE for more details. diff --git a/setup.py b/setup.py index b5dcdb3b..d22d1f7b 100644 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ setup( 'digicert_cis_source = lemur.plugins.lemur_digicert.plugin:DigiCertCISSourcePlugin', 'csr_export = lemur.plugins.lemur_csr.plugin:CSRExportPlugin', 'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin', - 'vault_desination = lemur.plugins.lemur_vault.plugin:VaultDestinationPlugin' + 'vault_desination = lemur.plugins.lemur_vault_dest.plugin:VaultDestinationPlugin' ], }, classifiers=[ From 4a027797e057d28900049c16190c4659d5bb48a5 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 5 Mar 2019 07:19:22 -0500 Subject: [PATCH 06/12] fixing linting issues --- lemur/plugins/lemur_vault_dest/plugin.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index a11c92ba..92089b02 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -10,8 +10,6 @@ .. moduleauthor:: Christopher Jolley """ import hvac - -#import lemur_vault from flask import current_app from lemur.common.defaults import common_name @@ -21,7 +19,6 @@ from lemur.plugins.bases import DestinationPlugin from cryptography import x509 from cryptography.hazmat.backends import default_backend - class VaultDestinationPlugin(DestinationPlugin): """Hashicorp Vault Destination plugin for Lemur""" title = 'Vault' @@ -79,7 +76,7 @@ class VaultDestinationPlugin(DestinationPlugin): :return: """ cname = common_name(parse_certificate(body)) - secret = {'data':{}} + secret = {'data': {}} key_name = '{0}.key'.format(cname) cert_name = '{0}.crt'.format(cname) chain_name = '{0}.chain'.format(cname) @@ -100,7 +97,6 @@ class VaultDestinationPlugin(DestinationPlugin): path = '{0}/{1}'.format(path, cname) secret = get_secret(url, token, mount, path) - if bundle == 'Nginx' and cert_chain: secret['data'][cert_name] = '{0}\n{1}'.format(body, cert_chain) @@ -120,6 +116,7 @@ class VaultDestinationPlugin(DestinationPlugin): current_app.logger.exception( "Exception uploading secret to vault: {0}".format(err), exc_info=True) + def get_san_list(body): """ parse certificate for SAN names and return list, return empty list on error """ try: @@ -127,15 +124,16 @@ def get_san_list(body): cert = x509.load_pem_x509_certificate(byte_body, default_backend()) ext = cert.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) return ext.value.get_values_for_type(x509.DNSName) - except: + except ValueError: pass return [] + def get_secret(url, token, mount, path): result = {'data': {}} try: client = hvac.Client(url=url, token=token) result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) - except: + except ConnectionError: pass return result From a1cb8ee266af23aa0ba7171f5cf9d40750b0220b Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 5 Mar 2019 07:37:04 -0500 Subject: [PATCH 07/12] fixing lint --- lemur/plugins/lemur_vault_dest/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 92089b02..774b6bb1 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -19,6 +19,7 @@ from lemur.plugins.bases import DestinationPlugin from cryptography import x509 from cryptography.hazmat.backends import default_backend + class VaultDestinationPlugin(DestinationPlugin): """Hashicorp Vault Destination plugin for Lemur""" title = 'Vault' From b8d3a4f9aac35ee82ed958433742701f965ca190 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 6 Mar 2019 11:13:34 -0800 Subject: [PATCH 08/12] Update requirements.in --- requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index bd408f1c..9b27f604 100644 --- a/requirements.in +++ b/requirements.in @@ -46,4 +46,4 @@ six SQLAlchemy-Utils tabulate xmltodict -pyyaml>=4.2b1 #high severity alert \ No newline at end of file +pyyaml>=4.2b1 #high severity alert From 752c9a086bd1aff975df3622c0b06422368170e4 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Thu, 7 Mar 2019 15:41:29 -0500 Subject: [PATCH 09/12] fixing error handling and better data formating --- lemur/plugins/lemur_vault_dest/plugin.py | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 774b6bb1..5924f387 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -34,7 +34,7 @@ class VaultDestinationPlugin(DestinationPlugin): 'name': 'vaultMount', 'type': 'str', 'required': True, - 'validation': '^[a-zA-Z0-9]+$', + 'validation': '^\S+$', 'helpMessage': 'Must be a valid Vault secrets mount name!' }, { @@ -77,11 +77,6 @@ class VaultDestinationPlugin(DestinationPlugin): :return: """ cname = common_name(parse_certificate(body)) - secret = {'data': {}} - key_name = '{0}.key'.format(cname) - cert_name = '{0}.crt'.format(cname) - chain_name = '{0}.chain'.format(cname) - sans_name = '{0}.san'.format(cname) token = current_app.config.get('VAULT_TOKEN') url = current_app.config.get('VAULT_URL') @@ -98,18 +93,19 @@ class VaultDestinationPlugin(DestinationPlugin): path = '{0}/{1}'.format(path, cname) secret = get_secret(url, token, mount, path) + secret['data'][cname] = {} if bundle == 'Nginx' and cert_chain: - secret['data'][cert_name] = '{0}\n{1}'.format(body, cert_chain) + secret['data'][cname]['crt'] = '{0}\n{1}'.format(body, cert_chain) elif bundle == 'Apache' and cert_chain: - secret['data'][cert_name] = body - secret['data'][chain_name] = cert_chain + secret['data'][cname]['crt'] = body + secret['data'][cname]['chain'] = cert_chain else: - secret['data'][cert_name] = body - secret['data'][key_name] = private_key + secret['data'][cname]['crt'] = body + secret['data'][cname]['key'] = private_key san_list = get_san_list(body) if isinstance(san_list, list): - secret['data'][sans_name] = san_list + secret['data'][cname]['san'] = san_list try: client.secrets.kv.v1.create_or_update_secret( path=path, mount_point=mount, secret=secret['data']) @@ -120,21 +116,25 @@ class VaultDestinationPlugin(DestinationPlugin): def get_san_list(body): """ parse certificate for SAN names and return list, return empty list on error """ + san_list = [] try: byte_body = body.encode('utf-8') cert = x509.load_pem_x509_certificate(byte_body, default_backend()) ext = cert.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) - return ext.value.get_values_for_type(x509.DNSName) - except ValueError: + san_list = ext.value.get_values_for_type(x509.DNSName) + except x509.extensions.ExtensionNotFound: pass - return [] + finally: + return san_list def get_secret(url, token, mount, path): + """ retreiive existing data from mount path and return dictionary """ result = {'data': {}} try: client = hvac.Client(url=url, token=token) result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) - except ConnectionError: - pass - return result + #except ConnectionError: + # pass + finally: + return result From f1c09a6f8f8f7c66d66b0d2c85cfa4420b200d00 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Thu, 7 Mar 2019 15:58:34 -0500 Subject: [PATCH 10/12] fixed comments --- lemur/plugins/lemur_vault_dest/plugin.py | 4 ++-- lemur/plugins/lemur_vault_dest/tests/conftest.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 lemur/plugins/lemur_vault_dest/tests/conftest.py diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 5924f387..2f2a2e82 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -134,7 +134,7 @@ def get_secret(url, token, mount, path): try: client = hvac.Client(url=url, token=token) result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) - #except ConnectionError: - # pass + except ConnectionError: + pass finally: return result diff --git a/lemur/plugins/lemur_vault_dest/tests/conftest.py b/lemur/plugins/lemur_vault_dest/tests/conftest.py new file mode 100644 index 00000000..0e1cd89f --- /dev/null +++ b/lemur/plugins/lemur_vault_dest/tests/conftest.py @@ -0,0 +1 @@ +from lemur.tests.conftest import * # noqa From f99b11d50ec91b5e344eeb1497fd60b96c1af107 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Wed, 20 Mar 2019 13:51:06 -0400 Subject: [PATCH 11/12] refactor url and token to support muiltiple instances of vault --- lemur/plugins/lemur_vault_dest/plugin.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 2f2a2e82..c47b49a3 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -30,6 +30,22 @@ class VaultDestinationPlugin(DestinationPlugin): author_url = 'https://github.com/alwaysjolley/lemur' options = [ + { + 'name': 'vaultUrl', + 'type': 'str', + 'required': True, + 'validation': '^https?://[a-zA-Z0-9.:-]+$', + 'helpMessage': 'Valid URL to Hashi Vault instance' + 'default': 'http://127.0.0.1:8200' + }, + { + 'name': 'vaultAuthTokenFile', + 'type': 'str', + 'required': True, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/etc/pki/secrets/vault/token' + }, { 'name': 'vaultMount', 'type': 'str', @@ -79,13 +95,17 @@ class VaultDestinationPlugin(DestinationPlugin): cname = common_name(parse_certificate(body)) token = current_app.config.get('VAULT_TOKEN') - url = current_app.config.get('VAULT_URL') - + #url = current_app.config.get('VAULT_URL') + url = self.get_option('vaultUrl', options) + token_file = self.get_option('vaultFile', options) mount = self.get_option('vaultMount', options) path = self.get_option('vaultPath', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) + with open(token_file, 'r') as file: + token = file.readline() + client = hvac.Client(url=url, token=token) if obj_name: path = '{0}/{1}'.format(path, obj_name) From fa4a5122bc7c723e5b8e9a8396f05a63577399db Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Wed, 20 Mar 2019 14:59:04 -0400 Subject: [PATCH 12/12] fixing file read to trim line endings and cleanup --- lemur/plugins/lemur_vault_dest/plugin.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index c47b49a3..91f6a07a 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -36,15 +36,13 @@ class VaultDestinationPlugin(DestinationPlugin): 'required': True, 'validation': '^https?://[a-zA-Z0-9.:-]+$', 'helpMessage': 'Valid URL to Hashi Vault instance' - 'default': 'http://127.0.0.1:8200' }, { 'name': 'vaultAuthTokenFile', 'type': 'str', 'required': True, 'validation': '(/[^/]+)+', - 'helpMessage': 'Must be a valid file path!', - 'default': '/etc/pki/secrets/vault/token' + 'helpMessage': 'Must be a valid file path!' }, { 'name': 'vaultMount', @@ -94,17 +92,15 @@ class VaultDestinationPlugin(DestinationPlugin): """ cname = common_name(parse_certificate(body)) - token = current_app.config.get('VAULT_TOKEN') - #url = current_app.config.get('VAULT_URL') url = self.get_option('vaultUrl', options) - token_file = self.get_option('vaultFile', options) + token_file = self.get_option('vaultAuthTokenFile', options) mount = self.get_option('vaultMount', options) path = self.get_option('vaultPath', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) with open(token_file, 'r') as file: - token = file.readline() + token = file.readline().rstrip('\n') client = hvac.Client(url=url, token=token) if obj_name: