From 2063baefc911ccff0c9d3a09b671eadcda45744d Mon Sep 17 00:00:00 2001 From: Jose Plana Date: Wed, 1 May 2019 00:47:56 +0200 Subject: [PATCH 1/5] Fixes userinfo using Bearer token --- lemur/auth/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lemur/auth/views.py b/lemur/auth/views.py index 7a1bb34c..6dad88d2 100644 --- a/lemur/auth/views.py +++ b/lemur/auth/views.py @@ -113,7 +113,10 @@ def retrieve_user(user_api_url, access_token): user_params = dict(access_token=access_token, schema='profile') # retrieve information about the current user. - r = requests.get(user_api_url, params=user_params) + r = requests.get( + user_api_url, + params=user_params, + headers={'Authorization': 'Bearer {}'.format(access_token)}) profile = r.json() user = user_service.get_by_email(profile['email']) From 6d5552afd38a82ecf0345730ccb770b48803106f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 6 May 2019 16:31:50 -0700 Subject: [PATCH 2/5] updating requirements --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 13 ++++++------- requirements-tests.txt | 9 ++++----- requirements.txt | 13 ++++++------- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0652df34..29509d99 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.15.2 +pre-commit==1.16.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer @@ -30,7 +30,7 @@ six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.31.1 # via twine twine==1.13.0 -urllib3==1.24.2 # via requests +urllib3==1.24.3 # via requests virtualenv==16.5.0 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.4.0 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index 4b75a502..fef37c08 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # -acme==0.33.1 +acme==0.34.1 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.10 @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.6.0.0 blinker==1.4 -boto3==1.9.138 -botocore==1.12.138 +boto3==1.9.143 +botocore==1.12.143 celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 @@ -56,13 +56,12 @@ kombu==4.5.0 lockfile==0.12.2 mako==1.0.9 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.16.2 +marshmallow-sqlalchemy==0.16.3 marshmallow==2.19.2 -mock==2.0.0 +mock==3.0.4 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 -pbr==5.2.0 pem==19.1.0 psycopg2==2.8.2 pyasn1-modules==0.2.5 @@ -101,7 +100,7 @@ sqlalchemy-utils==0.33.11 sqlalchemy==1.3.3 tabulate==0.8.3 twofish==0.3.0 -urllib3==1.24.2 +urllib3==1.24.3 vine==1.3.0 werkzeug==0.15.2 xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 0a4660d0..5d28412c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -9,9 +9,9 @@ atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-sam-translator==1.11.0 # via cfn-lint aws-xray-sdk==2.4.2 # via moto -boto3==1.9.138 # via aws-sam-translator, moto +boto3==1.9.143 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.138 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.12.143 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2019.3.9 # via requests cffi==1.12.3 # via cryptography cfn-lint==0.19.1 # via moto @@ -38,11 +38,10 @@ jsonpickle==1.1 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==2.6.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 -mock==2.0.0 # via moto +mock==3.0.4 # via moto more-itertools==7.0.0 # via pytest moto==1.3.8 nose==1.3.7 -pbr==5.2.0 # via mock pluggy==0.9.0 # via pytest py==1.8.0 # via pytest pyasn1==0.4.5 # via rsa @@ -62,7 +61,7 @@ rsa==4.0 # via python-jose s3transfer==0.2.0 # via boto3 six==1.12.0 # via aws-sam-translator, cfn-lint, cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.24.2 # via botocore, requests +urllib3==1.24.3 # via botocore, requests websocket-client==0.56.0 # via docker werkzeug==0.15.2 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk diff --git a/requirements.txt b/requirements.txt index 74290471..fe27838b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file requirements.txt requirements.in -U --no-index # -acme==0.33.1 +acme==0.34.1 alembic-autogenerate-enums==0.0.2 alembic==1.0.10 # via flask-migrate amqp==2.4.2 # via kombu @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.6.0.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.138 -botocore==1.12.138 +boto3==1.9.143 +botocore==1.12.143 celery[redis]==4.3.0 certifi==2019.3.9 certsrv==2.1.1 @@ -53,12 +53,11 @@ kombu==4.5.0 lockfile==0.12.2 mako==1.0.9 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.16.2 +marshmallow-sqlalchemy==0.16.3 marshmallow==2.19.2 -mock==2.0.0 # via acme +mock==3.0.4 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.2.0 # via mock pem==19.1.0 psycopg2==2.8.2 pyasn1-modules==0.2.5 # via pyjks, python-ldap @@ -86,7 +85,7 @@ sqlalchemy-utils==0.33.11 sqlalchemy==1.3.3 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 twofish==0.3.0 # via pyjks -urllib3==1.24.2 # via botocore, requests +urllib3==1.24.3 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==0.15.2 # via flask xmltodict==0.12.0 From a7af3cf8d279c00b74eac1b094653eab95a2a6e7 Mon Sep 17 00:00:00 2001 From: Daniel Iancu Date: Tue, 7 May 2019 02:39:49 +0300 Subject: [PATCH 3/5] Fix Cloudflare DNS --- lemur/plugins/lemur_acme/cloudflare.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_acme/cloudflare.py b/lemur/plugins/lemur_acme/cloudflare.py index 77052242..a6308025 100644 --- a/lemur/plugins/lemur_acme/cloudflare.py +++ b/lemur/plugins/lemur_acme/cloudflare.py @@ -66,11 +66,12 @@ def create_txt_record(host, value, account_number): return zone_id, r['id'] -def delete_txt_record(change_id, account_number, host, value): +def delete_txt_record(change_ids, account_number, host, value): cf = cf_api_call() - zone_id, record_id = change_id - current_app.logger.debug("Removing record with id {0}".format(record_id)) - try: - cf.zones.dns_records.delete(zone_id, record_id) - except Exception as e: - current_app.logger.error('/zones.dns_records.post: %s' % e) + for change_id in change_ids: + zone_id, record_id = change_id + current_app.logger.debug("Removing record with id {0}".format(record_id)) + try: + cf.zones.dns_records.delete(zone_id, record_id) + except Exception as e: + current_app.logger.error('/zones.dns_records.post: %s' % e) From fb3f0bd72a8f868284f7efb5805a2d132c4706ad Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 7 May 2019 09:37:30 -0400 Subject: [PATCH 4/5] adding Vault Source plugin --- lemur/plugins/lemur_vault_dest/plugin.py | 133 +++++++++++++++++++++-- setup.py | 1 + 2 files changed, 127 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 819ba22b..0ed035d0 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -17,11 +17,124 @@ from flask import current_app from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate from lemur.plugins.bases import DestinationPlugin +from lemur.plugins.bases import SourcePlugin from cryptography import x509 from cryptography.hazmat.backends import default_backend +class VaultSourcePlugin(SourcePlugin): + """ Class for importing certificates from Hashicorp Vault""" + title = 'Vault' + slug = 'vault-source' + description = 'Discovers all certificates in a given path' + + author = 'Christopher Jolley' + author_url = 'https://github.com/alwaysjolley/lemur' + + options = [ + { + 'name': 'vaultUrl', + 'type': 'str', + 'required': True, + 'validation': '^https?://[a-zA-Z0-9.:-]+$', + 'helpMessage': 'Valid URL to Hashi Vault instance' + }, + { + 'name': 'vaultKvApiVersion', + 'type': 'select', + 'value': '2', + 'available': [ + '1', + '2' + ], + 'required': True, + 'helpMessage': 'Version of the Vault KV API to use' + }, + { + 'name': 'vaultAuthTokenFile', + 'type': 'str', + 'required': True, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!' + }, + { + 'name': 'vaultMount', + 'type': 'str', + 'required': True, + 'validation': r'^\S+$', + 'helpMessage': 'Must be a valid Vault secrets mount name!' + }, + { + 'name': 'vaultPath', + 'type': 'str', + 'required': True, + 'validation': '^([a-zA-Z0-9_-]+/?)+$', + 'helpMessage': 'Must be a valid Vault secrets path' + }, + { + 'name': 'objectName', + 'type': 'str', + 'required': True, + 'validation': '[0-9a-zA-Z:_-]+', + 'helpMessage': 'Object Name to search' + }, + ] + + + def get_certificates(self, options, **kwargs): + """Pull certificates from objects in Hashicorp Vault""" + data = [] + cert = [] + body = '' + url = self.get_option('vaultUrl', options) + token_file = self.get_option('vaultAuthTokenFile', options) + mount = self.get_option('vaultMount', options) + path = self.get_option('vaultPath', options) + obj_name = self.get_option('objectName', options) + api_version = self.get_option('vaultKvApiVersion', options) + cert_filter = '-----BEGIN CERTIFICATE-----' + cert_delimiter = '-----END CERTIFICATE-----' + + with open(token_file, 'r') as tfile: + token = tfile.readline().rstrip('\n') + + client = hvac.Client(url=url, token=token) + client.secrets.kv.default_kv_version = api_version + + path = '{0}/{1}'.format(path, obj_name) + + secret = get_secret(client, mount, path) + for cname in secret['data']: + #current_app.logger.info("Certificate Data: {0}".format(secret['data'][cname])) + if 'crt' in secret['data'][cname]: + cert = secret['data'][cname]['crt'].split(cert_delimiter+'\n') + elif 'pem' in secret['data'][cname]: + cert = secret['data'][cname]['pem'].split(cert_delimiter+'\n') + else: + for key in secret['data'][cname]: + if secret['data'][cname][key].startswith(cert_filter): + cert = secret['data'][cname][key].split(cert_delimiter+'\n') + break + body = cert[0]+cert_delimiter + if 'chain' in secret['data'][cname]: + chain = secret['data'][cname]['chain'] + elif len(cert) > 1: + if cert[1].startswith(cert_filter): + chain = cert[1]+cert_delimiter + else: + chain = None + else: + chain = None + data.append({'body': body, 'chain': chain, 'name': cname}) + return [dict(body=c['body'], chain=c.get('chain'), name=c['name']) for c in data] + + def get_endpoints(self, options, **kwargs): + """ Not implemented yet """ + endpoints = [] + return endpoints + + class VaultDestinationPlugin(DestinationPlugin): """Hashicorp Vault Destination plugin for Lemur""" title = 'Vault' @@ -61,7 +174,7 @@ class VaultDestinationPlugin(DestinationPlugin): 'name': 'vaultMount', 'type': 'str', 'required': True, - 'validation': '^\S+$', + 'validation': r'^\S+$', 'helpMessage': 'Must be a valid Vault secrets mount name!' }, { @@ -85,6 +198,7 @@ class VaultDestinationPlugin(DestinationPlugin): 'available': [ 'Nginx', 'Apache', + 'PEM', 'no chain' ], 'required': True, @@ -100,6 +214,7 @@ class VaultDestinationPlugin(DestinationPlugin): } ] + def __init__(self, *args, **kwargs): super(VaultDestinationPlugin, self).__init__(*args, **kwargs) @@ -136,8 +251,8 @@ class VaultDestinationPlugin(DestinationPlugin): "Exception compiling regex filter: invalid filter", exc_info=True) - with open(token_file, 'r') as file: - token = file.readline().rstrip('\n') + with open(token_file, 'r') as tfile: + token = tfile.readline().rstrip('\n') client = hvac.Client(url=url, token=token) client.secrets.kv.default_kv_version = api_version @@ -150,14 +265,18 @@ class VaultDestinationPlugin(DestinationPlugin): secret = get_secret(client, mount, path) secret['data'][cname] = {} - if bundle == 'Nginx' and cert_chain: + if bundle == 'Nginx': secret['data'][cname]['crt'] = '{0}\n{1}'.format(body, cert_chain) - elif bundle == 'Apache' and cert_chain: + secret['data'][cname]['key'] = private_key + elif bundle == 'Apache': secret['data'][cname]['crt'] = body secret['data'][cname]['chain'] = cert_chain + secret['data'][cname]['key'] = private_key + elif bundle == 'PEM': + secret['data'][cname]['pem'] = '{0}\n{1}\n{2}'.format(body, cert_chain, private_key) else: secret['data'][cname]['crt'] = body - secret['data'][cname]['key'] = private_key + secret['data'][cname]['key'] = private_key if isinstance(san_list, list): secret['data'][cname]['san'] = san_list try: @@ -184,7 +303,7 @@ def get_san_list(body): def get_secret(client, mount, path): - """ retreiive existing data from mount path and return dictionary """ + """ retreive existing data from mount path and return dictionary """ result = {'data': {}} try: if client.secrets.kv.default_kv_version == '1': diff --git a/setup.py b/setup.py index 6fc55420..a01c110f 100644 --- a/setup.py +++ b/setup.py @@ -155,6 +155,7 @@ setup( 'digicert_cis_source = lemur.plugins.lemur_digicert.plugin:DigiCertCISSourcePlugin', 'csr_export = lemur.plugins.lemur_csr.plugin:CSRExportPlugin', 'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin', + 'vault_source = lemur.plugins.lemur_vault_dest.plugin:VaultSourcePlugin', 'vault_desination = lemur.plugins.lemur_vault_dest.plugin:VaultDestinationPlugin', 'adcs_issuer = lemur.plugins.lemur_adcs.plugin:ADCSIssuerPlugin', 'adcs_source = lemur.plugins.lemur_adcs.plugin:ADCSSourcePlugin' From b0c8901b0ac9cc80c0ac8927eb21e50d0e6bf0fe Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 7 May 2019 10:05:01 -0400 Subject: [PATCH 5/5] lint cleanup --- lemur/plugins/lemur_vault_dest/plugin.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 0ed035d0..803b0a0c 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -81,7 +81,6 @@ class VaultSourcePlugin(SourcePlugin): }, ] - def get_certificates(self, options, **kwargs): """Pull certificates from objects in Hashicorp Vault""" data = [] @@ -106,22 +105,21 @@ class VaultSourcePlugin(SourcePlugin): secret = get_secret(client, mount, path) for cname in secret['data']: - #current_app.logger.info("Certificate Data: {0}".format(secret['data'][cname])) if 'crt' in secret['data'][cname]: - cert = secret['data'][cname]['crt'].split(cert_delimiter+'\n') + cert = secret['data'][cname]['crt'].split(cert_delimiter + '\n') elif 'pem' in secret['data'][cname]: - cert = secret['data'][cname]['pem'].split(cert_delimiter+'\n') + cert = secret['data'][cname]['pem'].split(cert_delimiter + '\n') else: for key in secret['data'][cname]: if secret['data'][cname][key].startswith(cert_filter): - cert = secret['data'][cname][key].split(cert_delimiter+'\n') + cert = secret['data'][cname][key].split(cert_delimiter + '\n') break - body = cert[0]+cert_delimiter + body = cert[0] + cert_delimiter if 'chain' in secret['data'][cname]: chain = secret['data'][cname]['chain'] elif len(cert) > 1: if cert[1].startswith(cert_filter): - chain = cert[1]+cert_delimiter + chain = cert[1] + cert_delimiter else: chain = None else: @@ -214,7 +212,6 @@ class VaultDestinationPlugin(DestinationPlugin): } ] - def __init__(self, *args, **kwargs): super(VaultDestinationPlugin, self).__init__(*args, **kwargs)