Merge branch 'master' into aid_openid_roles_provider_integration

lemur/auth/views.py

@@ -113,7 +113,10 @@ def retrieve_user(user_api_url, access_token):
     user_params = dict(access_token=access_token, schema='profile')
 
     # retrieve information about the current user.
-    r = requests.get(user_api_url, params=user_params)
+    r = requests.get(
+        user_api_url,
+        params=user_params,
+        headers={'Authorization': 'Bearer {}'.format(access_token)})
     profile = r.json()
 
     user = user_service.get_by_email(profile['email'])

lemur/plugins/lemur_acme/cloudflare.py

@@ -66,11 +66,12 @@ def create_txt_record(host, value, account_number):
     return zone_id, r['id']
 
 
-def delete_txt_record(change_id, account_number, host, value):
+def delete_txt_record(change_ids, account_number, host, value):
     cf = cf_api_call()
-    zone_id, record_id = change_id
+    for change_id in change_ids:
-    current_app.logger.debug("Removing record with id {0}".format(record_id))
+        zone_id, record_id = change_id
-    try:
+        current_app.logger.debug("Removing record with id {0}".format(record_id))
-        cf.zones.dns_records.delete(zone_id, record_id)
+        try:
-    except Exception as e:
+            cf.zones.dns_records.delete(zone_id, record_id)
-        current_app.logger.error('/zones.dns_records.post: %s' % e)
+        except Exception as e:
+            current_app.logger.error('/zones.dns_records.post: %s' % e)

lemur/plugins/lemur_vault_dest/plugin.py

@@ -17,11 +17,122 @@ from flask import current_app
 from lemur.common.defaults import common_name
 from lemur.common.utils import parse_certificate
 from lemur.plugins.bases import DestinationPlugin
+from lemur.plugins.bases import SourcePlugin
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 
 
+class VaultSourcePlugin(SourcePlugin):
+    """ Class for importing certificates from Hashicorp Vault"""
+    title = 'Vault'
+    slug = 'vault-source'
+    description = 'Discovers all certificates in a given path'
+
+    author = 'Christopher Jolley'
+    author_url = 'https://github.com/alwaysjolley/lemur'
+
+    options = [
+        {
+            'name': 'vaultUrl',
+            'type': 'str',
+            'required': True,
+            'validation': '^https?://[a-zA-Z0-9.:-]+$',
+            'helpMessage': 'Valid URL to Hashi Vault instance'
+        },
+        {
+            'name': 'vaultKvApiVersion',
+            'type': 'select',
+            'value': '2',
+            'available': [
+                '1',
+                '2'
+            ],
+            'required': True,
+            'helpMessage': 'Version of the Vault KV API to use'
+        },
+        {
+            'name': 'vaultAuthTokenFile',
+            'type': 'str',
+            'required': True,
+            'validation': '(/[^/]+)+',
+            'helpMessage': 'Must be a valid file path!'
+        },
+        {
+            'name': 'vaultMount',
+            'type': 'str',
+            'required': True,
+            'validation': r'^\S+$',
+            'helpMessage': 'Must be a valid Vault secrets mount name!'
+        },
+        {
+            'name': 'vaultPath',
+            'type': 'str',
+            'required': True,
+            'validation': '^([a-zA-Z0-9_-]+/?)+$',
+            'helpMessage': 'Must be a valid Vault secrets path'
+        },
+        {
+            'name': 'objectName',
+            'type': 'str',
+            'required': True,
+            'validation': '[0-9a-zA-Z:_-]+',
+            'helpMessage': 'Object Name to search'
+        },
+    ]
+
+    def get_certificates(self, options, **kwargs):
+        """Pull certificates from objects in Hashicorp Vault"""
+        data = []
+        cert = []
+        body = ''
+        url = self.get_option('vaultUrl', options)
+        token_file = self.get_option('vaultAuthTokenFile', options)
+        mount = self.get_option('vaultMount', options)
+        path = self.get_option('vaultPath', options)
+        obj_name = self.get_option('objectName', options)
+        api_version = self.get_option('vaultKvApiVersion', options)
+        cert_filter = '-----BEGIN CERTIFICATE-----'
+        cert_delimiter = '-----END CERTIFICATE-----'
+
+        with open(token_file, 'r') as tfile:
+            token = tfile.readline().rstrip('\n')
+
+        client = hvac.Client(url=url, token=token)
+        client.secrets.kv.default_kv_version = api_version
+
+        path = '{0}/{1}'.format(path, obj_name)
+
+        secret = get_secret(client, mount, path)
+        for cname in secret['data']:
+            if 'crt' in secret['data'][cname]:
+                cert = secret['data'][cname]['crt'].split(cert_delimiter + '\n')
+            elif 'pem' in secret['data'][cname]:
+                cert = secret['data'][cname]['pem'].split(cert_delimiter + '\n')
+            else:
+                for key in secret['data'][cname]:
+                    if secret['data'][cname][key].startswith(cert_filter):
+                        cert = secret['data'][cname][key].split(cert_delimiter + '\n')
+                        break
+            body = cert[0] + cert_delimiter
+            if 'chain' in secret['data'][cname]:
+                chain = secret['data'][cname]['chain']
+            elif len(cert) > 1:
+                if cert[1].startswith(cert_filter):
+                    chain = cert[1] + cert_delimiter
+                else:
+                    chain = None
+            else:
+                chain = None
+            data.append({'body': body, 'chain': chain, 'name': cname})
+        return [dict(body=c['body'], chain=c.get('chain'), name=c['name']) for c in data]
+
+    def get_endpoints(self, options, **kwargs):
+        """ Not implemented yet """
+        endpoints = []
+        return endpoints
+
+
 class VaultDestinationPlugin(DestinationPlugin):
     """Hashicorp Vault Destination plugin for Lemur"""
     title = 'Vault'
@@ -61,7 +172,7 @@ class VaultDestinationPlugin(DestinationPlugin):
             'name': 'vaultMount',
             'type': 'str',
             'required': True,
-            'validation': '^\S+$',
+            'validation': r'^\S+$',
             'helpMessage': 'Must be a valid Vault secrets mount name!'
         },
         {
@@ -85,6 +196,7 @@ class VaultDestinationPlugin(DestinationPlugin):
             'available': [
                 'Nginx',
                 'Apache',
+                'PEM',
                 'no chain'
             ],
             'required': True,
@@ -136,8 +248,8 @@ class VaultDestinationPlugin(DestinationPlugin):
                         "Exception compiling regex filter: invalid filter",
                         exc_info=True)
 
-        with open(token_file, 'r') as file:
+        with open(token_file, 'r') as tfile:
-            token = file.readline().rstrip('\n')
+            token = tfile.readline().rstrip('\n')
 
         client = hvac.Client(url=url, token=token)
         client.secrets.kv.default_kv_version = api_version
@@ -150,14 +262,18 @@ class VaultDestinationPlugin(DestinationPlugin):
         secret = get_secret(client, mount, path)
         secret['data'][cname] = {}
 
-        if bundle == 'Nginx' and cert_chain:
+        if bundle == 'Nginx':
             secret['data'][cname]['crt'] = '{0}\n{1}'.format(body, cert_chain)
-        elif bundle == 'Apache' and cert_chain:
+            secret['data'][cname]['key'] = private_key
+        elif bundle == 'Apache':
             secret['data'][cname]['crt'] = body
             secret['data'][cname]['chain'] = cert_chain
+            secret['data'][cname]['key'] = private_key
+        elif bundle == 'PEM':
+            secret['data'][cname]['pem'] = '{0}\n{1}\n{2}'.format(body, cert_chain, private_key)
         else:
             secret['data'][cname]['crt'] = body
-        secret['data'][cname]['key'] = private_key
+            secret['data'][cname]['key'] = private_key
         if isinstance(san_list, list):
             secret['data'][cname]['san'] = san_list
         try:
@@ -184,7 +300,7 @@ def get_san_list(body):
 
 
 def get_secret(client, mount, path):
-    """ retreiive existing data from mount path and return dictionary """
+    """ retreive existing data from mount path and return dictionary """
     result = {'data': {}}
     try:
         if client.secrets.kv.default_kv_version == '1':

requirements-dev.txt

@@ -18,7 +18,7 @@ invoke==1.2.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.3
 pkginfo==1.5.0.1          # via twine
-pre-commit==1.15.2
+pre-commit==1.16.0
 pycodestyle==2.3.1        # via flake8
 pyflakes==1.6.0           # via flake8
 pygments==2.3.1           # via readme-renderer
@@ -30,7 +30,7 @@ six==1.12.0               # via bleach, cfgv, pre-commit, readme-renderer
 toml==0.10.0              # via pre-commit
 tqdm==4.31.1              # via twine
 twine==1.13.0
-urllib3==1.24.2           # via requests
+urllib3==1.24.3           # via requests
 virtualenv==16.5.0        # via pre-commit
 webencodings==0.5.1       # via bleach
 zipp==0.4.0               # via importlib-metadata

requirements-docs.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
 #
-acme==0.33.1
+acme==0.34.1
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.10
@@ -17,8 +17,8 @@ babel==2.6.0              # via sphinx
 bcrypt==3.1.6
 billiard==3.6.0.0
 blinker==1.4
-boto3==1.9.138
+boto3==1.9.143
-botocore==1.12.138
+botocore==1.12.143
 celery[redis]==4.3.0
 certifi==2019.3.9
 certsrv==2.1.1
@@ -56,13 +56,12 @@ kombu==4.5.0
 lockfile==0.12.2
 mako==1.0.9
 markupsafe==1.1.1
-marshmallow-sqlalchemy==0.16.2
+marshmallow-sqlalchemy==0.16.3
 marshmallow==2.19.2
-mock==2.0.0
+mock==3.0.4
 ndg-httpsclient==0.5.1
 packaging==19.0           # via sphinx
 paramiko==2.4.2
-pbr==5.2.0
 pem==19.1.0
 psycopg2==2.8.2
 pyasn1-modules==0.2.5
@@ -101,7 +100,7 @@ sqlalchemy-utils==0.33.11
 sqlalchemy==1.3.3
 tabulate==0.8.3
 twofish==0.3.0
-urllib3==1.24.2
+urllib3==1.24.3
 vine==1.3.0
 werkzeug==0.15.2
 xmltodict==0.12.0

requirements-tests.txt

@@ -9,9 +9,9 @@ atomicwrites==1.3.0       # via pytest
 attrs==19.1.0             # via pytest
 aws-sam-translator==1.11.0  # via cfn-lint
 aws-xray-sdk==2.4.2       # via moto
-boto3==1.9.138            # via aws-sam-translator, moto
+boto3==1.9.143            # via aws-sam-translator, moto
 boto==2.49.0              # via moto
-botocore==1.12.138        # via aws-xray-sdk, boto3, moto, s3transfer
+botocore==1.12.143        # via aws-xray-sdk, boto3, moto, s3transfer
 certifi==2019.3.9         # via requests
 cffi==1.12.3              # via cryptography
 cfn-lint==0.19.1          # via moto
@@ -38,11 +38,10 @@ jsonpickle==1.1           # via aws-xray-sdk
 jsonpointer==2.0          # via jsonpatch
 jsonschema==2.6.0         # via aws-sam-translator, cfn-lint
 markupsafe==1.1.1         # via jinja2
-mock==2.0.0               # via moto
+mock==3.0.4               # via moto
 more-itertools==7.0.0     # via pytest
 moto==1.3.8
 nose==1.3.7
-pbr==5.2.0                # via mock
 pluggy==0.9.0             # via pytest
 py==1.8.0                 # via pytest
 pyasn1==0.4.5             # via rsa
@@ -62,7 +61,7 @@ rsa==4.0                  # via python-jose
 s3transfer==0.2.0         # via boto3
 six==1.12.0               # via aws-sam-translator, cfn-lint, cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
 text-unidecode==1.2       # via faker
-urllib3==1.24.2           # via botocore, requests
+urllib3==1.24.3           # via botocore, requests
 websocket-client==0.56.0  # via docker
 werkzeug==0.15.2          # via flask, moto, pytest-flask
 wrapt==1.11.1             # via aws-xray-sdk

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in -U --no-index
 #
-acme==0.33.1
+acme==0.34.1
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.10           # via flask-migrate
 amqp==2.4.2               # via kombu
@@ -15,8 +15,8 @@ asyncpool==1.0
 bcrypt==3.1.6             # via flask-bcrypt, paramiko
 billiard==3.6.0.0         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.9.138
+boto3==1.9.143
-botocore==1.12.138
+botocore==1.12.143
 celery[redis]==4.3.0
 certifi==2019.3.9
 certsrv==2.1.1
@@ -53,12 +53,11 @@ kombu==4.5.0
 lockfile==0.12.2
 mako==1.0.9               # via alembic
 markupsafe==1.1.1         # via jinja2, mako
-marshmallow-sqlalchemy==0.16.2
+marshmallow-sqlalchemy==0.16.3
 marshmallow==2.19.2
-mock==2.0.0               # via acme
+mock==3.0.4               # via acme
 ndg-httpsclient==0.5.1
 paramiko==2.4.2
-pbr==5.2.0                # via mock
 pem==19.1.0
 psycopg2==2.8.2
 pyasn1-modules==0.2.5     # via pyjks, python-ldap
@@ -86,7 +85,7 @@ sqlalchemy-utils==0.33.11
 sqlalchemy==1.3.3         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.3
 twofish==0.3.0            # via pyjks
-urllib3==1.24.2           # via botocore, requests
+urllib3==1.24.3           # via botocore, requests
 vine==1.3.0               # via amqp, celery
 werkzeug==0.15.2          # via flask
 xmltodict==0.12.0

setup.py

@@ -155,6 +155,7 @@ setup(
             'digicert_cis_source = lemur.plugins.lemur_digicert.plugin:DigiCertCISSourcePlugin',
             'csr_export = lemur.plugins.lemur_csr.plugin:CSRExportPlugin',
             'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin',
+            'vault_source = lemur.plugins.lemur_vault_dest.plugin:VaultSourcePlugin',
             'vault_desination = lemur.plugins.lemur_vault_dest.plugin:VaultDestinationPlugin',
             'adcs_issuer = lemur.plugins.lemur_adcs.plugin:ADCSIssuerPlugin',
             'adcs_source = lemur.plugins.lemur_adcs.plugin:ADCSSourcePlugin'