From 48017a9d4c66344860bbf007bccc435cc02f9fd5 Mon Sep 17 00:00:00 2001 From: Non Sequitur Date: Wed, 17 Oct 2018 11:42:09 -0400 Subject: [PATCH 01/43] Added get_by_attributes to the certificates service, for fetching certs based on arbitrary attributes. Also associated test and extra tests for other service methods --- lemur/certificates/service.py | 18 ++++++- lemur/sources/service.py | 7 ++- lemur/tests/test_certificates.py | 83 ++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 0bd50694..c8a5365b 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -54,7 +54,7 @@ def get_by_name(name): def get_by_serial(serial): """ - Retrieves certificate by it's Serial. + Retrieves certificate(s) by serial number. :param serial: :return: """ @@ -64,6 +64,22 @@ def get_by_serial(serial): return Certificate.query.filter(Certificate.serial == serial).all() +def get_by_attributes(conditions): + """ + Retrieves certificate(s) by conditions given in a hash of given key=>value pairs. + :param serial: + :return: + """ + # Ensure that each of the given conditions corresponds to actual columns + # if not, silently remove it + for attr in conditions.keys(): + if attr not in Certificate.__table__.columns: + conditions.pop(attr) + + query = database.session_query(Certificate) + return database.find_all(query, Certificate, conditions).all() + + def delete(cert_id): """ Delete's a certificate. diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 227f1bce..5002041c 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -116,7 +116,12 @@ def sync_certificates(source, user): for certificate in certificates: exists = False - if certificate.get('name'): + + if certificate.get('search', None): + conditions = certificate.pop('search') + exists = certificate_service.get_by_attributes(conditions) + + if not exists and certificate.get('name'): result = certificate_service.get_by_name(certificate['name']) if result: exists = [result] diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 1a4d644b..0f46e4a5 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -41,6 +41,89 @@ def test_get_or_increase_name(session, certificate): assert get_or_increase_name('certificate1', int(serial, 16)) == 'certificate1-{}-1'.format(serial) +def test_get_all_certs(session, certificate): + from lemur.certificates.service import get_all_certs + assert len(get_all_certs()) > 1 + + +def test_get_by_name(session, certificate): + from lemur.certificates.service import get_by_name + + found = get_by_name(certificate.name) + + assert found + + +def test_get_by_serial(session, certificate): + from lemur.certificates.service import get_by_serial + + found = get_by_serial(certificate.serial) + + assert found + + +def test_delete_cert(session): + from lemur.certificates.service import delete, get + from lemur.tests.factories import CertificateFactory + + delete_this = CertificateFactory(name='DELETEME') + session.commit() + + cert_exists = get(delete_this.id) + + # it needs to exist first + assert cert_exists + + delete(delete_this.id) + cert_exists = get(delete_this.id) + + # then not exist after delete + assert not cert_exists + + +def test_get_by_attributes(session, certificate): + from lemur.certificates.service import get_by_attributes + + # Should get one cert + certificate1 = get_by_attributes({ + 'name': 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231' + }) + + # Should get one cert using multiple attrs + certificate2 = get_by_attributes({ + 'name': 'test-cert-11111111-1', + 'cn': 'san.example.org' + }) + + # Should get multiple certs + multiple = get_by_attributes({ + 'cn': 'LemurTrust Unittests Class 1 CA 2018', + 'issuer': 'LemurTrustUnittestsRootCA2018' + }) + + assert len(certificate1) == 1 + assert len(certificate2) == 1 + assert len(multiple) > 1 + + +def test_find_duplicates(session): + from lemur.certificates.service import find_duplicates + + cert = { + 'body': SAN_CERT_STR, + 'chain': INTERMEDIATE_CERT_STR + } + + dups1 = find_duplicates(cert) + + cert['chain'] = '' + + dups2 = find_duplicates(cert) + + assert len(dups1) > 0 + assert len(dups2) > 0 + + def test_get_certificate_primitives(certificate): from lemur.certificates.service import get_certificate_primitives From 51248c193803727c10e9d4c67de8e465210ee3f2 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Thu, 20 Dec 2018 18:13:59 +0200 Subject: [PATCH 02/43] Use special issuer values and in special cases This way it's easy to find/distinguish selfsigned certificates stored in Lemur. --- lemur/common/defaults.py | 13 +++++++++++-- lemur/common/utils.py | 37 +++++++++++++++++++++++++++++++++++- lemur/tests/conftest.py | 8 ++++++++ lemur/tests/test_defaults.py | 14 +++++++++++++- lemur/tests/test_utils.py | 12 ++++++++++++ lemur/tests/vectors.py | 1 + 6 files changed, 81 insertions(+), 4 deletions(-) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index 72e863c1..6b259f6b 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -3,6 +3,8 @@ import unicodedata from cryptography import x509 from flask import current_app + +from lemur.common.utils import is_selfsigned from lemur.extensions import sentry from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE @@ -229,15 +231,22 @@ def issuer(cert): """ Gets a sane issuer slug from a given certificate, stripping non-alphanumeric characters. - :param cert: + For self-signed certificates, the special value '' is returned. + If issuer cannot be determined, '' is returned. + + :param cert: Parsed certificate object :return: Issuer slug """ + # If certificate is self-signed, we return a special value -- there really is no distinct "issuer" for it + if is_selfsigned(cert): + return '' + # Try Common Name or fall back to Organization name attrs = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)) if not attrs: current_app.logger.error("Unable to get issuer! Cert serial {:x}".format(cert.serial_number)) - return "Unknown" + return '' return text_to_slug(attrs[0].value, '') diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 32271e89..f3ac5fe7 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -11,9 +11,10 @@ import string import sqlalchemy from cryptography import x509 +from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import rsa, ec +from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask_restful.reqparse import RequestParser from sqlalchemy import and_, func @@ -143,6 +144,40 @@ def generate_private_key(key_type): ) +def check_cert_signature(cert, issuer_public_key): + """ + Check a certificate's signature against an issuer public key. + On success, returns None; on failure, raises UnsupportedAlgorithm or InvalidSignature. + """ + if isinstance(issuer_public_key, rsa.RSAPublicKey): + # RSA requires padding, just to make life difficult for us poor developers :( + if cert.signature_algorithm_oid == x509.SignatureAlgorithmOID.RSASSA_PSS: + # In 2005, IETF devised a more secure padding scheme to replace PKCS #1 v1.5. To make sure that + # nobody can easily support or use it, they mandated lots of complicated parameters, unlike any + # other X.509 signature scheme. + # https://tools.ietf.org/html/rfc4056 + raise UnsupportedAlgorithm("RSASSA-PSS not supported") + else: + padder = padding.PKCS1v15() + issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, padder, cert.signature_hash_algorithm) + else: + # EllipticCurvePublicKey or DSAPublicKey + issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, cert.signature_hash_algorithm) + + +def is_selfsigned(cert): + """ + Returns True if the certificate is self-signed. + Returns False for failed verification or unsupported signing algorithm. + """ + try: + check_cert_signature(cert, cert.public_key()) + # If verification was successful, it's self-signed. + return True + except InvalidSignature: + return False + + def is_weekend(date): """ Determines if a given date is on a weekend. diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index 32733e51..b3dad8b2 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -3,6 +3,8 @@ import os import datetime import pytest from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes from flask import current_app from flask_principal import identity_changed, Identity @@ -263,6 +265,12 @@ def cert_builder(private_key): .not_valid_after(datetime.datetime(2040, 1, 1))) +@pytest.fixture +def selfsigned_cert(cert_builder, private_key): + # cert_builder uses the same cert public key as 'private_key' + return cert_builder.sign(private_key, hashes.SHA256(), default_backend()) + + @pytest.fixture(scope='function') def aws_credentials(): os.environ['AWS_ACCESS_KEY_ID'] = 'testing' diff --git a/lemur/tests/test_defaults.py b/lemur/tests/test_defaults.py index ffa19727..da9d6c79 100644 --- a/lemur/tests/test_defaults.py +++ b/lemur/tests/test_defaults.py @@ -81,6 +81,13 @@ def test_create_name(client): datetime(2015, 5, 12, 0, 0, 0), False ) == 'xn--mnchen-3ya.de-VertrauenswurdigAutoritat-20150507-20150512' + assert certificate_name( + 'selfie.example.org', + '', + datetime(2015, 5, 7, 0, 0, 0), + datetime(2025, 5, 12, 13, 37, 0), + False + ) == 'selfie.example.org-selfsigned-20150507-20250512' def test_issuer(client, cert_builder, issuer_private_key): @@ -106,4 +113,9 @@ def test_issuer(client, cert_builder, issuer_private_key): cert = (cert_builder .issuer_name(x509.Name([])) .sign(issuer_private_key, hashes.SHA256(), default_backend())) - assert issuer(cert) == 'Unknown' + assert issuer(cert) == '' + + +def test_issuer_selfsigned(selfsigned_cert): + from lemur.common.defaults import issuer + assert issuer(selfsigned_cert) == '' diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 62d021a4..3e226f0f 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -1,5 +1,7 @@ import pytest +from lemur.tests.vectors import SAN_CERT, INTERMEDIATE_CERT, ROOTCA_CERT + def test_generate_private_key(): from lemur.common.utils import generate_private_key @@ -71,3 +73,13 @@ KFfxwrO1 -----END CERTIFICATE-----''' authority_key = get_authority_key(test_cert) assert authority_key == 'feacb541be81771293affa412d8dc9f66a3ebb80' + + +def test_is_selfsigned(selfsigned_cert): + from lemur.common.utils import is_selfsigned + + assert is_selfsigned(selfsigned_cert) is True + assert is_selfsigned(SAN_CERT) is False + assert is_selfsigned(INTERMEDIATE_CERT) is False + # Root CA certificates are also technically self-signed + assert is_selfsigned(ROOTCA_CERT) is True diff --git a/lemur/tests/vectors.py b/lemur/tests/vectors.py index 6a836b30..5da37c61 100644 --- a/lemur/tests/vectors.py +++ b/lemur/tests/vectors.py @@ -45,6 +45,7 @@ ssvobJ6Xe2D4cCVjUmsqtFEztMgdqgmlcWyGdUKeXdi7CMoeTb4uO+9qRQq46wYW n7K1z+W0Kp5yhnnPAoOioAP4vjASDx3z3RnLaZvMmcO7YdCIwhE5oGV0 -----END CERTIFICATE----- """ +ROOTCA_CERT = parse_certificate(ROOTCA_CERT_STR) ROOTCA_KEY = """\ -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAvyVpe0tfIzri3l3PYH2r7hW86wKF58GLY+Ua52rEO5E3eXQq From 70a70663a2f766523bed0a688456aac99d265919 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 Feb 2019 09:51:34 -0800 Subject: [PATCH 03/43] updating requirements --- requirements-dev.txt | 5 ++--- requirements-docs.txt | 22 +++++++++++----------- requirements-tests.txt | 16 ++++++++-------- requirements.txt | 20 ++++++++++---------- 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ac35f3e9..440f932b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,18 +14,17 @@ flake8==3.5.0 identify==1.2.1 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit -importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.2 +pre-commit==1.14.3 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine -requests-toolbelt==0.9.0 # via twine +requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 15085766..194708ed 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -8,7 +8,7 @@ acme==0.30.2 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.7 -amqp==2.4.0 +amqp==2.4.1 aniso8601==4.1.0 arrow==0.13.0 asn1crypto==0.24.0 @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.86 -botocore==1.12.86 +boto3==1.9.89 +botocore==1.12.89 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 @@ -49,17 +49,17 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.2.post1 +kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 -marshmallow-sqlalchemy==0.15.0 +marshmallow-sqlalchemy==0.16.0 marshmallow==2.18.0 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 -pbr==5.1.1 +pbr==5.1.2 pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 @@ -71,20 +71,20 @@ pynacl==1.3.0 pyopenssl==19.0.0 pyparsing==2.3.1 # via packaging pyrfc3339==1.1 -python-dateutil==2.7.5 -python-editor==1.0.3 +python-dateutil==2.8.0 +python-editor==1.0.4 pytz==2018.9 pyyaml==3.13 raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.9.0 +requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 -s3transfer==0.1.13 +s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.2 -sphinx==1.8.3 +sphinx==1.8.4 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 diff --git a/requirements-tests.txt b/requirements-tests.txt index c326e951..174e60ff 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,12 +5,12 @@ # pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography -atomicwrites==1.2.1 # via pytest +atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.86 # via moto +boto3==1.9.89 # via moto boto==2.49.0 # via moto -botocore==1.12.86 # via boto3, moto, s3transfer +botocore==1.12.89 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -37,7 +37,7 @@ mock==2.0.0 # via moto more-itertools==5.0.0 # via pytest moto==1.3.7 nose==1.3.7 -pbr==5.1.1 # via mock +pbr==5.1.2 # via mock pluggy==0.8.1 # via pytest py==1.7.0 # via pytest pyaml==18.11.0 # via moto @@ -45,16 +45,16 @@ pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose pyflakes==2.1.0 pytest-flask==0.14.0 -pytest-mock==1.10.0 -pytest==4.1.1 -python-dateutil==2.7.5 # via botocore, faker, freezegun, moto +pytest-mock==1.10.1 +pytest==4.2.0 +python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.10.5 # via moto -s3transfer==0.1.13 # via boto3 +s3transfer==0.2.0 # via boto3 six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests diff --git a/requirements.txt b/requirements.txt index c595e509..db661030 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ acme==0.30.2 alembic-autogenerate-enums==0.0.2 alembic==1.0.7 # via flask-migrate -amqp==2.4.0 # via kombu +amqp==2.4.1 # via kombu aniso8601==4.1.0 # via flask-restful arrow==0.13.0 asn1crypto==0.24.0 # via cryptography @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.86 -botocore==1.12.86 +boto3==1.9.89 +botocore==1.12.89 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl @@ -46,16 +46,16 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.2.post1 # via celery +kombu==4.3.0 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako -marshmallow-sqlalchemy==0.15.0 +marshmallow-sqlalchemy==0.16.0 marshmallow==2.18.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.1.1 # via mock +pbr==5.1.2 # via mock pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 # via python-ldap @@ -65,17 +65,17 @@ pyjwt==1.7.1 pynacl==1.3.0 # via paramiko pyopenssl==19.0.0 pyrfc3339==1.1 # via acme -python-dateutil==2.7.5 # via alembic, arrow, botocore -python-editor==1.0.3 # via alembic +python-dateutil==2.8.0 # via alembic, arrow, botocore +python-editor==1.0.4 # via alembic python-ldap==3.1.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.9.0 # via acme +requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 -s3transfer==0.1.13 # via boto3 +s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils From a43c6cf954bcaff127f9703be3c91bc594968ca2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 Feb 2019 09:57:42 -0800 Subject: [PATCH 04/43] Update requirements-docs.txt --- requirements-docs.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index e68bfc5e..194708ed 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,6 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 - boto3==1.9.89 botocore==1.12.89 celery[redis]==4.2.1 From fd60b163423167aaf51b6a770f058d1002c006fb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 Feb 2019 17:12:37 -0800 Subject: [PATCH 05/43] updating requirements, pinning pyyaml to patched version. --- requirements-docs.txt | 4 ++-- requirements.in | 2 ++ requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 194708ed..4ebea0a0 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.30.2 +acme==0.31.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.7 @@ -74,7 +74,7 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 pytz==2018.9 -pyyaml==3.13 +pyyaml==4.2b4 raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.9.1 diff --git a/requirements.in b/requirements.in index 0aea4591..b085f5c7 100644 --- a/requirements.in +++ b/requirements.in @@ -44,3 +44,5 @@ six SQLAlchemy-Utils tabulate xmltodict +pyyaml>=4.2b1 #high severity alert + diff --git a/requirements.txt b/requirements.txt index f391d016..fd164c3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.30.2 +acme==0.31.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.7 # via flask-migrate amqp==2.4.1 # via kombu @@ -70,7 +70,7 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-ldap==3.1.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==3.13 # via cloudflare +pyyaml==4.2b4 raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.9.1 # via acme From 73a474bd352b80a21751738506c30f3706ffc59c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 8 Feb 2019 08:23:42 -0800 Subject: [PATCH 06/43] pinning pyyaml to ensure only using the patched version --- requirements-dev.in | 3 ++- requirements-dev.txt | 2 +- requirements-docs.txt | 5 +++-- requirements-tests.in | 1 + requirements-tests.txt | 6 +++--- requirements.txt | 6 +++--- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index 84104679..2ffc5488 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -4,4 +4,5 @@ flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" pre-commit invoke twine -nodeenv \ No newline at end of file +nodeenv +pyyaml>=4.2b1 \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 440f932b..fd491663 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,7 +22,7 @@ pre-commit==1.14.3 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer -pyyaml==3.13 # via aspy.yaml, pre-commit +pyyaml==4.2b4 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine diff --git a/requirements-docs.txt b/requirements-docs.txt index 4ebea0a0..a6c05582 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,10 +17,11 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.89 -botocore==1.12.89 +boto3==1.9.90 +botocore==1.12.90 celery[redis]==4.2.1 certifi==2018.11.29 +certsrv==2.1.1 cffi==1.11.5 chardet==3.0.4 click==7.0 diff --git a/requirements-tests.in b/requirements-tests.in index 02a2b0ae..dcd3d0c7 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -11,3 +11,4 @@ pytest pytest-flask pytest-mock requests-mock +pyyaml>=4.2b1 \ No newline at end of file diff --git a/requirements-tests.txt b/requirements-tests.txt index 174e60ff..e4a34412 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.89 # via moto +boto3==1.9.90 # via moto boto==2.49.0 # via moto -botocore==1.12.89 # via boto3, moto, s3transfer +botocore==1.12.90 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -50,7 +50,7 @@ pytest==4.2.0 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto -pyyaml==3.13 # via pyaml +pyyaml==4.2b4 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.10.5 # via moto diff --git a/requirements.txt b/requirements.txt index fd164c3d..f24d274e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,11 +15,11 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.89 -botocore==1.12.89 +boto3==1.9.90 +botocore==1.12.90 celery[redis]==4.2.1 certifi==2018.11.29 -certsrv==2.1.0 +certsrv==2.1.1 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask From 42af082d3a5a4095300f961533665991947fbdae Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 11 Feb 2019 10:22:54 -0800 Subject: [PATCH 07/43] updating requirements --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 6 +++--- requirements-tests.txt | 6 +++--- requirements.txt | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index fd491663..f5d6be3c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,9 +28,9 @@ requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.30.0 # via twine +tqdm==4.31.1 # via twine twine==1.12.1 urllib3==1.24.1 # via requests -virtualenv==16.3.0 # via pre-commit +virtualenv==16.4.0 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index a6c05582..80822929 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.90 -botocore==1.12.90 +boto3==1.9.91 +botocore==1.12.91 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 @@ -94,4 +94,4 @@ tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 werkzeug==0.14.1 -xmltodict==0.11.0 +xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index e4a34412..60cda2d7 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.90 # via moto +boto3==1.9.91 # via moto boto==2.49.0 # via moto -botocore==1.12.90 # via boto3, moto, s3transfer +botocore==1.12.91 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -61,4 +61,4 @@ urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk -xmltodict==0.11.0 # via moto +xmltodict==0.12.0 # via moto diff --git a/requirements.txt b/requirements.txt index f24d274e..8bc96ac2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.90 -botocore==1.12.90 +boto3==1.9.91 +botocore==1.12.91 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 @@ -84,4 +84,4 @@ tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask -xmltodict==0.11.0 +xmltodict==0.12.0 From 8abf95063cf53c0087b5c54c2e5a2e952bac4b5f Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Thu, 14 Feb 2019 11:57:27 +0100 Subject: [PATCH 08/43] Implement a ALLOW_CERT_DELETION option (boolean, default False). When enabled, the certificate delete API call will work and the UI will no longer display deleted certificates. When disabled (the default), the delete API call will not work (405 method not allowed) and the UI will show all certificates, regardless of the 'deleted' flag. --- docs/administration.rst | 7 +++++++ lemur/certificates/models.py | 2 +- lemur/certificates/service.py | 3 +++ lemur/certificates/views.py | 16 +++++++-------- lemur/migrations/versions/318b66568358_.py | 23 ++++++++++++++++++++++ lemur/tests/conf.py | 2 ++ lemur/tests/test_certificates.py | 4 ++-- 7 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 lemur/migrations/versions/318b66568358_.py diff --git a/docs/administration.rst b/docs/administration.rst index 9d6c8d12..352318f5 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -161,6 +161,13 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c Dump all imported or generated CSR and certificate details to stdout using OpenSSL. (default: `False`) +.. data:: ALLOW_CERT_DELETION + :noindex: + + When set to True, certificates can be marked as deleted via the API and deleted certificates will not be displayed + in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed" + and deleted certificates will always be visible in the UI. (default: `False`) + Certificate Default Options --------------------------- diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 34305cc2..ab43cd01 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -101,7 +101,7 @@ class Certificate(db.Model): issuer = Column(String(128)) serial = Column(String(128)) cn = Column(String(128)) - deleted = Column(Boolean, index=True) + deleted = Column(Boolean, index=True, default=False) dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='CASCADE'), nullable=True) not_before = Column(ArrowType) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index d5012012..22009043 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -381,6 +381,9 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now) + if current_app.config.get('ALLOW_CERT_DELETION', False): + query = query.filter(Certificate.deleted == False) # noqa + result = database.sort_and_page(query, Certificate, args) return result diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 948c44d6..37ebf518 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -6,10 +6,9 @@ .. moduleauthor:: Kevin Glisson """ import base64 -import arrow from builtins import str -from flask import Blueprint, make_response, jsonify, g +from flask import Blueprint, make_response, jsonify, g, current_app from flask_restful import reqparse, Api, inputs from lemur.common.schema import validate_schema @@ -678,17 +677,21 @@ class Certificates(AuthenticatedResource): .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 204 OK :reqheader Authorization: OAuth token to authenticate :statuscode 204: no error :statuscode 403: unauthenticated :statusoode 404: certificate not found + :statusoode 405: certificate deletion is disabled """ + if not current_app.config.get('ALLOW_CERT_DELETION', False): + return dict(message="Certificate deletion is disabled"), 405 + cert = service.get(certificate_id) - if not cert: + if not cert or cert.deleted: return dict(message="Cannot find specified certificate"), 404 # allow creators @@ -699,12 +702,9 @@ class Certificates(AuthenticatedResource): if not permission.can(): return dict(message='You are not authorized to delete this certificate'), 403 - if arrow.get(cert.not_after) > arrow.utcnow(): - return dict(message='Certificate is still valid, only expired certificates can be deleted'), 412 - service.update(certificate_id, deleted=True) log_service.create(g.current_user, 'delete_cert', certificate=cert) - return '', 204 + return 'Certificate deleted', 204 class NotificationCertificatesList(AuthenticatedResource): diff --git a/lemur/migrations/versions/318b66568358_.py b/lemur/migrations/versions/318b66568358_.py new file mode 100644 index 00000000..9d4aa48d --- /dev/null +++ b/lemur/migrations/versions/318b66568358_.py @@ -0,0 +1,23 @@ +""" Set 'deleted' flag from null to false on all certificates once + +Revision ID: 318b66568358 +Revises: 9f79024fe67b +Create Date: 2019-02-05 15:42:25.477587 + +""" + +# revision identifiers, used by Alembic. +revision = '318b66568358' +down_revision = '9f79024fe67b' + +from alembic import op + + +def upgrade(): + connection = op.get_bind() + # Delete duplicate entries + connection.execute('UPDATE certificates SET deleted = false WHERE deleted IS NULL') + + +def downgrade(): + pass diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index bbe155cd..525200cf 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -186,3 +186,5 @@ LDAP_BASE_DN = 'dc=example,dc=com' LDAP_EMAIL_DOMAIN = 'example.com' LDAP_REQUIRED_GROUP = 'Lemur Access' LDAP_DEFAULT_ROLE = 'role1' + +ALLOW_CERT_DELETION = True diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 8247c36b..75a29e16 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -737,8 +737,8 @@ def test_certificate_put_with_data(client, certificate, issuer_plugin): @pytest.mark.parametrize("token,status", [ (VALID_USER_HEADER_TOKEN, 403), - (VALID_ADMIN_HEADER_TOKEN, 412), - (VALID_ADMIN_API_TOKEN, 412), + (VALID_ADMIN_HEADER_TOKEN, 204), + (VALID_ADMIN_API_TOKEN, 404), ('', 401) ]) def test_certificate_delete(client, token, status): From 29bda6c00d2351cc4a08ed797c9940d8615a9c73 Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Thu, 14 Feb 2019 11:58:29 +0100 Subject: [PATCH 09/43] Fix typo's --- lemur/certificates/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 37ebf518..b464b3ed 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -682,8 +682,8 @@ class Certificates(AuthenticatedResource): :reqheader Authorization: OAuth token to authenticate :statuscode 204: no error :statuscode 403: unauthenticated - :statusoode 404: certificate not found - :statusoode 405: certificate deletion is disabled + :statuscode 404: certificate not found + :statuscode 405: certificate deletion is disabled """ if not current_app.config.get('ALLOW_CERT_DELETION', False): From eaa73998a0b17858d83fe21080f78f2ec4d2c1f3 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 19 Feb 2019 15:03:15 -0500 Subject: [PATCH 10/43] adding lemur_vault destination plugin --- lemur/plugins/lemur_vault/__init__.py | 5 ++ lemur/plugins/lemur_vault/plugin.py | 85 +++++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 lemur/plugins/lemur_vault/__init__.py create mode 100644 lemur/plugins/lemur_vault/plugin.py diff --git a/lemur/plugins/lemur_vault/__init__.py b/lemur/plugins/lemur_vault/__init__.py new file mode 100644 index 00000000..8ce5a7f3 --- /dev/null +++ b/lemur/plugins/lemur_vault/__init__.py @@ -0,0 +1,5 @@ +try: + VERSION = __import__('pkg_resources') \ + .get_distribution(__name__).version +except Exception as e: + VERSION = 'unknown' diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault/plugin.py new file mode 100644 index 00000000..505170ad --- /dev/null +++ b/lemur/plugins/lemur_vault/plugin.py @@ -0,0 +1,85 @@ +""" +.. module: lemur.plugins.lemur_vault.plugin + :platform: Unix + :copyright: (c) 2019 + :license: Apache, see LICENCE for more details. + + Plugin for uploading certificates and private key as secret to hashi vault + that can be pulled down by end point nodes. + +.. moduleauthor:: Christopher Jolley +""" +import hvac + +#import lemur_vault +from flask import current_app + +from lemur.common.defaults import common_name +from lemur.common.utils import parse_certificate +from lemur.plugins.bases import DestinationPlugin + +class VaultDestinationPlugin(DestinationPlugin): + """Hashicorp Vault Destination plugin for Lemur""" + title = 'Vault' + slug = 'hashi-vault-destination' + description = 'Allow the uploading of certificates to Hashi Vault as secret' + + author = 'Christopher Jolley' + author_url = 'https://github.com/alwaysjolley/lemur' + + options = [ + { + 'name': 'vaultMount', + 'type': 'str', + 'required': True, + 'validation': '^[a-zA-Z0-9]+$', + 'helpMessage': 'Must be a valid Vault secrets mount name!' + }, + { + 'name': 'vaultPath', + 'type': 'str', + 'required': True, + 'validation': '^([a-zA-Z0-9_-]+/?)+$', + 'helpMessage': 'Must be a valid Vault secrets path' + }, + { + 'name': 'vaultUrl', + 'type': 'str', + 'required': True, + 'validation': '^https?://[a-zA-Z0-9.-]+(?::[0-9]+)?$', + 'helpMessage': 'Must be a valid Vault server url' + } + ] + + def __init__(self, *args, **kwargs): + super(VaultDestinationPlugin, self).__init__(*args, **kwargs) + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + """ + Upload certificate and private key + + :param private_key: + :param cert_chain: + :return: + """ + cn = common_name(parse_certificate(body)) + data = {} + #current_app.logger.warning("Cert body content: {0}".format(body)) + + token = current_app.config.get('VAULT_TOKEN') + + mount = self.get_option('vaultMount', options) + path = '{0}/{1}'.format(self.get_option('vaultPath', options),cn) + url = self.get_option('vaultUrl', options) + + client = hvac.Client(url=url, token=token) + + data['cert'] = cert_chain + data['key'] = private_key + + ## upload certificate and key + try: + client.secrets.kv.v1.create_or_update_secret(path=path, mount_point=mount, secret=data) + except Exception as err: + current_app.logger.exception( + "Exception uploading secret to vault: {0}".format(err), exc_info=True) diff --git a/setup.py b/setup.py index 1511b013..b5dcdb3b 100644 --- a/setup.py +++ b/setup.py @@ -154,7 +154,8 @@ setup( 'digicert_cis_issuer = lemur.plugins.lemur_digicert.plugin:DigiCertCISIssuerPlugin', 'digicert_cis_source = lemur.plugins.lemur_digicert.plugin:DigiCertCISSourcePlugin', 'csr_export = lemur.plugins.lemur_csr.plugin:CSRExportPlugin', - 'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin' + 'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin', + 'vault_desination = lemur.plugins.lemur_vault.plugin:VaultDestinationPlugin' ], }, classifiers=[ From a0ca486f0f9975eeffe6dddb13ed3fe60eee9661 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 19 Feb 2019 15:22:11 -0500 Subject: [PATCH 11/43] adding hvac and updating requrements --- requirements-dev.txt | 12 ++++++------ requirements-docs.txt | 39 ++++++++++++++++++++------------------- requirements-tests.txt | 24 ++++++++++++------------ requirements.in | 3 ++- requirements.txt | 35 ++++++++++++++++++----------------- 5 files changed, 58 insertions(+), 55 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index ac35f3e9..6e2a3fb9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.2.1 # via pre-commit +identify==1.2.2 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit importlib-resources==1.0.2 # via pre-commit @@ -19,19 +19,19 @@ invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.2 +pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine -requests-toolbelt==0.9.0 # via twine +requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.30.0 # via twine -twine==1.12.1 +tqdm==4.31.1 # via twine +twine==1.13.0 urllib3==1.24.1 # via requests -virtualenv==16.3.0 # via pre-commit +virtualenv==16.4.0 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index 15085766..e9dd92cb 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,24 +4,24 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.30.2 +acme==0.31.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.7 -amqp==2.4.0 +amqp==2.4.1 aniso8601==4.1.0 -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.86 -botocore==1.12.86 +boto3==1.9.98 +botocore==1.12.98 celery[redis]==4.2.1 certifi==2018.11.29 -cffi==1.11.5 +cffi==1.12.1 chardet==3.0.4 click==7.0 cloudflare==2.1.0 @@ -33,7 +33,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -41,6 +41,7 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 +hvac==0.7.2 idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 @@ -49,17 +50,17 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.2.post1 +kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 -marshmallow-sqlalchemy==0.15.0 -marshmallow==2.18.0 +marshmallow-sqlalchemy==0.16.0 +marshmallow==2.18.1 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 -pbr==5.1.1 +pbr==5.1.2 pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 @@ -71,26 +72,26 @@ pynacl==1.3.0 pyopenssl==19.0.0 pyparsing==2.3.1 # via packaging pyrfc3339==1.1 -python-dateutil==2.7.5 -python-editor==1.0.3 +python-dateutil==2.8.0 +python-editor==1.0.4 pytz==2018.9 pyyaml==3.13 raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.9.0 +requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 -s3transfer==0.1.13 +s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.4.2 -sphinx==1.8.3 +sphinx-rtd-theme==0.4.3 +sphinx==1.8.4 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 +sqlalchemy==1.2.18 tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 werkzeug==0.14.1 -xmltodict==0.11.0 +xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index c326e951..1bb8ba03 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,14 +5,14 @@ # pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography -atomicwrites==1.2.1 # via pytest +atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.86 # via moto +boto3==1.9.98 # via moto boto==2.49.0 # via moto -botocore==1.12.86 # via boto3, moto, s3transfer +botocore==1.12.98 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests -cffi==1.11.5 # via cryptography +cffi==1.12.1 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 @@ -34,10 +34,10 @@ jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==5.0.0 # via pytest +more-itertools==6.0.0 # via pytest moto==1.3.7 nose==1.3.7 -pbr==5.1.1 # via mock +pbr==5.1.2 # via mock pluggy==0.8.1 # via pytest py==1.7.0 # via pytest pyaml==18.11.0 # via moto @@ -45,20 +45,20 @@ pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose pyflakes==2.1.0 pytest-flask==0.14.0 -pytest-mock==1.10.0 -pytest==4.1.1 -python-dateutil==2.7.5 # via botocore, faker, freezegun, moto +pytest-mock==1.10.1 +pytest==4.3.0 +python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.10.5 # via moto -s3transfer==0.1.13 # via boto3 -six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client +s3transfer==0.2.0 # via boto3 +six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk -xmltodict==0.11.0 # via moto +xmltodict==0.12.0 # via moto diff --git a/requirements.in b/requirements.in index 9824650b..1147cc8d 100644 --- a/requirements.in +++ b/requirements.in @@ -23,6 +23,7 @@ Flask Flask-Cors future gunicorn +hvac # required for the vault destination plugin inflection jinja2 lockfile @@ -42,4 +43,4 @@ retrying six SQLAlchemy-Utils tabulate -xmltodict \ No newline at end of file +xmltodict diff --git a/requirements.txt b/requirements.txt index c595e509..edd56b09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,22 +4,22 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.30.2 +acme==0.31.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.7 # via flask-migrate -amqp==2.4.0 # via kombu +amqp==2.4.1 # via kombu aniso8601==4.1.0 # via flask-restful -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.86 -botocore==1.12.86 +boto3==1.9.98 +botocore==1.12.98 celery[redis]==4.2.1 certifi==2018.11.29 -cffi==1.11.5 # via bcrypt, cryptography, pynacl +cffi==1.12.1 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 @@ -31,7 +31,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -39,6 +39,7 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 +hvac==0.7.2 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask @@ -46,16 +47,16 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.2.post1 # via celery +kombu==4.3.0 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako -marshmallow-sqlalchemy==0.15.0 -marshmallow==2.18.0 +marshmallow-sqlalchemy==0.16.0 +marshmallow==2.18.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.1.1 # via mock +pbr==5.1.2 # via mock pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 # via python-ldap @@ -65,22 +66,22 @@ pyjwt==1.7.1 pynacl==1.3.0 # via paramiko pyopenssl==19.0.0 pyrfc3339==1.1 # via acme -python-dateutil==2.7.5 # via alembic, arrow, botocore -python-editor==1.0.3 # via alembic +python-dateutil==2.8.0 # via alembic, arrow, botocore +python-editor==1.0.4 # via alembic python-ldap==3.1.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.9.0 # via acme +requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 -s3transfer==0.1.13 # via boto3 +s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.18 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask -xmltodict==0.11.0 +xmltodict==0.12.0 From ef0c08dfd9927e5d9db149a64685c9b8ba9fb350 Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Thu, 21 Feb 2019 16:33:43 +0100 Subject: [PATCH 12/43] Fix: when no alias is entered when exporting a certificate, the alias is set to 'blah'. This fix sets it to the common name instead. --- lemur/plugins/lemur_java/plugin.py | 4 +++- lemur/plugins/lemur_openssl/plugin.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_java/plugin.py b/lemur/plugins/lemur_java/plugin.py index 5aab5342..7eb33b90 100644 --- a/lemur/plugins/lemur_java/plugin.py +++ b/lemur/plugins/lemur_java/plugin.py @@ -15,6 +15,8 @@ from cryptography.fernet import Fernet from lemur.utils import mktempfile, mktemppath from lemur.plugins.bases import ExportPlugin from lemur.plugins import lemur_java as java +from lemur.common.utils import parse_certificate +from lemur.common.defaults import common_name def run_process(command): @@ -233,7 +235,7 @@ class JavaKeystoreExportPlugin(ExportPlugin): if self.get_option('alias', options): alias = self.get_option('alias', options) else: - alias = "blah" + alias = common_name(parse_certificate(body)) with mktemppath() as jks_tmp: create_keystore(body, chain, jks_tmp, key, alias, passphrase) diff --git a/lemur/plugins/lemur_openssl/plugin.py b/lemur/plugins/lemur_openssl/plugin.py index 9ddce925..6d6f89aa 100644 --- a/lemur/plugins/lemur_openssl/plugin.py +++ b/lemur/plugins/lemur_openssl/plugin.py @@ -14,7 +14,8 @@ from flask import current_app from lemur.utils import mktempfile, mktemppath from lemur.plugins.bases import ExportPlugin from lemur.plugins import lemur_openssl as openssl -from lemur.common.utils import get_psuedo_random_string +from lemur.common.utils import get_psuedo_random_string, parse_certificate +from lemur.common.defaults import common_name def run_process(command): @@ -122,7 +123,7 @@ class OpenSSLExportPlugin(ExportPlugin): if self.get_option('alias', options): alias = self.get_option('alias', options) else: - alias = "blah" + alias = common_name(parse_certificate(body)) type = self.get_option('type', options) From 14d8596b8a175cca79d41eaba19fc53a3dd249fd Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 21 Feb 2019 20:19:14 -0800 Subject: [PATCH 13/43] updating requirements --- requirements-dev.txt | 8 ++++---- requirements-docs.txt | 18 +++++++++--------- requirements-tests.txt | 16 ++++++++-------- requirements.txt | 16 ++++++++-------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f5d6be3c..c7d7986c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in +# pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index # aspy.yaml==1.1.2 # via pre-commit bleach==3.1.0 # via readme-renderer @@ -11,14 +11,14 @@ cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.2.1 # via pre-commit +identify==1.2.2 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.3 +pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer @@ -29,7 +29,7 @@ requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.31.1 # via twine -twine==1.12.1 +twine==1.13.0 urllib3==1.24.1 # via requests virtualenv==16.4.0 # via pre-commit webencodings==0.5.1 # via bleach diff --git a/requirements-docs.txt b/requirements-docs.txt index 80822929..c3848b44 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in +# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # acme==0.31.0 alabaster==0.7.12 # via sphinx @@ -10,19 +10,19 @@ alembic-autogenerate-enums==0.0.2 alembic==1.0.7 amqp==2.4.1 aniso8601==4.1.0 -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.91 -botocore==1.12.91 +boto3==1.9.100 +botocore==1.12.100 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 -cffi==1.11.5 +cffi==1.12.1 chardet==3.0.4 click==7.0 cloudflare==2.1.0 @@ -34,7 +34,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -55,7 +55,7 @@ lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.0 +marshmallow==2.18.1 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx @@ -84,12 +84,12 @@ retrying==1.3.3 s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.4.2 +sphinx-rtd-theme==0.4.3 sphinx==1.8.4 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 +sqlalchemy==1.2.18 tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 60cda2d7..ad97675e 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,17 +2,17 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in +# pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.91 # via moto +boto3==1.9.100 # via moto boto==2.49.0 # via moto -botocore==1.12.91 # via boto3, moto, s3transfer +botocore==1.12.100 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests -cffi==1.11.5 # via cryptography +cffi==1.12.1 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 @@ -34,19 +34,19 @@ jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==5.0.0 # via pytest +more-itertools==6.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.2 # via mock pluggy==0.8.1 # via pytest -py==1.7.0 # via pytest +py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose pyflakes==2.1.0 pytest-flask==0.14.0 pytest-mock==1.10.1 -pytest==4.2.0 +pytest==4.3.0 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto @@ -55,7 +55,7 @@ requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.10.5 # via moto s3transfer==0.2.0 # via boto3 -six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client +six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker diff --git a/requirements.txt b/requirements.txt index 8bc96ac2..0319fa3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,25 +2,25 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements.txt requirements.in +# pip-compile --output-file requirements.txt requirements.in -U --no-index # acme==0.31.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.7 # via flask-migrate amqp==2.4.1 # via kombu aniso8601==4.1.0 # via flask-restful -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.91 -botocore==1.12.91 +boto3==1.9.100 +botocore==1.12.100 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 -cffi==1.11.5 # via bcrypt, cryptography, pynacl +cffi==1.12.1 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 @@ -32,7 +32,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -52,7 +52,7 @@ lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.0 +marshmallow==2.18.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 @@ -79,7 +79,7 @@ retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.18 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp From cd65a36437e05d6cd4f5e29b76c6f5e76567beb1 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Mon, 25 Feb 2019 09:42:07 -0500 Subject: [PATCH 14/43] - support multiple bundle configuration, nginx, apache, cert only - update vault destination to support multi cert under one object - added san list as key value - read and update object with new keys, keeping other keys, allowing us to keep an iterable list of keys in an object for deploying multiple certs to a single node --- .gitignore | 5 ++ lemur/plugins/lemur_vault/plugin.py | 81 +++++++++++++++++++++++++---- requirements-dev.txt | 6 +-- requirements-docs.txt | 10 ++-- requirements-tests.txt | 16 +++--- requirements.txt | 10 ++-- 6 files changed, 98 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 97af00ca..72e85f26 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,11 @@ package-lock.json /lemur/static/dist/ /lemur/static/app/vendor/ /wheelhouse +/lemur/lib +/lemur/bin +/lemur/lib64 +/lemur/include + docs/_build .editorconfig .idea diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault/plugin.py index 505170ad..58a9e601 100644 --- a/lemur/plugins/lemur_vault/plugin.py +++ b/lemur/plugins/lemur_vault/plugin.py @@ -18,6 +18,10 @@ from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate from lemur.plugins.bases import DestinationPlugin +from cryptography import x509 +from cryptography.hazmat.backends import default_backend + + class VaultDestinationPlugin(DestinationPlugin): """Hashicorp Vault Destination plugin for Lemur""" title = 'Vault' @@ -48,6 +52,25 @@ class VaultDestinationPlugin(DestinationPlugin): 'required': True, 'validation': '^https?://[a-zA-Z0-9.-]+(?::[0-9]+)?$', 'helpMessage': 'Must be a valid Vault server url' + }, + { + 'name': 'bundleChain', + 'type': 'select', + 'value': 'cert only', + 'available': [ + 'Nginx', + 'Apache', + 'no chain' + ], + 'required': True, + 'helpMessage': 'Bundle the chain into the certificate' + }, + { + 'name': 'objectName', + 'type': 'str', + 'required': False, + 'validation': '[0-9a-zA-Z:_-]+', + 'helpMessage': 'Name to bundle certs under, if blank use cn' } ] @@ -62,24 +85,64 @@ class VaultDestinationPlugin(DestinationPlugin): :param cert_chain: :return: """ - cn = common_name(parse_certificate(body)) - data = {} - #current_app.logger.warning("Cert body content: {0}".format(body)) + cname = common_name(parse_certificate(body)) + secret = {'data':{}} + key_name = '{0}.key'.format(cname) + cert_name = '{0}.crt'.format(cname) + chain_name = '{0}.chain'.format(cname) + sans_name = '{0}.san'.format(cname) token = current_app.config.get('VAULT_TOKEN') mount = self.get_option('vaultMount', options) - path = '{0}/{1}'.format(self.get_option('vaultPath', options),cn) + path = self.get_option('vaultPath', options) url = self.get_option('vaultUrl', options) + bundle = self.get_option('bundleChain', options) + obj_name = self.get_option('objectName', options) client = hvac.Client(url=url, token=token) + if obj_name: + path = '{0}/{1}'.format(path, obj_name) + else: + path = '{0}/{1}'.format(path, cname) - data['cert'] = cert_chain - data['key'] = private_key + secret = get_secret(url, token, mount, path) + - ## upload certificate and key + if bundle == 'Nginx' and cert_chain: + secret['data'][cert_name] = '{0}\n{1}'.format(body, cert_chain) + elif bundle == 'Apache' and cert_chain: + secret['data'][cert_name] = body + secret['data'][chain_name] = cert_chain + else: + secret['data'][cert_name] = body + secret['data'][key_name] = private_key + san_list = get_san_list(body) + if isinstance(san_list, list): + secret['data'][sans_name] = san_list try: - client.secrets.kv.v1.create_or_update_secret(path=path, mount_point=mount, secret=data) - except Exception as err: + client.secrets.kv.v1.create_or_update_secret( + path=path, mount_point=mount, secret=secret['data']) + except ConnectionError as err: current_app.logger.exception( "Exception uploading secret to vault: {0}".format(err), exc_info=True) + +def get_san_list(body): + """ parse certificate for SAN names and return list, return empty list on error """ + try: + byte_body = body.encode('utf-8') + cert = x509.load_pem_x509_certificate(byte_body, default_backend()) + ext = cert.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) + return ext.value.get_values_for_type(x509.DNSName) + except: + pass + return [] + +def get_secret(url, token, mount, path): + result = {'data': {}} + try: + client = hvac.Client(url=url, token=token) + result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) + except: + pass + return result diff --git a/requirements-dev.txt b/requirements-dev.txt index 6e2a3fb9..fd487bd7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in +# pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index # aspy.yaml==1.1.2 # via pre-commit bleach==3.1.0 # via readme-renderer @@ -11,7 +11,7 @@ cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.2.2 # via pre-commit +identify==1.3.0 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit importlib-resources==1.0.2 # via pre-commit @@ -32,6 +32,6 @@ toml==0.10.0 # via pre-commit tqdm==4.31.1 # via twine twine==1.13.0 urllib3==1.24.1 # via requests -virtualenv==16.4.0 # via pre-commit +virtualenv==16.4.1 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index e9dd92cb..8b9c3f2b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in +# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # acme==0.31.0 alabaster==0.7.12 # via sphinx @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.98 -botocore==1.12.98 +boto3==1.9.101 +botocore==1.12.101 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.12.1 @@ -47,13 +47,13 @@ imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 jinja2==2.10 -jmespath==0.9.3 +jmespath==0.9.4 josepy==1.1.0 jsonlines==1.2.0 kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 -markupsafe==1.1.0 +markupsafe==1.1.1 marshmallow-sqlalchemy==0.16.0 marshmallow==2.18.1 mock==2.0.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 1bb8ba03..1c3a4969 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,15 +2,15 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in +# pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.98 # via moto +boto3==1.9.101 # via moto boto==2.49.0 # via moto -botocore==1.12.98 # via boto3, moto, s3transfer +botocore==1.12.101 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.12.1 # via cryptography chardet==3.0.4 # via requests @@ -29,17 +29,17 @@ future==0.17.1 # via python-jose idna==2.8 # via requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto -jmespath==0.9.3 # via boto3, botocore +jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk -markupsafe==1.1.0 # via jinja2 +markupsafe==1.1.1 # via jinja2 mock==2.0.0 # via moto more-itertools==6.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.2 # via mock -pluggy==0.8.1 # via pytest -py==1.7.0 # via pytest +pluggy==0.9.0 # via pytest +py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose @@ -58,7 +58,7 @@ s3transfer==0.2.0 # via boto3 six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests -websocket-client==0.54.0 # via docker +websocket-client==0.55.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements.txt b/requirements.txt index edd56b09..a8615094 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements.txt requirements.in +# pip-compile --output-file requirements.txt requirements.in -U --no-index # acme==0.31.0 alembic-autogenerate-enums==0.0.2 @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.98 -botocore==1.12.98 +boto3==1.9.101 +botocore==1.12.101 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.12.1 # via bcrypt, cryptography, pynacl @@ -44,13 +44,13 @@ idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask jinja2==2.10 -jmespath==0.9.3 # via boto3, botocore +jmespath==0.9.4 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.3.0 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic -markupsafe==1.1.0 # via jinja2, mako +markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.16.0 marshmallow==2.18.1 mock==2.0.0 # via acme From 40fac02d8b0dc9d411331eb25a4f16fddb774ab0 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 25 Feb 2019 19:05:54 -0800 Subject: [PATCH 15/43] the check_cert_signature() method was attempting to compare RSA and ECC signatures. If a ec public-key certificate is signed with an RSA key, then it can't be a self-signed certificate, in which case we just raise InvalidSignature. --- lemur/common/utils.py | 9 +++++++-- lemur/tests/test_utils.py | 3 ++- lemur/tests/vectors.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index f3ac5fe7..7c9269cf 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -147,6 +147,8 @@ def generate_private_key(key_type): def check_cert_signature(cert, issuer_public_key): """ Check a certificate's signature against an issuer public key. + Before EC validation, make sure public key and signature are of the same type, + otherwise verification not possible (raise InvalidSignature) On success, returns None; on failure, raises UnsupportedAlgorithm or InvalidSignature. """ if isinstance(issuer_public_key, rsa.RSAPublicKey): @@ -160,9 +162,10 @@ def check_cert_signature(cert, issuer_public_key): else: padder = padding.PKCS1v15() issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, padder, cert.signature_hash_algorithm) + elif isinstance(issuer_public_key, ec.EllipticCurvePublicKey) and isinstance(cert.signature_hash_algorithm, ec.ECDSA): + issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, cert.signature_hash_algorithm) else: - # EllipticCurvePublicKey or DSAPublicKey - issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, cert.signature_hash_algorithm) + raise InvalidSignature def is_selfsigned(cert): @@ -176,6 +179,8 @@ def is_selfsigned(cert): return True except InvalidSignature: return False + except UnsupportedAlgorithm as e: + raise Exception(e) def is_weekend(date): diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 3e226f0f..c44f7b9c 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from lemur.tests.vectors import SAN_CERT, INTERMEDIATE_CERT, ROOTCA_CERT +from lemur.tests.vectors import SAN_CERT, INTERMEDIATE_CERT, ROOTCA_CERT, EC_CERT_EXAMPLE def test_generate_private_key(): @@ -83,3 +83,4 @@ def test_is_selfsigned(selfsigned_cert): assert is_selfsigned(INTERMEDIATE_CERT) is False # Root CA certificates are also technically self-signed assert is_selfsigned(ROOTCA_CERT) is True + assert is_selfsigned(EC_CERT_EXAMPLE) is False diff --git a/lemur/tests/vectors.py b/lemur/tests/vectors.py index 5da37c61..9af77bf6 100644 --- a/lemur/tests/vectors.py +++ b/lemur/tests/vectors.py @@ -394,3 +394,31 @@ zm3Cn4Ul8DO26w9QS4fmZjmnPOZFXYMWoOR6osHzb62PWQ8FBMqXcdToBV2Q9Iw4 PiFAxlc0tVjlLqQ= -----END CERTIFICATE REQUEST----- """ + + +EC_CERT_STR = """ +-----BEGIN CERTIFICATE----- +MIIDxzCCAq+gAwIBAgIIHsJeci1JWAkwDQYJKoZIhvcNAQELBQAwVDELMAkGA1UE +BhMCVVMxHjAcBgNVBAoTFUdvb2dsZSBUcnVzdCBTZXJ2aWNlczElMCMGA1UEAxMc +R29vZ2xlIEludGVybmV0IEF1dGhvcml0eSBHMzAeFw0xOTAyMTMxNTM1NTdaFw0x +OTA1MDgxNTM1MDBaMGgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRYwFAYDVQQHDA1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKDApHb29nbGUgTExDMRcw +FQYDVQQDDA53d3cuZ29vZ2xlLmNvbTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BKwMlIbd4rAwf6eWoa6RrR2w0s5k1M40XOORPf96PByPmld+qhjRMLvA/xcAxdCR +XdcMfaX6EUr0Zw8CepitMB2jggFSMIIBTjATBgNVHSUEDDAKBggrBgEFBQcDATAO +BgNVHQ8BAf8EBAMCB4AwGQYDVR0RBBIwEIIOd3d3Lmdvb2dsZS5jb20waAYIKwYB +BQUHAQEEXDBaMC0GCCsGAQUFBzAChiFodHRwOi8vcGtpLmdvb2cvZ3NyMi9HVFNH +SUFHMy5jcnQwKQYIKwYBBQUHMAGGHWh0dHA6Ly9vY3NwLnBraS5nb29nL0dUU0dJ +QUczMB0GA1UdDgQWBBQLovm8GG0oG91gOGCL58YPNoAlejAMBgNVHRMBAf8EAjAA +MB8GA1UdIwQYMBaAFHfCuFCaZ3Z2sS3ChtCDoH6mfrpLMCEGA1UdIAQaMBgwDAYK +KwYBBAHWeQIFAzAIBgZngQwBAgIwMQYDVR0fBCowKDAmoCSgIoYgaHR0cDovL2Ny +bC5wa2kuZ29vZy9HVFNHSUFHMy5jcmwwDQYJKoZIhvcNAQELBQADggEBAKFbmNOA +e3pJ7UVI5EmkAMZgSDRdrsLHV6F7WluuyYCyE/HFpZjBd6y8xgGtYWcask6edwrq +zrcXNEN/GY34AYre0M+p0xAs+lKSwkrJd2sCgygmzsBFtGwjW6lhjm+rg83zPHhH +mQZ0ShUR1Kp4TvzXgxj44RXOsS5ZyDe3slGiG4aw/hl+igO8Y8JMvcv/Tpzo+V75 +BkDAFmLRi08NayfeyCqK/TcRpzxKMKhS7jEHK8Pzu5P+FyFHKqIsobi+BA+psOix +5nZLhrweLdKNz387mE2lSSKzr7qeLGHSOMt+ajQtZio4YVyZqJvg4Y++J0n5+Rjw +MXp8GrvTfn1DQ+o= +-----END CERTIFICATE----- +""" +EC_CERT_EXAMPLE = parse_certificate(EC_CERT_STR) From e64de7d312816aa5a16369d57ba403072d52b436 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 25 Feb 2019 19:12:20 -0800 Subject: [PATCH 16/43] updating requirements --- requirements-dev.txt | 12 ++++++------ requirements-docs.txt | 24 ++++++++++++------------ requirements-tests.txt | 26 +++++++++++++------------- requirements.txt | 22 +++++++++++----------- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f5d6be3c..1cfbe393 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in +# pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index # aspy.yaml==1.1.2 # via pre-commit bleach==3.1.0 # via readme-renderer @@ -11,26 +11,26 @@ cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.2.1 # via pre-commit +identify==1.3.0 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.3 +pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer -pyyaml==4.2b4 +pyyaml==5.1b1 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.31.1 # via twine -twine==1.12.1 +twine==1.13.0 urllib3==1.24.1 # via requests -virtualenv==16.4.0 # via pre-commit +virtualenv==16.4.1 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index 80822929..db20a4b9 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in +# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # acme==0.31.0 alabaster==0.7.12 # via sphinx @@ -10,19 +10,19 @@ alembic-autogenerate-enums==0.0.2 alembic==1.0.7 amqp==2.4.1 aniso8601==4.1.0 -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.91 -botocore==1.12.91 +boto3==1.9.102 +botocore==1.12.102 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 -cffi==1.11.5 +cffi==1.12.1 chardet==3.0.4 click==7.0 cloudflare==2.1.0 @@ -34,7 +34,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -47,15 +47,15 @@ imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 jinja2==2.10 -jmespath==0.9.3 +jmespath==0.9.4 josepy==1.1.0 jsonlines==1.2.0 kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 -markupsafe==1.1.0 +markupsafe==1.1.1 marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.0 +marshmallow==2.18.1 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx @@ -75,7 +75,7 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 pytz==2018.9 -pyyaml==4.2b4 +pyyaml==5.1b1 raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.9.1 @@ -84,12 +84,12 @@ retrying==1.3.3 s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.4.2 +sphinx-rtd-theme==0.4.3 sphinx==1.8.4 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 +sqlalchemy==1.2.18 tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 60cda2d7..d1d6ae7f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -2,17 +2,17 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in +# pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.91 # via moto +boto3==1.9.102 # via moto boto==2.49.0 # via moto -botocore==1.12.91 # via boto3, moto, s3transfer +botocore==1.12.102 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests -cffi==1.11.5 # via cryptography +cffi==1.12.1 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 @@ -29,36 +29,36 @@ future==0.17.1 # via python-jose idna==2.8 # via requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto -jmespath==0.9.3 # via boto3, botocore +jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.1 # via aws-xray-sdk -markupsafe==1.1.0 # via jinja2 +markupsafe==1.1.1 # via jinja2 mock==2.0.0 # via moto -more-itertools==5.0.0 # via pytest +more-itertools==6.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.2 # via mock -pluggy==0.8.1 # via pytest -py==1.7.0 # via pytest +pluggy==0.9.0 # via pytest +py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose pyflakes==2.1.0 pytest-flask==0.14.0 pytest-mock==1.10.1 -pytest==4.2.0 +pytest==4.3.0 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto -pyyaml==4.2b4 +pyyaml==5.1b1 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.10.5 # via moto s3transfer==0.2.0 # via boto3 -six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client +six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests -websocket-client==0.54.0 # via docker +websocket-client==0.55.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements.txt b/requirements.txt index 8bc96ac2..72a37692 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,25 +2,25 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --no-index --output-file requirements.txt requirements.in +# pip-compile --output-file requirements.txt requirements.in -U --no-index # acme==0.31.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.7 # via flask-migrate amqp==2.4.1 # via kombu aniso8601==4.1.0 # via flask-restful -arrow==0.13.0 +arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.91 -botocore==1.12.91 +boto3==1.9.102 +botocore==1.12.102 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 -cffi==1.11.5 # via bcrypt, cryptography, pynacl +cffi==1.12.1 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 @@ -32,7 +32,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.1 +flask-migrate==2.4.0 flask-principal==0.4.0 flask-restful==0.3.7 flask-script==2.0.6 @@ -44,15 +44,15 @@ idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask jinja2==2.10 -jmespath==0.9.3 # via boto3, botocore +jmespath==0.9.4 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.3.0 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic -markupsafe==1.1.0 # via jinja2, mako +markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.0 +marshmallow==2.18.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 @@ -70,7 +70,7 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-ldap==3.1.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==4.2b4 +pyyaml==5.1b1 raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.9.1 # via acme @@ -79,7 +79,7 @@ retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.18 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp From 53301728fa9e052214f7c2f8211a693ff5313ac9 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 26 Feb 2019 09:15:12 -0500 Subject: [PATCH 17/43] Moved url to config file instead of plugin option. One one url can be supported unless both the token and url are moved to the plugin options. --- lemur/plugins/lemur_vault/plugin.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault/plugin.py index 58a9e601..2e46b155 100644 --- a/lemur/plugins/lemur_vault/plugin.py +++ b/lemur/plugins/lemur_vault/plugin.py @@ -47,11 +47,11 @@ class VaultDestinationPlugin(DestinationPlugin): 'helpMessage': 'Must be a valid Vault secrets path' }, { - 'name': 'vaultUrl', + 'name': 'objectName', 'type': 'str', - 'required': True, - 'validation': '^https?://[a-zA-Z0-9.-]+(?::[0-9]+)?$', - 'helpMessage': 'Must be a valid Vault server url' + 'required': False, + 'validation': '[0-9a-zA-Z:_-]+', + 'helpMessage': 'Name to bundle certs under, if blank use cn' }, { 'name': 'bundleChain', @@ -64,13 +64,6 @@ class VaultDestinationPlugin(DestinationPlugin): ], 'required': True, 'helpMessage': 'Bundle the chain into the certificate' - }, - { - 'name': 'objectName', - 'type': 'str', - 'required': False, - 'validation': '[0-9a-zA-Z:_-]+', - 'helpMessage': 'Name to bundle certs under, if blank use cn' } ] @@ -93,10 +86,10 @@ class VaultDestinationPlugin(DestinationPlugin): sans_name = '{0}.san'.format(cname) token = current_app.config.get('VAULT_TOKEN') + url = current_app.config.get('VAULT_URL') mount = self.get_option('vaultMount', options) path = self.get_option('vaultPath', options) - url = self.get_option('vaultUrl', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) From 16a18cc4b71d780821e3480f19787e117aec96f9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 26 Feb 2019 16:35:49 -0800 Subject: [PATCH 18/43] adding more edge test cases for EC-certs --- lemur/tests/test_utils.py | 9 +++++- lemur/tests/vectors.py | 67 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index c44f7b9c..74c11643 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from lemur.tests.vectors import SAN_CERT, INTERMEDIATE_CERT, ROOTCA_CERT, EC_CERT_EXAMPLE +from lemur.tests.vectors import SAN_CERT, INTERMEDIATE_CERT, ROOTCA_CERT, EC_CERT_EXAMPLE, ECDSA_PRIME256V1_CERT, ECDSA_SECP384r1_CERT, DSA_CERT def test_generate_private_key(): @@ -84,3 +84,10 @@ def test_is_selfsigned(selfsigned_cert): # Root CA certificates are also technically self-signed assert is_selfsigned(ROOTCA_CERT) is True assert is_selfsigned(EC_CERT_EXAMPLE) is False + + # selfsigned certs + assert is_selfsigned(ECDSA_PRIME256V1_CERT) is True + assert is_selfsigned(ECDSA_SECP384r1_CERT) is True + # unsupported algorithm (DSA) + with pytest.raises(Exception): + is_selfsigned(DSA_CERT) diff --git a/lemur/tests/vectors.py b/lemur/tests/vectors.py index 9af77bf6..06e7445a 100644 --- a/lemur/tests/vectors.py +++ b/lemur/tests/vectors.py @@ -422,3 +422,70 @@ MXp8GrvTfn1DQ+o= -----END CERTIFICATE----- """ EC_CERT_EXAMPLE = parse_certificate(EC_CERT_STR) + + +ECDSA_PRIME256V1_CERT_STR = """ +-----BEGIN CERTIFICATE----- +MIICUTCCAfYCCQCvH7H/e2nuiDAKBggqhkjOPQQDAjCBrzELMAkGA1UEBhMCVVMx +EzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczEjMCEGA1UE +CgwaTGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsMHVVuaXR0ZXN0 +aW5nIE9wZXJhdGlvbnMgQ2VudGVyMSowKAYDVQQDDCFMZW11clRydXN0IFVuaXR0 +ZXN0cyBSb290IENBIDIwMTkwHhcNMTkwMjI2MTgxMTUyWhcNMjkwMjIzMTgxMTUy +WjCBrzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcM +CUxvcyBHYXRvczEjMCEGA1UECgwaTGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQx +JjAkBgNVBAsMHVVuaXR0ZXN0aW5nIE9wZXJhdGlvbnMgQ2VudGVyMSowKAYDVQQD +DCFMZW11clRydXN0IFVuaXR0ZXN0cyBSb290IENBIDIwMTkwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAAQsnAVUtpDCFMK/k9Chynu8BWRVUBUYbGQ9Q9xeLR60J4fD +uBt48YpTqg5RMZEclVknMReXqTmqphOBo37/YVdlMAoGCCqGSM49BAMCA0kAMEYC +IQDQZ6xfBiCTHxY4GM4+zLeG1iPBUSfIJOjkFNViFZY/XAIhAJYmrkVQb/YjWCdd +Vl89McYhmV4IV7WDgUmUhkUSFXgy +-----END CERTIFICATE----- +""" +ECDSA_PRIME256V1_CERT = parse_certificate(ECDSA_PRIME256V1_CERT_STR) + + +ECDSA_SECP384r1_CERT_STR = """ +-----BEGIN CERTIFICATE----- +MIICjjCCAhMCCQD2UadeQ7ub1jAKBggqhkjOPQQDAjCBrzELMAkGA1UEBhMCVVMx +EzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczEjMCEGA1UE +CgwaTGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsMHVVuaXR0ZXN0 +aW5nIE9wZXJhdGlvbnMgQ2VudGVyMSowKAYDVQQDDCFMZW11clRydXN0IFVuaXR0 +ZXN0cyBSb290IENBIDIwMTgwHhcNMTkwMjI2MTgxODU2WhcNMjkwMjIzMTgxODU2 +WjCBrzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcM +CUxvcyBHYXRvczEjMCEGA1UECgwaTGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQx +JjAkBgNVBAsMHVVuaXR0ZXN0aW5nIE9wZXJhdGlvbnMgQ2VudGVyMSowKAYDVQQD +DCFMZW11clRydXN0IFVuaXR0ZXN0cyBSb290IENBIDIwMTgwdjAQBgcqhkjOPQIB +BgUrgQQAIgNiAARuKyHIRp2e6PB5UcY8L/bUdavkL5Zf3IegNKvaAsvkDenhDGAI +zwWgsk3rOo7jmpMibn7yJQn404uZovwyeKcApn8uVv8ltheeYAx+ySzzn/APxNGy +cye/nv1D9cDW628wCgYIKoZIzj0EAwIDaQAwZgIxANl1ljDH4ykNK2OaRqKOkBOW +cKk1SvtiEZDS/wytiZGCeaxYteSYF+3GE8V2W1geWAIxAI8D7DY0HU5zw+oxAlTD +Uw/TeHA6q0QV4otPvrINW3V09iXDwFSPe265fTkHSfT6hQ== +-----END CERTIFICATE----- +""" +ECDSA_SECP384r1_CERT = parse_certificate(ECDSA_SECP384r1_CERT_STR) + +DSA_CERT_STR = """ +-----BEGIN CERTIFICATE----- +MIIDmTCCA1YCCQD5h/cM7xYO9jALBglghkgBZQMEAwIwga8xCzAJBgNVBAYTAlVT +MRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlMb3MgR2F0b3MxIzAhBgNV +BAoMGkxlbXVyVHJ1c3QgRW50ZXJwcmlzZXMgTHRkMSYwJAYDVQQLDB1Vbml0dGVz +dGluZyBPcGVyYXRpb25zIENlbnRlcjEqMCgGA1UEAwwhTGVtdXJUcnVzdCBVbml0 +dGVzdHMgUm9vdCBDQSAyMDE4MB4XDTE5MDIyNjE4MjUyMloXDTI5MDIyMzE4MjUy +Mlowga8xCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQH +DAlMb3MgR2F0b3MxIzAhBgNVBAoMGkxlbXVyVHJ1c3QgRW50ZXJwcmlzZXMgTHRk +MSYwJAYDVQQLDB1Vbml0dGVzdGluZyBPcGVyYXRpb25zIENlbnRlcjEqMCgGA1UE +AwwhTGVtdXJUcnVzdCBVbml0dGVzdHMgUm9vdCBDQSAyMDE4MIIBtjCCASsGByqG +SM44BAEwggEeAoGBAO2+6wO20rn9K7RtXJ7/kCSVFzYZsY1RKvmJ6BBkMFIepBkz +2pk62tRhJgNH07GKF7pyTPRRKqt38CaPK4ERUpavx3Ok6vZ3PKq8tMac/PMKBmT1 +Xfpch54KDlCdreEMJqYiCwbIyiSCR4+PCH+7xC5Uh0PIZo6otNWe3Wkk53CfAhUA +8d4YAtto6D30f7qkEa7DMAccUS8CgYAiv8r0k0aUEaeioblcCAjmhvE0v8/tD5u1 +anHO4jZIIv7uOrNFIGfqcNEOBs5AQkt5Bxn6x0b/VvtZ0FSrD0j4f36pTgro6noG +/0oRt0JngxsMSfo0LV4+bY62v21A0SneNgTgY+ugdfgGWvb0+9tpsIhiY69T+7c8 +Oa0S6OWSPAOBhAACgYB5wa+nJJNZPoTWFum27JlWGYLO2flg5EpWlOvcEE0o5RfB +FPnMM033kKQQEI0YpCAq9fIMKhhUMk1X4mKUBUTt+Nrn1pY2l/wt5G6AQdHI8QXz +P1ecBbHPNZtWe3iVnfOgz/Pd8tU9slcXP9z5XbZ7R/oGcF/TPRTtbLEkYZNaDDAL +BglghkgBZQMEAwIDMAAwLQIVANubSNMSLt8plN9ZV3cp4pe3lMYCAhQPLLE7rTgm +92X+hWfyz000QEpYEQ== +-----END CERTIFICATE----- +""" +DSA_CERT = parse_certificate(DSA_CERT_STR) From 9dbae39604a705544b541370b33e8b164bd48a28 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 26 Feb 2019 16:36:59 -0800 Subject: [PATCH 19/43] updating cryptography API call, to create right signing algorithm object. --- lemur/common/utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 7c9269cf..f5db3d75 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -162,10 +162,10 @@ def check_cert_signature(cert, issuer_public_key): else: padder = padding.PKCS1v15() issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, padder, cert.signature_hash_algorithm) - elif isinstance(issuer_public_key, ec.EllipticCurvePublicKey) and isinstance(cert.signature_hash_algorithm, ec.ECDSA): - issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, cert.signature_hash_algorithm) + elif isinstance(issuer_public_key, ec.EllipticCurvePublicKey) and isinstance(ec.ECDSA(cert.signature_hash_algorithm), ec.ECDSA): + issuer_public_key.verify(cert.signature, cert.tbs_certificate_bytes, ec.ECDSA(cert.signature_hash_algorithm)) else: - raise InvalidSignature + raise UnsupportedAlgorithm("Unsupported Algorithm '{var}'.".format(var=cert.signature_algorithm_oid._name)) def is_selfsigned(cert): @@ -179,8 +179,6 @@ def is_selfsigned(cert): return True except InvalidSignature: return False - except UnsupportedAlgorithm as e: - raise Exception(e) def is_weekend(date): From 658c58e4b63ef50f5e4e4a040f4ce1bab21dab25 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 26 Feb 2019 17:04:43 -0800 Subject: [PATCH 20/43] clarifying comments --- lemur/common/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index f5db3d75..13e6e067 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -147,8 +147,7 @@ def generate_private_key(key_type): def check_cert_signature(cert, issuer_public_key): """ Check a certificate's signature against an issuer public key. - Before EC validation, make sure public key and signature are of the same type, - otherwise verification not possible (raise InvalidSignature) + Before EC validation, make sure we support the algorithm, otherwise raise UnsupportedAlgorithm On success, returns None; on failure, raises UnsupportedAlgorithm or InvalidSignature. """ if isinstance(issuer_public_key, rsa.RSAPublicKey): From 63de8047ce51f2da0ff181035afb330097017355 Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Wed, 27 Feb 2019 09:38:25 +0100 Subject: [PATCH 21/43] Return 'already deleted' instead of 'not found' when cert has already been deleted --- lemur/certificates/views.py | 5 ++++- lemur/tests/test_certificates.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index b464b3ed..e77160b2 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -691,9 +691,12 @@ class Certificates(AuthenticatedResource): cert = service.get(certificate_id) - if not cert or cert.deleted: + if not cert: return dict(message="Cannot find specified certificate"), 404 + if cert.deleted: + return dict(message="Certificate is already deleted"), 412 + # allow creators if g.current_user != cert.user: owner_role = role_service.get_by_name(cert.owner) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 75a29e16..a020ac6b 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -738,7 +738,7 @@ def test_certificate_put_with_data(client, certificate, issuer_plugin): @pytest.mark.parametrize("token,status", [ (VALID_USER_HEADER_TOKEN, 403), (VALID_ADMIN_HEADER_TOKEN, 204), - (VALID_ADMIN_API_TOKEN, 404), + (VALID_ADMIN_API_TOKEN, 412), ('', 401) ]) def test_certificate_delete(client, token, status): From 5d2f603c847771ef0b9bd1651e5c751bc55043f2 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Fri, 1 Mar 2019 09:49:52 -0500 Subject: [PATCH 22/43] renamed vault destination plugin to avoid conflict with vault pki plugin --- lemur/plugins/{lemur_vault => lemur_vault_dest}/__init__.py | 0 lemur/plugins/{lemur_vault => lemur_vault_dest}/plugin.py | 2 +- setup.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename lemur/plugins/{lemur_vault => lemur_vault_dest}/__init__.py (100%) rename lemur/plugins/{lemur_vault => lemur_vault_dest}/plugin.py (98%) diff --git a/lemur/plugins/lemur_vault/__init__.py b/lemur/plugins/lemur_vault_dest/__init__.py similarity index 100% rename from lemur/plugins/lemur_vault/__init__.py rename to lemur/plugins/lemur_vault_dest/__init__.py diff --git a/lemur/plugins/lemur_vault/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py similarity index 98% rename from lemur/plugins/lemur_vault/plugin.py rename to lemur/plugins/lemur_vault_dest/plugin.py index 2e46b155..a11c92ba 100644 --- a/lemur/plugins/lemur_vault/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -1,5 +1,5 @@ """ -.. module: lemur.plugins.lemur_vault.plugin +.. module: lemur.plugins.lemur_vault_dest.plugin :platform: Unix :copyright: (c) 2019 :license: Apache, see LICENCE for more details. diff --git a/setup.py b/setup.py index b5dcdb3b..d22d1f7b 100644 --- a/setup.py +++ b/setup.py @@ -155,7 +155,7 @@ setup( 'digicert_cis_source = lemur.plugins.lemur_digicert.plugin:DigiCertCISSourcePlugin', 'csr_export = lemur.plugins.lemur_csr.plugin:CSRExportPlugin', 'sftp_destination = lemur.plugins.lemur_sftp.plugin:SFTPDestinationPlugin', - 'vault_desination = lemur.plugins.lemur_vault.plugin:VaultDestinationPlugin' + 'vault_desination = lemur.plugins.lemur_vault_dest.plugin:VaultDestinationPlugin' ], }, classifiers=[ From 10cec063c2c8561ecda82b6b238c58ec82072fbe Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Wed, 20 Jun 2018 18:42:34 +0300 Subject: [PATCH 23/43] Check that stored certificate chain matches certificate Similar to how the private key is checked. --- lemur/certificates/models.py | 6 +++- lemur/certificates/schemas.py | 14 ++++++++-- lemur/common/utils.py | 21 ++++++++++++++ lemur/common/validators.py | 48 ++++++++++++++++++++++---------- lemur/tests/factories.py | 1 + lemur/tests/test_certificates.py | 26 ++++++++++++++++- 6 files changed, 96 insertions(+), 20 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 34305cc2..7cc4813c 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -192,12 +192,16 @@ class Certificate(db.Model): def check_integrity(self): """ - Integrity checks: Does the cert have a matching private key? + Integrity checks: Does the cert have a valid chain and matching private key? """ if self.private_key: validators.verify_private_key_match(utils.parse_private_key(self.private_key), self.parsed_cert, error_class=AssertionError) + if self.chain: + chain = [self.parsed_cert] + utils.parse_cert_chain(self.chain) + validators.verify_cert_chain(chain, error_class=AssertionError) + @cached_property def parsed_cert(self): assert self.body, "Certificate body not set" diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 946bd541..d20fd5a7 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -245,8 +245,7 @@ class CertificateUploadInputSchema(CertificateCreationSchema): external_id = fields.String(missing=None, allow_none=True) private_key = fields.String() body = fields.String(required=True) - chain = fields.String(validate=validators.public_certificate, missing=None, - allow_none=True) # TODO this could be multiple certificates + chain = fields.String(missing=None, allow_none=True) destinations = fields.Nested(AssociatedDestinationSchema, missing=[], many=True) notifications = fields.Nested(AssociatedNotificationSchema, missing=[], many=True) @@ -260,7 +259,7 @@ class CertificateUploadInputSchema(CertificateCreationSchema): raise ValidationError('Destinations require private key.') @validates_schema - def validate_cert_private_key(self, data): + def validate_cert_private_key_chain(self, data): cert = None key = None if data.get('body'): @@ -279,6 +278,15 @@ class CertificateUploadInputSchema(CertificateCreationSchema): # Throws ValidationError validators.verify_private_key_match(key, cert) + if data.get('chain'): + try: + chain = utils.parse_cert_chain(data['chain']) + except ValueError: + raise ValidationError("Invalid certificate in certificate chain.", field_names=['chain']) + + # Throws ValidationError + validators.verify_cert_chain([cert] + chain) + class CertificateExportInputSchema(LemurInputSchema): plugin = fields.Nested(PluginInputSchema) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 13e6e067..62c3182b 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -7,6 +7,7 @@ .. moduleauthor:: Kevin Glisson """ import random +import re import string import sqlalchemy @@ -67,6 +68,26 @@ def parse_private_key(private_key): return load_pem_private_key(private_key.encode('utf8'), password=None, backend=default_backend()) +def split_pem(data): + """ + Split a string of several PEM payloads to a list of strings. + + :param data: String + :return: List of strings + """ + return re.split("\n(?=-----BEGIN )", data) + + +def parse_cert_chain(pem_chain): + """ + Helper function to split and parse a series of PEM certificates. + + :param pem_chain: string + :return: List of parsed certificates + """ + return [parse_certificate(cert) for cert in split_pem(pem_chain) if pem_chain] + + def parse_csr(csr): """ Helper function that parses a CSR. diff --git a/lemur/common/validators.py b/lemur/common/validators.py index 90169553..91b831ba 100644 --- a/lemur/common/validators.py +++ b/lemur/common/validators.py @@ -1,27 +1,14 @@ import re from cryptography import x509 +from cryptography.exceptions import UnsupportedAlgorithm, InvalidSignature from cryptography.hazmat.backends import default_backend from cryptography.x509 import NameOID from flask import current_app from marshmallow.exceptions import ValidationError from lemur.auth.permissions import SensitiveDomainPermission -from lemur.common.utils import parse_certificate, is_weekend - - -def public_certificate(body): - """ - Determines if specified string is valid public certificate. - - :param body: - :return: - """ - try: - parse_certificate(body) - except Exception as e: - current_app.logger.exception(e) - raise ValidationError('Public certificate presented is not valid.') +from lemur.common.utils import check_cert_signature, is_weekend def common_name(value): @@ -138,3 +125,34 @@ def verify_private_key_match(key, cert, error_class=ValidationError): """ if key.public_key().public_numbers() != cert.public_key().public_numbers(): raise error_class("Private key does not match certificate.") + + +def verify_cert_chain(certs, error_class=ValidationError): + """ + Verifies that the certificates in the chain are correct. + + We don't bother with full cert validation but just check that certs in the chain are signed by the next, to avoid + basic human errors -- such as pasting the wrong certificate. + + :param certs: List of parsed certificates, use parse_cert_chain() + :param error_class: Exception class to raise on error + """ + cert = certs[0] + for issuer in certs[1:]: + # Use the current cert's public key to verify the previous signature. + # "certificate validation is a complex problem that involves much more than just signature checks" + try: + check_cert_signature(cert, issuer.public_key()) + + except InvalidSignature: + # Avoid circular import. + from lemur.common import defaults + + raise error_class("Incorrect chain certificate(s) provided: '%s' is not signed by '%s'" + % (defaults.common_name(cert) or 'Unknown', defaults.common_name(issuer))) + + except UnsupportedAlgorithm as err: + current_app.logger.warning("Skipping chain validation: %s", err) + + # Next loop will validate that *this issuer* cert is signed by the next chain cert. + cert = issuer diff --git a/lemur/tests/factories.py b/lemur/tests/factories.py index a4af3d43..de78f8a3 100644 --- a/lemur/tests/factories.py +++ b/lemur/tests/factories.py @@ -140,6 +140,7 @@ class CACertificateFactory(CertificateFactory): class InvalidCertificateFactory(CertificateFactory): body = INVALID_CERT_STR private_key = '' + chain = '' class AuthorityFactory(BaseFactory): diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 8247c36b..f94dd713 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -512,7 +512,7 @@ def test_certificate_upload_schema_invalid_chain(client): 'owner': 'pwner@example.com', } data, errors = CertificateUploadInputSchema().load(data) - assert errors == {'chain': ['Public certificate presented is not valid.']} + assert errors == {'chain': ['Invalid certificate in certificate chain.']} def test_certificate_upload_schema_wrong_pkey(client): @@ -527,6 +527,30 @@ def test_certificate_upload_schema_wrong_pkey(client): assert errors == {'_schema': ['Private key does not match certificate.']} +def test_certificate_upload_schema_wrong_chain(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'chain': ROOTCA_CERT_STR, + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'_schema': ["Incorrect chain certificate(s) provided: 'san.example.org' is not signed by " + "'LemurTrust Unittests Root CA 2018'"]} + + +def test_certificate_upload_schema_wrong_chain_2nd(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'chain': INTERMEDIATE_CERT_STR + '\n' + SAN_CERT_STR, + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'_schema': ["Incorrect chain certificate(s) provided: 'LemurTrust Unittests Class 1 CA 2018' is " + "not signed by 'san.example.org'"]} + + def test_create_basic_csr(client): csr_config = dict( common_name='example.com', From dd2900bdbc8ff315c937ca9860eeaf01959b22b2 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 4 Mar 2019 10:04:06 -0800 Subject: [PATCH 24/43] Relax search;update requirements --- lemur/certificates/service.py | 2 +- requirements-dev.txt | 11 ++++++----- requirements-docs.txt | 19 ++++++++++--------- requirements-tests.txt | 16 ++++++++-------- requirements.txt | 19 ++++++++++--------- 5 files changed, 35 insertions(+), 32 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index d5012012..2488115b 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -317,7 +317,7 @@ def render(args): if filt: terms = filt.split(';') - term = '{0}%'.format(terms[1]) + term = '%{0}%'.format(terms[1]) # Exact matches for quotes. Only applies to name, issuer, and cn if terms[1].startswith('"') and terms[1].endswith('"'): term = terms[1][1:-1] diff --git a/requirements-dev.txt b/requirements-dev.txt index 1cfbe393..e67aea64 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,16 +4,17 @@ # # pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index # -aspy.yaml==1.1.2 # via pre-commit +aspy.yaml==1.2.0 # via pre-commit bleach==3.1.0 # via readme-renderer certifi==2018.11.29 # via requests -cfgv==1.4.0 # via pre-commit +cfgv==1.5.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.3.0 # via pre-commit +identify==1.4.0 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 @@ -22,7 +23,7 @@ pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer -pyyaml==5.1b1 +pyyaml==5.1b3 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine @@ -31,6 +32,6 @@ toml==0.10.0 # via pre-commit tqdm==4.31.1 # via twine twine==1.13.0 urllib3==1.24.1 # via requests -virtualenv==16.4.1 # via pre-commit +virtualenv==16.4.3 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index db20a4b9..50a2a077 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -8,8 +8,8 @@ acme==0.31.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.7 -amqp==2.4.1 -aniso8601==4.1.0 +amqp==2.4.2 +aniso8601==5.1.0 arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 @@ -17,16 +17,16 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.102 -botocore==1.12.102 +boto3==1.9.106 +botocore==1.12.106 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 -cffi==1.12.1 +cffi==1.12.2 chardet==3.0.4 click==7.0 cloudflare==2.1.0 -cryptography==2.5 +cryptography==2.6.1 dnspython3==1.15.0 dnspython==1.15.0 docutils==0.14 @@ -50,7 +50,7 @@ jinja2==2.10 jmespath==0.9.4 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.3.0 +kombu==4.4.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.1 @@ -60,7 +60,7 @@ mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 -pbr==5.1.2 +pbr==5.1.3 pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 @@ -75,9 +75,10 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 pytz==2018.9 -pyyaml==5.1b1 +pyyaml==5.1b3 raven[flask]==6.10.0 redis==2.10.6 +relativetimebuilder==0.2.0 requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 diff --git a/requirements-tests.txt b/requirements-tests.txt index d1d6ae7f..84c59d0d 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,17 +6,17 @@ # asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest -attrs==18.2.0 # via pytest +attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.102 # via moto +boto3==1.9.106 # via moto boto==2.49.0 # via moto -botocore==1.12.102 # via boto3, moto, s3transfer +botocore==1.12.106 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests -cffi==1.12.1 # via cryptography +cffi==1.12.2 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 -cryptography==2.5 # via moto +cryptography==2.6.1 # via moto docker-pycreds==0.4.0 # via docker docker==3.7.0 # via moto docutils==0.14 # via botocore @@ -37,20 +37,20 @@ mock==2.0.0 # via moto more-itertools==6.0.0 # via pytest moto==1.3.7 nose==1.3.7 -pbr==5.1.2 # via mock +pbr==5.1.3 # via mock pluggy==0.9.0 # via pytest py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi pycryptodome==3.7.3 # via python-jose -pyflakes==2.1.0 +pyflakes==2.1.1 pytest-flask==0.14.0 pytest-mock==1.10.1 pytest==4.3.0 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto -pyyaml==5.1b1 +pyyaml==5.1b3 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.10.5 # via moto diff --git a/requirements.txt b/requirements.txt index 72a37692..dd442b5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,24 +7,24 @@ acme==0.31.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.7 # via flask-migrate -amqp==2.4.1 # via kombu -aniso8601==4.1.0 # via flask-restful +amqp==2.4.2 # via kombu +aniso8601==5.1.0 # via flask-restful, relativetimebuilder arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.102 -botocore==1.12.102 +boto3==1.9.106 +botocore==1.12.106 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 -cffi==1.12.1 # via bcrypt, cryptography, pynacl +cffi==1.12.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 -cryptography==2.5 +cryptography==2.6.1 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore @@ -47,7 +47,7 @@ jinja2==2.10 jmespath==0.9.4 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.3.0 # via celery +kombu==4.4.0 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.1 # via jinja2, mako @@ -56,7 +56,7 @@ marshmallow==2.18.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.1.2 # via mock +pbr==5.1.3 # via mock pem==18.2.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 # via python-ldap @@ -70,9 +70,10 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-ldap==3.1.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1b1 +pyyaml==5.1b3 raven[flask]==6.10.0 redis==2.10.6 +relativetimebuilder==0.2.0 # via aniso8601 requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 From 4a027797e057d28900049c16190c4659d5bb48a5 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 5 Mar 2019 07:19:22 -0500 Subject: [PATCH 25/43] fixing linting issues --- lemur/plugins/lemur_vault_dest/plugin.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index a11c92ba..92089b02 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -10,8 +10,6 @@ .. moduleauthor:: Christopher Jolley """ import hvac - -#import lemur_vault from flask import current_app from lemur.common.defaults import common_name @@ -21,7 +19,6 @@ from lemur.plugins.bases import DestinationPlugin from cryptography import x509 from cryptography.hazmat.backends import default_backend - class VaultDestinationPlugin(DestinationPlugin): """Hashicorp Vault Destination plugin for Lemur""" title = 'Vault' @@ -79,7 +76,7 @@ class VaultDestinationPlugin(DestinationPlugin): :return: """ cname = common_name(parse_certificate(body)) - secret = {'data':{}} + secret = {'data': {}} key_name = '{0}.key'.format(cname) cert_name = '{0}.crt'.format(cname) chain_name = '{0}.chain'.format(cname) @@ -100,7 +97,6 @@ class VaultDestinationPlugin(DestinationPlugin): path = '{0}/{1}'.format(path, cname) secret = get_secret(url, token, mount, path) - if bundle == 'Nginx' and cert_chain: secret['data'][cert_name] = '{0}\n{1}'.format(body, cert_chain) @@ -120,6 +116,7 @@ class VaultDestinationPlugin(DestinationPlugin): current_app.logger.exception( "Exception uploading secret to vault: {0}".format(err), exc_info=True) + def get_san_list(body): """ parse certificate for SAN names and return list, return empty list on error """ try: @@ -127,15 +124,16 @@ def get_san_list(body): cert = x509.load_pem_x509_certificate(byte_body, default_backend()) ext = cert.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) return ext.value.get_values_for_type(x509.DNSName) - except: + except ValueError: pass return [] + def get_secret(url, token, mount, path): result = {'data': {}} try: client = hvac.Client(url=url, token=token) result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) - except: + except ConnectionError: pass return result From a1cb8ee266af23aa0ba7171f5cf9d40750b0220b Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 5 Mar 2019 07:37:04 -0500 Subject: [PATCH 26/43] fixing lint --- lemur/plugins/lemur_vault_dest/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 92089b02..774b6bb1 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -19,6 +19,7 @@ from lemur.plugins.bases import DestinationPlugin from cryptography import x509 from cryptography.hazmat.backends import default_backend + class VaultDestinationPlugin(DestinationPlugin): """Hashicorp Vault Destination plugin for Lemur""" title = 'Vault' From 20ac4bd3dd380b311c3b6f570c55a939e1d1399a Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 5 Mar 2019 07:34:30 -0800 Subject: [PATCH 27/43] downgrade kombu --- requirements-docs.txt | 8 ++++---- requirements-tests.txt | 4 ++-- requirements.txt | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 50a2a077..894defcb 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -7,7 +7,7 @@ acme==0.31.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.7 +alembic==1.0.8 amqp==2.4.2 aniso8601==5.1.0 arrow==0.13.1 @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.106 -botocore==1.12.106 +boto3==1.9.107 +botocore==1.12.107 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 @@ -90,7 +90,7 @@ sphinx==1.8.4 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.18 +sqlalchemy==1.3.0 tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 84c59d0d..55e38cbf 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.106 # via moto +boto3==1.9.107 # via moto boto==2.49.0 # via moto -botocore==1.12.106 # via boto3, moto, s3transfer +botocore==1.12.107 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.12.2 # via cryptography chardet==3.0.4 # via requests diff --git a/requirements.txt b/requirements.txt index dd442b5f..cf2be225 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # acme==0.31.0 alembic-autogenerate-enums==0.0.2 -alembic==1.0.7 # via flask-migrate +alembic==1.0.8 # via flask-migrate amqp==2.4.2 # via kombu aniso8601==5.1.0 # via flask-restful, relativetimebuilder arrow==0.13.1 @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.106 -botocore==1.12.106 +boto3==1.9.107 +botocore==1.12.107 celery[redis]==4.2.1 certifi==2018.11.29 certsrv==2.1.1 @@ -80,7 +80,7 @@ retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.18 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.3.0 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp From 077ae1eedd7b827b809bb2908a20491342a331a0 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 5 Mar 2019 09:45:59 -0800 Subject: [PATCH 28/43] Downgrade Kombu for real this time --- requirements-docs.txt | 2 +- requirements.in | 1 + requirements.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 894defcb..e936c197 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -50,7 +50,7 @@ jinja2==2.10 jmespath==0.9.4 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.4.0 +kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.1 diff --git a/requirements.in b/requirements.in index b085f5c7..e3d0c66b 100644 --- a/requirements.in +++ b/requirements.in @@ -26,6 +26,7 @@ future gunicorn inflection jinja2 +kombu==4.3.0 # kombu 4.4.0 requires redis 3 lockfile marshmallow-sqlalchemy marshmallow diff --git a/requirements.txt b/requirements.txt index cf2be225..2aa5f157 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ jinja2==2.10 jmespath==0.9.4 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.4.0 # via celery +kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.1 # via jinja2, mako From cc6d53fdeb4ff3cd24dd96ac2caa0b292f101208 Mon Sep 17 00:00:00 2001 From: Kevin Glisson Date: Tue, 5 Mar 2019 15:39:37 -0800 Subject: [PATCH 29/43] Ensuring that configs passed via the command line are respected. --- lemur/__init__.py | 4 ++-- lemur/manage.py | 4 ++-- lemur/tests/conftest.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/__init__.py b/lemur/__init__.py index 1cdb3468..769e0cec 100644 --- a/lemur/__init__.py +++ b/lemur/__init__.py @@ -62,8 +62,8 @@ LEMUR_BLUEPRINTS = ( ) -def create_app(config=None): - app = factory.create_app(app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config) +def create_app(config_path=None): + app = factory.create_app(app_name=__name__, blueprints=LEMUR_BLUEPRINTS, config=config_path) configure_hook(app) return app diff --git a/lemur/manage.py b/lemur/manage.py index 184b9aa6..9161109b 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -50,7 +50,7 @@ from lemur.pending_certificates.models import PendingCertificate # noqa from lemur.dns_providers.models import DnsProvider # noqa manager = Manager(create_app) -manager.add_option('-c', '--config', dest='config') +manager.add_option('-c', '--config', dest='config_path', required=False) migrate = Migrate(create_app) @@ -391,7 +391,7 @@ class LemurServer(Command): # run startup tasks on an app like object validate_conf(current_app, REQUIRED_VARIABLES) - app.app_uri = 'lemur:create_app(config="{0}")'.format(current_app.config.get('CONFIG_PATH')) + app.app_uri = 'lemur:create_app(config_path="{0}")'.format(current_app.config.get('CONFIG_PATH')) return app.run() diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index b3dad8b2..43fa7163 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -43,7 +43,7 @@ def app(request): Creates a new Flask application for a test duration. Uses application factory `create_app`. """ - _app = create_app(os.path.dirname(os.path.realpath(__file__)) + '/conf.py') + _app = create_app(config_path=os.path.dirname(os.path.realpath(__file__)) + '/conf.py') ctx = _app.app_context() ctx.push() From b8d3a4f9aac35ee82ed958433742701f965ca190 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 6 Mar 2019 11:13:34 -0800 Subject: [PATCH 30/43] Update requirements.in --- requirements.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index bd408f1c..9b27f604 100644 --- a/requirements.in +++ b/requirements.in @@ -46,4 +46,4 @@ six SQLAlchemy-Utils tabulate xmltodict -pyyaml>=4.2b1 #high severity alert \ No newline at end of file +pyyaml>=4.2b1 #high severity alert From 752c9a086bd1aff975df3622c0b06422368170e4 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Thu, 7 Mar 2019 15:41:29 -0500 Subject: [PATCH 31/43] fixing error handling and better data formating --- lemur/plugins/lemur_vault_dest/plugin.py | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 774b6bb1..5924f387 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -34,7 +34,7 @@ class VaultDestinationPlugin(DestinationPlugin): 'name': 'vaultMount', 'type': 'str', 'required': True, - 'validation': '^[a-zA-Z0-9]+$', + 'validation': '^\S+$', 'helpMessage': 'Must be a valid Vault secrets mount name!' }, { @@ -77,11 +77,6 @@ class VaultDestinationPlugin(DestinationPlugin): :return: """ cname = common_name(parse_certificate(body)) - secret = {'data': {}} - key_name = '{0}.key'.format(cname) - cert_name = '{0}.crt'.format(cname) - chain_name = '{0}.chain'.format(cname) - sans_name = '{0}.san'.format(cname) token = current_app.config.get('VAULT_TOKEN') url = current_app.config.get('VAULT_URL') @@ -98,18 +93,19 @@ class VaultDestinationPlugin(DestinationPlugin): path = '{0}/{1}'.format(path, cname) secret = get_secret(url, token, mount, path) + secret['data'][cname] = {} if bundle == 'Nginx' and cert_chain: - secret['data'][cert_name] = '{0}\n{1}'.format(body, cert_chain) + secret['data'][cname]['crt'] = '{0}\n{1}'.format(body, cert_chain) elif bundle == 'Apache' and cert_chain: - secret['data'][cert_name] = body - secret['data'][chain_name] = cert_chain + secret['data'][cname]['crt'] = body + secret['data'][cname]['chain'] = cert_chain else: - secret['data'][cert_name] = body - secret['data'][key_name] = private_key + secret['data'][cname]['crt'] = body + secret['data'][cname]['key'] = private_key san_list = get_san_list(body) if isinstance(san_list, list): - secret['data'][sans_name] = san_list + secret['data'][cname]['san'] = san_list try: client.secrets.kv.v1.create_or_update_secret( path=path, mount_point=mount, secret=secret['data']) @@ -120,21 +116,25 @@ class VaultDestinationPlugin(DestinationPlugin): def get_san_list(body): """ parse certificate for SAN names and return list, return empty list on error """ + san_list = [] try: byte_body = body.encode('utf-8') cert = x509.load_pem_x509_certificate(byte_body, default_backend()) ext = cert.extensions.get_extension_for_oid(x509.oid.ExtensionOID.SUBJECT_ALTERNATIVE_NAME) - return ext.value.get_values_for_type(x509.DNSName) - except ValueError: + san_list = ext.value.get_values_for_type(x509.DNSName) + except x509.extensions.ExtensionNotFound: pass - return [] + finally: + return san_list def get_secret(url, token, mount, path): + """ retreiive existing data from mount path and return dictionary """ result = {'data': {}} try: client = hvac.Client(url=url, token=token) result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) - except ConnectionError: - pass - return result + #except ConnectionError: + # pass + finally: + return result From f1c09a6f8f8f7c66d66b0d2c85cfa4420b200d00 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Thu, 7 Mar 2019 15:58:34 -0500 Subject: [PATCH 32/43] fixed comments --- lemur/plugins/lemur_vault_dest/plugin.py | 4 ++-- lemur/plugins/lemur_vault_dest/tests/conftest.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 lemur/plugins/lemur_vault_dest/tests/conftest.py diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 5924f387..2f2a2e82 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -134,7 +134,7 @@ def get_secret(url, token, mount, path): try: client = hvac.Client(url=url, token=token) result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) - #except ConnectionError: - # pass + except ConnectionError: + pass finally: return result diff --git a/lemur/plugins/lemur_vault_dest/tests/conftest.py b/lemur/plugins/lemur_vault_dest/tests/conftest.py new file mode 100644 index 00000000..0e1cd89f --- /dev/null +++ b/lemur/plugins/lemur_vault_dest/tests/conftest.py @@ -0,0 +1 @@ +from lemur.tests.conftest import * # noqa From d220e9326c0c2a66106a9f3ec74832091c3c7ff2 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 12 Mar 2019 14:45:43 -0700 Subject: [PATCH 33/43] Skip a task if similar task already active --- lemur/common/celery.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f2a2f826..56837cba 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -47,6 +47,19 @@ def make_celery(app): celery = make_celery(flask_app) +def is_task_active(fun: str, task_id: str, args: str) -> bool: + from celery.task.control import inspect + i = inspect() + active_tasks: dict = i.active() + for _, tasks in active_tasks.items(): + for task in tasks: + if task.get("id") == task_id: + continue + if task.get("name") == fun and task.get("args") == str(args): + return True + return False + + @celery.task() def fetch_acme_cert(id): """ @@ -224,5 +237,21 @@ def sync_source(source): :param source: :return: """ - current_app.logger.debug("Syncing source {}".format(source)) + + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = celery.current_task.request.id + log_data = { + "function": function, + "message": "Syncing source", + "source": source, + "task_id": task_id, + } + current_app.logger.debug(log_data) + + if is_task_active(function, task_id, (source,)): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return sync([source]) + log_data["message"] = "Done syncing source" + current_app.logger.debug(log_data) From 1a5a91ccc72ec869ed1ea5f940f33ea92d77c50e Mon Sep 17 00:00:00 2001 From: Curtis Date: Tue, 12 Mar 2019 15:11:13 -0700 Subject: [PATCH 34/43] Update celery.py --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 56837cba..b7f23c32 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -50,7 +50,7 @@ celery = make_celery(flask_app) def is_task_active(fun: str, task_id: str, args: str) -> bool: from celery.task.control import inspect i = inspect() - active_tasks: dict = i.active() + active_tasks = i.active() for _, tasks in active_tasks.items(): for task in tasks: if task.get("id") == task_id: From f38e5b0879225255fb9ba514cb1d81f86fdb8d3c Mon Sep 17 00:00:00 2001 From: Curtis Date: Tue, 12 Mar 2019 15:29:04 -0700 Subject: [PATCH 35/43] Update celery.py --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b7f23c32..90b6f9a2 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -47,7 +47,7 @@ def make_celery(app): celery = make_celery(flask_app) -def is_task_active(fun: str, task_id: str, args: str) -> bool: +def is_task_active(fun, task_id, args): from celery.task.control import inspect i = inspect() active_tasks = i.active() From c445297357e3ab47116beed3d9fe0a4b50582595 Mon Sep 17 00:00:00 2001 From: Curtis Date: Tue, 12 Mar 2019 15:41:24 -0700 Subject: [PATCH 36/43] Update celery.py --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 90b6f9a2..991dac2c 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -238,7 +238,7 @@ def sync_source(source): :return: """ - function = f"{__name__}.{sys._getframe().f_code.co_name}" + function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) task_id = celery.current_task.request.id log_data = { "function": function, From f7452e837974dd39bf7452a8ee059ec7738f18ef Mon Sep 17 00:00:00 2001 From: Javier Ramos Date: Fri, 15 Mar 2019 09:18:33 +0100 Subject: [PATCH 37/43] Parse DNSNames from CSR into Lemur Certificate --- lemur/certificates/schemas.py | 6 +++++ lemur/certificates/utils.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 lemur/certificates/utils.py diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index d20fd5a7..e9b61539 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -10,6 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError from lemur.authorities.schemas import AuthorityNestedOutputSchema +from lemur.certificates import utils as cert_utils from lemur.common import missing, utils, validators from lemur.common.fields import ArrowDateTime, Hex from lemur.common.schema import LemurInputSchema, LemurOutputSchema @@ -107,6 +108,11 @@ class CertificateInputSchema(CertificateCreationSchema): def load_data(self, data): if data.get('replacements'): data['replaces'] = data['replacements'] # TODO remove when field is deprecated + if data['csr']: + dns_names = cert_utils.get_dns_names_from_csr(data['csr']) + if not data['extensions']['subAltNames']['names']: + data['extensions']['subAltNames']['names'] = [] + data['extensions']['subAltNames']['names'] += dns_names return missing.convert_validity_years(data) diff --git a/lemur/certificates/utils.py b/lemur/certificates/utils.py new file mode 100644 index 00000000..933fe45e --- /dev/null +++ b/lemur/certificates/utils.py @@ -0,0 +1,42 @@ +""" +Utils to parse certificate data. + +.. module: lemur.certificates.hooks + :platform: Unix + :copyright: (c) 2019 by Javier Ramos, see AUTHORS for more + :license: Apache, see LICENSE for more details. + +.. moduleauthor:: Javier Ramos +""" + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from marshmallow.exceptions import ValidationError + + +def get_dns_names_from_csr(data): + """ + Fetches DNSNames from CSR. + Potentially extendable to any kind of SubjectAlternativeName + :param data: PEM-encoded string with CSR + :return: + """ + dns_names = [] + try: + request = x509.load_pem_x509_csr(data.encode('utf-8'), default_backend()) + except Exception: + raise ValidationError('CSR presented is not valid.') + + try: + alt_names = request.extensions.get_extension_for_class(x509.SubjectAlternativeName) + + for name in alt_names.value.get_values_for_type(x509.DNSName): + dns_name = { + 'nameType': 'DNSName', + 'value': name + } + dns_names.append(dns_name) + except x509.ExtensionNotFound: + pass + + return dns_names From 9e5496b484fd5dcc87120938beabaf7b28031dcf Mon Sep 17 00:00:00 2001 From: Javier Ramos Date: Fri, 15 Mar 2019 10:19:25 +0100 Subject: [PATCH 38/43] Update schemas.py --- lemur/certificates/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index e9b61539..25fc2c46 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -108,7 +108,7 @@ class CertificateInputSchema(CertificateCreationSchema): def load_data(self, data): if data.get('replacements'): data['replaces'] = data['replacements'] # TODO remove when field is deprecated - if data['csr']: + if data.get('csr'): dns_names = cert_utils.get_dns_names_from_csr(data['csr']) if not data['extensions']['subAltNames']['names']: data['extensions']['subAltNames']['names'] = [] From dbd948be6eda9ce70731dee908f71de5a15910dc Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 18 Mar 2019 12:50:18 -0700 Subject: [PATCH 39/43] updating requirements --- requirements-dev.txt | 5 ++--- requirements-docs.txt | 21 ++++++++++----------- requirements-tests.txt | 16 ++++++++-------- requirements.txt | 21 ++++++++++----------- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e67aea64..36e2c9a4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # aspy.yaml==1.2.0 # via pre-commit bleach==3.1.0 # via readme-renderer -certifi==2018.11.29 # via requests +certifi==2019.3.9 # via requests cfgv==1.5.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer @@ -14,7 +14,6 @@ flake8==3.5.0 identify==1.4.0 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit -importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 @@ -23,7 +22,7 @@ pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer -pyyaml==5.1b3 +pyyaml==5.1 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine diff --git a/requirements-docs.txt b/requirements-docs.txt index e936c197..7879c667 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,12 +4,12 @@ # # pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # -acme==0.31.0 +acme==0.32.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.8 amqp==2.4.2 -aniso8601==5.1.0 +aniso8601==6.0.0 arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 @@ -17,10 +17,10 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.107 -botocore==1.12.107 +boto3==1.9.116 +botocore==1.12.116 celery[redis]==4.2.1 -certifi==2018.11.29 +certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 chardet==3.0.4 @@ -54,8 +54,8 @@ kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.1 +marshmallow-sqlalchemy==0.16.1 +marshmallow==2.19.1 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx @@ -75,10 +75,9 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 pytz==2018.9 -pyyaml==5.1b3 +pyyaml==5.1 raven[flask]==6.10.0 redis==2.10.6 -relativetimebuilder==0.2.0 requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 @@ -86,11 +85,11 @@ s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==1.8.4 +sphinx==1.8.5 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.0 +sqlalchemy==1.3.1 tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 55e38cbf..da3b4482 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,21 +8,21 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.107 # via moto +boto3==1.9.116 # via moto boto==2.49.0 # via moto -botocore==1.12.107 # via boto3, moto, s3transfer -certifi==2018.11.29 # via requests +botocore==1.12.116 # via boto3, moto, s3transfer +certifi==2019.3.9 # via requests cffi==1.12.2 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask -coverage==4.5.2 +coverage==4.5.3 cryptography==2.6.1 # via moto docker-pycreds==0.4.0 # via docker docker==3.7.0 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==1.0.2 +faker==1.0.4 flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.17.1 # via python-jose @@ -46,14 +46,14 @@ pycryptodome==3.7.3 # via python-jose pyflakes==2.1.1 pytest-flask==0.14.0 pytest-mock==1.10.1 -pytest==4.3.0 +pytest==4.3.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto -pyyaml==5.1b3 +pyyaml==5.1 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.5 # via moto +responses==0.10.6 # via moto s3transfer==0.2.0 # via boto3 six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker diff --git a/requirements.txt b/requirements.txt index 2aa5f157..5bd37693 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,21 +4,21 @@ # # pip-compile --output-file requirements.txt requirements.in -U --no-index # -acme==0.31.0 +acme==0.32.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.8 # via flask-migrate amqp==2.4.2 # via kombu -aniso8601==5.1.0 # via flask-restful, relativetimebuilder +aniso8601==6.0.0 # via flask-restful arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.107 -botocore==1.12.107 +boto3==1.9.116 +botocore==1.12.116 celery[redis]==4.2.1 -certifi==2018.11.29 +certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -51,8 +51,8 @@ kombu==4.3.0 lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.1 +marshmallow-sqlalchemy==0.16.1 +marshmallow==2.19.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 @@ -68,19 +68,18 @@ pyopenssl==19.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic -python-ldap==3.1.0 +python-ldap==3.2.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1b3 +pyyaml==5.1 raven[flask]==6.10.0 redis==2.10.6 -relativetimebuilder==0.2.0 # via aniso8601 requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.0 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp From f99b11d50ec91b5e344eeb1497fd60b96c1af107 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Wed, 20 Mar 2019 13:51:06 -0400 Subject: [PATCH 40/43] refactor url and token to support muiltiple instances of vault --- lemur/plugins/lemur_vault_dest/plugin.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 2f2a2e82..c47b49a3 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -30,6 +30,22 @@ class VaultDestinationPlugin(DestinationPlugin): author_url = 'https://github.com/alwaysjolley/lemur' options = [ + { + 'name': 'vaultUrl', + 'type': 'str', + 'required': True, + 'validation': '^https?://[a-zA-Z0-9.:-]+$', + 'helpMessage': 'Valid URL to Hashi Vault instance' + 'default': 'http://127.0.0.1:8200' + }, + { + 'name': 'vaultAuthTokenFile', + 'type': 'str', + 'required': True, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/etc/pki/secrets/vault/token' + }, { 'name': 'vaultMount', 'type': 'str', @@ -79,13 +95,17 @@ class VaultDestinationPlugin(DestinationPlugin): cname = common_name(parse_certificate(body)) token = current_app.config.get('VAULT_TOKEN') - url = current_app.config.get('VAULT_URL') - + #url = current_app.config.get('VAULT_URL') + url = self.get_option('vaultUrl', options) + token_file = self.get_option('vaultFile', options) mount = self.get_option('vaultMount', options) path = self.get_option('vaultPath', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) + with open(token_file, 'r') as file: + token = file.readline() + client = hvac.Client(url=url, token=token) if obj_name: path = '{0}/{1}'.format(path, obj_name) From fa4a5122bc7c723e5b8e9a8396f05a63577399db Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Wed, 20 Mar 2019 14:59:04 -0400 Subject: [PATCH 41/43] fixing file read to trim line endings and cleanup --- lemur/plugins/lemur_vault_dest/plugin.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index c47b49a3..91f6a07a 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -36,15 +36,13 @@ class VaultDestinationPlugin(DestinationPlugin): 'required': True, 'validation': '^https?://[a-zA-Z0-9.:-]+$', 'helpMessage': 'Valid URL to Hashi Vault instance' - 'default': 'http://127.0.0.1:8200' }, { 'name': 'vaultAuthTokenFile', 'type': 'str', 'required': True, 'validation': '(/[^/]+)+', - 'helpMessage': 'Must be a valid file path!', - 'default': '/etc/pki/secrets/vault/token' + 'helpMessage': 'Must be a valid file path!' }, { 'name': 'vaultMount', @@ -94,17 +92,15 @@ class VaultDestinationPlugin(DestinationPlugin): """ cname = common_name(parse_certificate(body)) - token = current_app.config.get('VAULT_TOKEN') - #url = current_app.config.get('VAULT_URL') url = self.get_option('vaultUrl', options) - token_file = self.get_option('vaultFile', options) + token_file = self.get_option('vaultAuthTokenFile', options) mount = self.get_option('vaultMount', options) path = self.get_option('vaultPath', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) with open(token_file, 'r') as file: - token = file.readline() + token = file.readline().rstrip('\n') client = hvac.Client(url=url, token=token) if obj_name: From c2158ff8fb284062afb70a2fef40fbbbc94092d9 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 25 Mar 2019 08:28:23 -0700 Subject: [PATCH 42/43] Add order URI during LE cert creation failure; Fail properly when invalid CA passed; Update reqs --- lemur/certificates/schemas.py | 3 +++ lemur/plugins/lemur_acme/plugin.py | 5 ++++- requirements-dev.txt | 4 ++-- requirements-docs.txt | 31 +++++++++++++++--------------- requirements-tests.txt | 26 ++++++++++++------------- requirements.txt | 31 +++++++++++++++--------------- 6 files changed, 52 insertions(+), 48 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index d20fd5a7..f790d92f 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -96,6 +96,9 @@ class CertificateInputSchema(CertificateCreationSchema): @validates_schema def validate_authority(self, data): + if isinstance(data['authority'], str): + raise ValidationError("Authority not found.") + if not data['authority'].active: raise ValidationError("The authority is inactive.", ['authority']) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 66295ed2..59cde380 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -459,7 +459,10 @@ class ACMEIssuerPlugin(IssuerPlugin): "pending_cert": entry["pending_cert"], }) except (PollError, AcmeError, Exception) as e: - current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) + order_url = order.uri + current_app.logger.error( + "Unable to resolve pending cert: {}. " + "Check out {} for more information.".format(pending_cert, order_url), exc_info=True) certs.append({ "cert": False, "pending_cert": entry["pending_cert"], diff --git a/requirements-dev.txt b/requirements-dev.txt index e67aea64..37202d97 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # aspy.yaml==1.2.0 # via pre-commit bleach==3.1.0 # via readme-renderer -certifi==2018.11.29 # via requests +certifi==2019.3.9 # via requests cfgv==1.5.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer @@ -23,7 +23,7 @@ pre-commit==1.14.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer -pyyaml==5.1b3 +pyyaml==5.1 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.21.0 # via requests-toolbelt, twine diff --git a/requirements-docs.txt b/requirements-docs.txt index e936c197..40cd73de 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,12 +4,12 @@ # # pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index # -acme==0.31.0 +acme==0.32.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.8 amqp==2.4.2 -aniso8601==5.1.0 +aniso8601==6.0.0 arrow==0.13.1 asn1crypto==0.24.0 asyncpool==1.0 @@ -17,10 +17,10 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.107 -botocore==1.12.107 -celery[redis]==4.2.1 -certifi==2018.11.29 +boto3==1.9.120 +botocore==1.12.120 +celery[redis]==4.2.2 +certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 chardet==3.0.4 @@ -52,16 +52,16 @@ josepy==1.1.0 jsonlines==1.2.0 kombu==4.3.0 lockfile==0.12.2 -mako==1.0.7 +mako==1.0.8 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.1 +marshmallow-sqlalchemy==0.16.1 +marshmallow==2.19.1 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx paramiko==2.4.2 pbr==5.1.3 -pem==18.2.0 +pem==19.1.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 pyasn1==0.4.5 @@ -75,10 +75,9 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 pytz==2018.9 -pyyaml==5.1b3 +pyyaml==5.1 raven[flask]==6.10.0 redis==2.10.6 -relativetimebuilder==0.2.0 requests-toolbelt==0.9.1 requests[security]==2.21.0 retrying==1.3.3 @@ -86,13 +85,13 @@ s3transfer==0.2.0 six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==1.8.4 +sphinx==1.8.5 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.0 +sqlalchemy==1.3.1 tabulate==0.8.3 urllib3==1.24.1 -vine==1.2.0 -werkzeug==0.14.1 +vine==1.3.0 +werkzeug==0.15.1 xmltodict==0.12.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 55e38cbf..ed48cfdd 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,21 +8,21 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest attrs==19.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.107 # via moto +boto3==1.9.120 # via moto boto==2.49.0 # via moto -botocore==1.12.107 # via boto3, moto, s3transfer -certifi==2018.11.29 # via requests +botocore==1.12.120 # via boto3, moto, s3transfer +certifi==2019.3.9 # via requests cffi==1.12.2 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask -coverage==4.5.2 +coverage==4.5.3 cryptography==2.6.1 # via moto docker-pycreds==0.4.0 # via docker -docker==3.7.0 # via moto +docker==3.7.1 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==1.0.2 +faker==1.0.4 flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.17.1 # via python-jose @@ -42,23 +42,23 @@ pluggy==0.9.0 # via pytest py==1.8.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi -pycryptodome==3.7.3 # via python-jose +pycryptodome==3.8.0 # via python-jose pyflakes==2.1.1 pytest-flask==0.14.0 -pytest-mock==1.10.1 -pytest==4.3.0 +pytest-mock==1.10.2 +pytest==4.3.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto -pyyaml==5.1b3 +pyyaml==5.1 requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.5 # via moto +responses==0.10.6 # via moto s3transfer==0.2.0 # via boto3 six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests -websocket-client==0.55.0 # via docker -werkzeug==0.14.1 # via flask, moto, pytest-flask +websocket-client==0.56.0 # via docker +werkzeug==0.15.1 # via flask, moto, pytest-flask wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto diff --git a/requirements.txt b/requirements.txt index 2aa5f157..9adbdf37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,21 +4,21 @@ # # pip-compile --output-file requirements.txt requirements.in -U --no-index # -acme==0.31.0 +acme==0.32.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.8 # via flask-migrate amqp==2.4.2 # via kombu -aniso8601==5.1.0 # via flask-restful, relativetimebuilder +aniso8601==6.0.0 # via flask-restful arrow==0.13.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.107 -botocore==1.12.107 -celery[redis]==4.2.1 -certifi==2018.11.29 +boto3==1.9.120 +botocore==1.12.120 +celery[redis]==4.2.2 +certifi==2019.3.9 certsrv==2.1.1 cffi==1.12.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -49,15 +49,15 @@ josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.3.0 lockfile==0.12.2 -mako==1.0.7 # via alembic +mako==1.0.8 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.16.0 -marshmallow==2.18.1 +marshmallow-sqlalchemy==0.16.1 +marshmallow==2.19.1 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.3 # via mock -pem==18.2.0 +pem==19.1.0 psycopg2==2.7.7 pyasn1-modules==0.2.4 # via python-ldap pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap @@ -68,21 +68,20 @@ pyopenssl==19.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic -python-ldap==3.1.0 +python-ldap==3.2.0 pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1b3 +pyyaml==5.1 raven[flask]==6.10.0 redis==2.10.6 -relativetimebuilder==0.2.0 # via aniso8601 requests-toolbelt==0.9.1 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.2.0 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.0 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.3.1 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests -vine==1.2.0 # via amqp -werkzeug==0.14.1 # via flask +vine==1.3.0 # via amqp +werkzeug==0.15.1 # via flask xmltodict==0.12.0 From e10007ef7b135eff08dea58b6b542dbb36ec72e3 Mon Sep 17 00:00:00 2001 From: Ryan DeShone Date: Fri, 29 Mar 2019 10:32:49 -0400 Subject: [PATCH 43/43] Add support for Vault KV API v2 This adds the ability to target KV API v1 or v2. --- lemur/plugins/lemur_vault_dest/plugin.py | 29 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_vault_dest/plugin.py b/lemur/plugins/lemur_vault_dest/plugin.py index 91f6a07a..6868b7b0 100644 --- a/lemur/plugins/lemur_vault_dest/plugin.py +++ b/lemur/plugins/lemur_vault_dest/plugin.py @@ -37,6 +37,17 @@ class VaultDestinationPlugin(DestinationPlugin): 'validation': '^https?://[a-zA-Z0-9.:-]+$', 'helpMessage': 'Valid URL to Hashi Vault instance' }, + { + 'name': 'vaultKvApiVersion', + 'type': 'select', + 'value': '2', + 'available': [ + '1', + '2' + ], + 'required': True, + 'helpMessage': 'Version of the Vault KV API to use' + }, { 'name': 'vaultAuthTokenFile', 'type': 'str', @@ -98,17 +109,20 @@ class VaultDestinationPlugin(DestinationPlugin): path = self.get_option('vaultPath', options) bundle = self.get_option('bundleChain', options) obj_name = self.get_option('objectName', options) + api_version = self.get_option('vaultKvApiVersion', options) with open(token_file, 'r') as file: token = file.readline().rstrip('\n') client = hvac.Client(url=url, token=token) + client.secrets.kv.default_kv_version = api_version + if obj_name: path = '{0}/{1}'.format(path, obj_name) else: path = '{0}/{1}'.format(path, cname) - secret = get_secret(url, token, mount, path) + secret = get_secret(client, mount, path) secret['data'][cname] = {} if bundle == 'Nginx' and cert_chain: @@ -123,8 +137,9 @@ class VaultDestinationPlugin(DestinationPlugin): if isinstance(san_list, list): secret['data'][cname]['san'] = san_list try: - client.secrets.kv.v1.create_or_update_secret( - path=path, mount_point=mount, secret=secret['data']) + client.secrets.kv.create_or_update_secret( + path=path, mount_point=mount, secret=secret['data'] + ) except ConnectionError as err: current_app.logger.exception( "Exception uploading secret to vault: {0}".format(err), exc_info=True) @@ -144,12 +159,14 @@ def get_san_list(body): return san_list -def get_secret(url, token, mount, path): +def get_secret(client, mount, path): """ retreiive existing data from mount path and return dictionary """ result = {'data': {}} try: - client = hvac.Client(url=url, token=token) - result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) + if client.secrets.kv.default_kv_version == '1': + result = client.secrets.kv.v1.read_secret(path=path, mount_point=mount) + else: + result = client.secrets.kv.v2.read_secret_version(path=path, mount_point=mount) except ConnectionError: pass finally: