From 56ed416cb7664f20a7bd28dfb517f9183cc64cd5 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 29 Oct 2018 09:10:43 -0700 Subject: [PATCH 001/110] Celery task for sync job --- lemur/common/celery.py | 25 ++++++++++++++++++++++++- lemur/sources/cli.py | 1 + requirements-docs.in | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 8dbb6c29..c12c6f06 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -19,7 +19,7 @@ from lemur.factory import create_app from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins -from lemur.sources.cli import clean, validate_sources +from lemur.sources.cli import clean, sync, validate_sources flask_app = create_app() @@ -188,3 +188,26 @@ def clean_source(source): """ current_app.logger.debug("Cleaning source {}".format(source)) clean([source], True) + + +@celery.task() +def sync_all_sources(): + """ + This function will sync certificates from all sources. This function triggers one celery task per source. + """ + sources = validate_sources("all") + for source in sources: + current_app.logger.debug("Creating celery task to sync source {}".format(source.label)) + sync_source.delay(source.label) + + +@celery.task() +def sync_source(source): + """ + This celery task will sync the specified source. + + :param source: + :return: + """ + current_app.logger.debug("Syncing source {}".format(source)) + sync([source], True) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index 1f2fd9b0..0ab8c9f8 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -93,6 +93,7 @@ def sync(source_strings): ) sentry.captureException() + metrics.send('source_sync_fail', 'counter', 1, metric_tags={'source': source.label, 'status': status}) metrics.send('source_sync', 'counter', 1, metric_tags={'source': source.label, 'status': status}) diff --git a/requirements-docs.in b/requirements-docs.in index cf598240..d04d510b 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -4,4 +4,4 @@ -r requirements.txt sphinx sphinxcontrib-httpdomain -sphinx-rtd-theme \ No newline at end of file +sphinx-rtd-theme From d4880f3e9d5f80f90098c1999b078887fdb58144 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 29 Oct 2018 09:39:12 -0700 Subject: [PATCH 002/110] update itsdangerous --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 6b49b64d..812a947f 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -44,7 +44,7 @@ gunicorn==19.9.0 idna==2.7 imagesize==1.1.0 # via sphinx inflection==0.3.1 -itsdangerous==1.0.0 +itsdangerous==1.1.0 jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index a851e620..2ad412db 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -29,7 +29,7 @@ flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.16.0 # via python-jose idna==2.7 # via cryptography, requests -itsdangerous==1.0.0 # via flask +itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto diff --git a/requirements.txt b/requirements.txt index c8cb2e2e..ffe6372f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ future==0.16.0 gunicorn==19.9.0 idna==2.7 # via cryptography, requests inflection==0.3.1 -itsdangerous==1.0.0 # via flask +itsdangerous==1.1.0 # via flask jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme From 50761d9d3b1603dc2780ea5c11acd7ed26056bef Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 29 Oct 2018 13:22:50 -0700 Subject: [PATCH 003/110] safer reissue, fix celery sync job --- lemur/certificates/cli.py | 12 +++++++++++- lemur/common/celery.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 013a4cb1..7a46138c 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -238,7 +238,17 @@ def reissue(old_certificate_name, commit): if not old_cert: for certificate in get_all_pending_reissue(): - request_reissue(certificate, commit) + try: + request_reissue(certificate, commit) + except Exception as e: + sentry.captureException() + current_app.logger.exception( + "Error reissuing certificate: {}".format(certificate.name), exc_info=True) + print( + "[!] Failed to reissue certificates. Reason: {}".format( + e + ) + ) else: request_reissue(old_cert, commit) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index c12c6f06..1711b452 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -210,4 +210,4 @@ def sync_source(source): :return: """ current_app.logger.debug("Syncing source {}".format(source)) - sync([source], True) + sync([source]) From 0277e4dc0554adb99855c183ad4c300715678b9d Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 29 Oct 2018 13:53:30 -0700 Subject: [PATCH 004/110] get_or_increase_name fix for pendingcertificates --- lemur/pending_certificates/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index 1261177d..7dc8e602 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import relationship from sqlalchemy_utils import JSONType from sqlalchemy_utils.types.arrow import ArrowType -from lemur.certificates.models import get_or_increase_name +from lemur.certificates.models import get_sequence from lemur.common import defaults, utils from lemur.database import db from lemur.models import pending_cert_source_associations, \ @@ -19,6 +19,28 @@ from lemur.models import pending_cert_source_associations, \ from lemur.utils import Vault +def get_or_increase_name(name, serial): + certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(name))).all() + + if not certificates: + return name + + serial_name = '{0}-{1}'.format(name, hex(int(serial))[2:].upper()) + certificates = PendingCertificate.query.filter(PendingCertificate.name.ilike('{0}%'.format(serial_name))).all() + + if not certificates: + return serial_name + + ends = [0] + root, end = get_sequence(serial_name) + for cert in certificates: + root, end = get_sequence(cert.name) + if end: + ends.append(end) + + return '{0}-{1}'.format(root, max(ends) + 1) + + class PendingCertificate(db.Model): __tablename__ = 'pending_certs' id = Column(Integer, primary_key=True) From 52e773230d3fcc5c01fe36b644835381f733b339 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 10:29:11 -0800 Subject: [PATCH 005/110] Add new gin index to optimize ILIKE queries --- lemur/certificates/models.py | 8 +++++ lemur/certificates/service.py | 3 +- lemur/common/celery.py | 4 +-- lemur/domains/models.py | 7 ++++- lemur/migrations/versions/ee827d1e1974_.py | 35 ++++++++++++++++++++++ requirements-dev.txt | 20 ++++++------- requirements-docs.txt | 2 +- requirements-tests.txt | 25 ++++++++-------- requirements.txt | 22 +++++++------- 9 files changed, 86 insertions(+), 40 deletions(-) create mode 100644 lemur/migrations/versions/ee827d1e1974_.py diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 6b438c06..7a1706f4 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -77,6 +77,14 @@ def get_or_increase_name(name, serial): class Certificate(db.Model): __tablename__ = 'certificates' + __table_args__ = ( + Index('ix_certificates_cn', "cn", + postgresql_ops={"cn": "gin_trgm_ops"}, + postgresql_using='gin'), + Index('ix_certificates_name', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True) external_id = Column(String(128)) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index fd6142ee..8fc031c4 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -362,7 +362,8 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now) - return database.sort_and_page(query, Certificate, args) + result = database.sort_and_page(query, Certificate, args) + return result def create_csr(**csr_config): diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 1711b452..69bd9ce1 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -193,8 +193,8 @@ def clean_source(source): @celery.task() def sync_all_sources(): """ - This function will sync certificates from all sources. This function triggers one celery task per source. - """ + This function will sync certificates from all sources. This function triggers one celery task per source. + """ sources = validate_sources("all") for source in sources: current_app.logger.debug("Creating celery task to sync source {}".format(source.label)) diff --git a/lemur/domains/models.py b/lemur/domains/models.py index d0c7fef4..05fccd9c 100644 --- a/lemur/domains/models.py +++ b/lemur/domains/models.py @@ -7,13 +7,18 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy import Column, Integer, String, Boolean +from sqlalchemy import Column, Integer, String, Boolean, Index from lemur.database import db class Domain(db.Model): __tablename__ = 'domains' + __table_args__ = ( + Index('ix_domains_name_gin', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) name = Column(String(256), index=True) sensitive = Column(Boolean, default=False) diff --git a/lemur/migrations/versions/ee827d1e1974_.py b/lemur/migrations/versions/ee827d1e1974_.py new file mode 100644 index 00000000..26f7cea4 --- /dev/null +++ b/lemur/migrations/versions/ee827d1e1974_.py @@ -0,0 +1,35 @@ +"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries. + +Revision ID: ee827d1e1974 +Revises: 7ead443ba911 +Create Date: 2018-11-05 09:49:40.226368 + +""" + +# revision identifiers, used by Alembic. +revision = 'ee827d1e1974' +down_revision = '7ead443ba911' + +from alembic import op +from sqlalchemy.exc import ProgrammingError + +def upgrade(): + try: + connection = op.get_bind() + connection.execute("CREATE EXTENSION pg_trgm") + except ProgrammingError as e: + # Extension is most likely already enabled + connection.execute("ROLLBACK") + + op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'}, + postgresql_using='gin') + op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False, + postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin') + op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'}, + postgresql_using='gin') + + +def downgrade(): + op.drop_index('ix_domains_name', table_name='domains') + op.drop_index('ix_certificates_name', table_name='certificates') + op.drop_index('ix_certificates_cn', table_name='certificates') diff --git a/requirements-dev.txt b/requirements-dev.txt index c473aa56..428d2082 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,32 +8,30 @@ aspy.yaml==1.1.1 # via pre-commit bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.10.15 # via requests -cffi==1.11.5 # via cmarkgfm cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests -cmarkgfm==0.4.2 # via readme-renderer docutils==0.14 # via readme-renderer -flake8==3.5.0 -future==0.16.0 # via readme-renderer +flake8==3.6.0 identify==1.1.7 # via pre-commit idna==2.7 # via requests +importlib-metadata==0.6 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine -pre-commit==1.11.2 -pycodestyle==2.3.1 # via flake8 -pycparser==2.19 # via cffi -pyflakes==1.6.0 # via flake8 +pre-commit==1.12.0 +pycodestyle==2.4.0 # via flake8 +pyflakes==2.0.0 # via flake8 pygments==2.2.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit -readme-renderer==22.0 # via twine +readme-renderer==24.0 # via twine requests-toolbelt==0.8.0 # via twine requests==2.20.0 # via requests-toolbelt, twine six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.28.1 # via twine twine==1.12.1 -urllib3==1.24 # via requests -virtualenv==16.0.0 # via pre-commit +urllib3==1.24.1 # via requests +virtualenv==16.1.0 # via pre-commit webencodings==0.5.1 # via bleach diff --git a/requirements-docs.txt b/requirements-docs.txt index 812a947f..c89ca75b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -69,7 +69,7 @@ pygments==2.2.0 # via sphinx pyjwt==1.6.4 pynacl==1.3.0 pyopenssl==18.0.0 -pyparsing==2.2.2 # via packaging +pyparsing==2.3.0 # via packaging pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 diff --git a/requirements-tests.txt b/requirements-tests.txt index 2ad412db..64875903 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,10 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -biscuits==0.1.1 # via responses -boto3==1.9.28 # via moto +boto3==1.9.37 # via moto boto==2.49.0 # via moto -botocore==1.12.28 # via boto3, moto, s3transfer +botocore==1.12.37 # via boto3, moto, s3transfer certifi==2018.10.15 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -27,40 +26,40 @@ factory-boy==2.11.1 faker==0.9.2 flask==1.0.2 # via pytest-flask freezegun==0.3.11 -future==0.16.0 # via python-jose +future==0.17.1 # via python-jose idna==2.7 # via cryptography, requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.0 # via aws-xray-sdk -markupsafe==1.0 # via jinja2 +markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.3.0 # via pytest moto==1.3.4 nose==1.3.7 -pbr==5.0.0 # via mock +pbr==5.1.0 # via mock pluggy==0.8.0 # via pytest py==1.7.0 # via pytest pyaml==17.12.1 # via moto pycparser==2.19 # via cffi -pycryptodome==3.6.6 # via python-jose +pycryptodome==3.7.0 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==3.9.1 -python-dateutil==2.7.3 # via botocore, faker, freezegun, moto +pytest==3.10.0 +python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.5 # via moto +pytz==2018.7 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.1 # via moto +responses==0.10.2 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.23 # via botocore, requests -websocket-client==0.53.0 # via docker +urllib3==1.24.1 # via botocore, requests +websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.10.11 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.txt b/requirements.txt index ffe6372f..51d93b09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,9 @@ # acme==0.27.1 alembic-autogenerate-enums==0.0.2 -alembic==1.0.1 # via flask-migrate +alembic==1.0.2 # via flask-migrate amqp==2.3.2 # via kombu -aniso8601==3.0.2 # via flask-restful +aniso8601==4.0.1 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 @@ -37,7 +37,7 @@ flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 -future==0.16.0 +future==0.17.1 gunicorn==19.9.0 idna==2.7 # via cryptography, requests inflection==0.3.1 @@ -49,13 +49,13 @@ jsonlines==1.2.0 # via cloudflare kombu==4.2.1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic -markupsafe==1.0 # via jinja2, mako -marshmallow-sqlalchemy==0.14.1 -marshmallow==2.16.0 +markupsafe==1.1.0 # via jinja2, mako +marshmallow-sqlalchemy==0.15.0 +marshmallow==2.16.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.0.0 # via mock +pbr==5.1.0 # via mock pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap @@ -65,10 +65,10 @@ pyjwt==1.6.4 pynacl==1.3.0 # via paramiko pyopenssl==18.0.0 pyrfc3339==1.1 # via acme -python-dateutil==2.7.3 # via alembic, arrow, botocore +python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339 +pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 redis==2.10.6 # via celery @@ -78,9 +78,9 @@ retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.13 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 -urllib3==1.24 # via requests +urllib3==1.24.1 # via requests vine==1.1.4 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0 From b6cc8180fe521fbdf1d2cc93b2a2b177e0bf470a Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 11:20:11 -0800 Subject: [PATCH 006/110] downgrade flake8 --- requirements-dev.in | 2 +- requirements-dev.txt | 6 +++--- requirements-docs.txt | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index de8b60d3..84104679 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,6 @@ # Run `make up-reqs` to update pinned dependencies in requirement text files -flake8>=3.2,<4.0 +flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors. pre-commit invoke twine diff --git a/requirements-dev.txt b/requirements-dev.txt index 428d2082..1ded25a2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ certifi==2018.10.15 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer -flake8==3.6.0 +flake8==3.5.0 identify==1.1.7 # via pre-commit idna==2.7 # via requests importlib-metadata==0.6 # via pre-commit @@ -21,8 +21,8 @@ mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine pre-commit==1.12.0 -pycodestyle==2.4.0 # via flake8 -pyflakes==2.0.0 # via flake8 +pycodestyle==2.3.1 # via flake8 +pyflakes==1.6.0 # via flake8 pygments==2.2.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index c89ca75b..8937be9e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -7,9 +7,9 @@ acme==0.27.1 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.1 +alembic==1.0.2 amqp==2.3.2 -aniso8601==3.0.2 +aniso8601==4.0.1 arrow==0.12.1 asn1crypto==0.24.0 asyncpool==1.0 @@ -39,7 +39,7 @@ flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 -future==0.16.0 +future==0.17.1 gunicorn==19.9.0 idna==2.7 imagesize==1.1.0 # via sphinx @@ -52,14 +52,14 @@ jsonlines==1.2.0 kombu==4.2.1 lockfile==0.12.2 mako==1.0.7 -markupsafe==1.0 -marshmallow-sqlalchemy==0.14.1 -marshmallow==2.16.0 +markupsafe==1.1.0 +marshmallow-sqlalchemy==0.15.0 +marshmallow==2.16.3 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==18.0 # via sphinx paramiko==2.4.2 -pbr==5.0.0 +pbr==5.1.0 pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 @@ -71,9 +71,9 @@ pynacl==1.3.0 pyopenssl==18.0.0 pyparsing==2.3.0 # via packaging pyrfc3339==1.1 -python-dateutil==2.7.3 +python-dateutil==2.7.5 python-editor==1.0.3 -pytz==2018.5 +pytz==2018.7 pyyaml==3.13 raven[flask]==6.9.0 redis==2.10.6 @@ -88,9 +88,9 @@ sphinx==1.8.1 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 +sqlalchemy==1.2.13 tabulate==0.8.2 -urllib3==1.24 +urllib3==1.24.1 vine==1.1.4 werkzeug==0.14.1 xmltodict==0.11.0 From 0b697b9d531d7fb329ba694f6b921d5c0259ed38 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 12:19:49 -0800 Subject: [PATCH 007/110] Adding travis declaration for pg_trgm extension --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 37ec1434..73006815 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ env: before_script: - psql -c "create database lemur;" -U postgres - psql -c "create user lemur with password 'lemur;'" -U postgres + - psql -c "CREATE EXTENSION pg_trgm" -U postgres - npm config set registry https://registry.npmjs.org - npm install -g bower - pip install --upgrade setuptools From bb36d0e0fa325d164436b8e1ffacebf95d686a7b Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 12:47:05 -0800 Subject: [PATCH 008/110] Add semicolon --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 73006815..c91682fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ env: before_script: - psql -c "create database lemur;" -U postgres - psql -c "create user lemur with password 'lemur;'" -U postgres - - psql -c "CREATE EXTENSION pg_trgm" -U postgres + - psql -c "CREATE EXTENSION pg_trgm;" -U postgres - npm config set registry https://registry.npmjs.org - npm install -g bower - pip install --upgrade setuptools From 73e4396edd5acce0bc58cd0f197822b9d912c161 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 12:58:39 -0800 Subject: [PATCH 009/110] Enable on all schemas --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c91682fc..d3138c4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ env: before_script: - psql -c "create database lemur;" -U postgres - psql -c "create user lemur with password 'lemur;'" -U postgres - - psql -c "CREATE EXTENSION pg_trgm;" -U postgres + - psql -c "create extension pg_trgm with schema pg_catalog;" -U postgres - npm config set registry https://registry.npmjs.org - npm install -g bower - pip install --upgrade setuptools From 61738dde9eeb49719d8a7eee993ed912153976c1 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 13:15:53 -0800 Subject: [PATCH 010/110] Run query on DB --- .travis.yml | 2 +- lemur/migrations/versions/ee827d1e1974_.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index d3138c4e..35323b5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ env: before_script: - psql -c "create database lemur;" -U postgres - psql -c "create user lemur with password 'lemur;'" -U postgres - - psql -c "create extension pg_trgm with schema pg_catalog;" -U postgres + - psql lemur -c "create extension IF NOT EXISTS pg_trgm with schema pg_catalog;" -U postgres - npm config set registry https://registry.npmjs.org - npm install -g bower - pip install --upgrade setuptools diff --git a/lemur/migrations/versions/ee827d1e1974_.py b/lemur/migrations/versions/ee827d1e1974_.py index 26f7cea4..62ac6222 100644 --- a/lemur/migrations/versions/ee827d1e1974_.py +++ b/lemur/migrations/versions/ee827d1e1974_.py @@ -14,12 +14,8 @@ from alembic import op from sqlalchemy.exc import ProgrammingError def upgrade(): - try: - connection = op.get_bind() - connection.execute("CREATE EXTENSION pg_trgm") - except ProgrammingError as e: - # Extension is most likely already enabled - connection.execute("ROLLBACK") + connection = op.get_bind() + connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'}, postgresql_using='gin') From b9f511ed0208829425b0c3e830d4c294227a790c Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 13:19:22 -0800 Subject: [PATCH 011/110] Updat email on travisci --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35323b5c..b540937d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ env: before_script: - psql -c "create database lemur;" -U postgres - psql -c "create user lemur with password 'lemur;'" -U postgres - - psql lemur -c "create extension IF NOT EXISTS pg_trgm with schema pg_catalog;" -U postgres + - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres - npm config set registry https://registry.npmjs.org - npm install -g bower - pip install --upgrade setuptools @@ -46,4 +46,4 @@ after_success: notifications: email: - kglisson@netflix.com + ccastrapel@netflix.com From 75183ef2f2270927fac4be24b69d38a20918d186 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 14:37:52 -0800 Subject: [PATCH 012/110] Unpin most dependencies, and fix moto --- Makefile | 2 ++ lemur/plugins/lemur_acme/plugin.py | 6 +++++- lemur/tests/conftest.py | 8 ++++++++ requirements-docs.txt | 6 +++--- requirements-tests.in | 2 +- requirements-tests.txt | 5 ++--- requirements.in | 18 +++++++++--------- requirements.txt | 12 ++++++------ 8 files changed, 36 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index f740faab..19a69236 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,8 @@ reset-db: dropdb lemur || true @echo "--> Creating 'lemur' database" createdb -E utf-8 lemur + @echo "--> Enabling pg_trgm extension" + psql lemur -c "create extension IF NOT EXISTS pg_trgm;" @echo "--> Applying migrations" lemur db upgrade diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 3f0e8314..62e647c4 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -44,7 +44,11 @@ class AuthorizationRecord(object): class AcmeHandler(object): def __init__(self): self.dns_providers_for_domain = {} - self.all_dns_providers = dns_provider_service.get_all_dns_providers() + try: + self.all_dns_providers = dns_provider_service.get_all_dns_providers() + except Exception as e: + current_app.logger.error("Unable to fetch DNS Providers: {}".format(e)) + self.all_dns_providers = [] def find_dns_challenge(self, authorizations): dns_challenges = [] diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index fcd3005d..d0175c83 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -240,3 +240,11 @@ def cert_builder(private_key): .public_key(private_key.public_key()) .not_valid_before(datetime.datetime(2017, 12, 22)) .not_valid_after(datetime.datetime(2040, 1, 1))) + + +@pytest.fixture(scope='function') +def aws_credentials(): + os.environ['AWS_ACCESS_KEY_ID'] = 'testing' + os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' + os.environ['AWS_SECURITY_TOKEN'] = 'testing' + os.environ['AWS_SESSION_TOKEN'] = 'testing' diff --git a/requirements-docs.txt b/requirements-docs.txt index 8937be9e..b18291d3 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.4 billiard==3.5.0.4 blinker==1.4 -boto3==1.7.79 -botocore==1.10.84 +boto3==1.9.37 +botocore==1.12.37 celery[redis]==4.2.1 certifi==2018.10.15 cffi==1.11.5 @@ -59,7 +59,7 @@ mock==2.0.0 ndg-httpsclient==0.5.1 packaging==18.0 # via sphinx paramiko==2.4.2 -pbr==5.1.0 +pbr==5.1.1 pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 diff --git a/requirements-tests.in b/requirements-tests.in index efb4570a..02a2b0ae 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -4,7 +4,7 @@ coverage factory-boy Faker freezegun -moto==1.3.4 # Issue with moto: https://github.com/spulec/moto/issues/1813 +moto nose pyflakes pytest diff --git a/requirements-tests.txt b/requirements-tests.txt index 64875903..a483c3fc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -15,7 +15,6 @@ certifi==2018.10.15 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask -cookies==2.2.1 # via moto coverage==4.5.1 cryptography==2.3.1 # via moto docker-pycreds==0.3.0 # via docker @@ -36,9 +35,9 @@ jsonpickle==1.0 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.3.0 # via pytest -moto==1.3.4 +moto==1.3.7 nose==1.3.7 -pbr==5.1.0 # via mock +pbr==5.1.1 # via mock pluggy==0.8.0 # via pytest py==1.7.0 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.in b/requirements.in index a2b920c3..5c5e2d93 100644 --- a/requirements.in +++ b/requirements.in @@ -4,22 +4,22 @@ acme alembic-autogenerate-enums arrow asyncpool -boto3==1.7.79 # Issue with moto: https://github.com/spulec/moto/issues/1813 -botocore== 1.10.84 # Issue with moto: https://github.com/spulec/moto/issues/1813 +boto3 +botocore celery[redis] certifi CloudFlare cryptography dnspython3 dyn -Flask-Bcrypt==0.7.1 -Flask-Mail==0.9.1 -Flask-Migrate==2.1.1 -Flask-Principal==0.4.0 -Flask-RESTful==0.3.6 -Flask-Script==2.0.6 +Flask-Bcrypt +Flask-Mail +Flask-Migrate +Flask-Principal +Flask-RESTful +Flask-Script Flask-SQLAlchemy -Flask==0.12 +Flask==0.12.4 Flask-Cors future gunicorn diff --git a/requirements.txt b/requirements.txt index 51d93b09..613979af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko billiard==3.5.0.4 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.79 -botocore==1.10.84 +boto3==1.9.37 +botocore==1.12.37 celery[redis]==4.2.1 certifi==2018.10.15 cffi==1.11.5 # via bcrypt, cryptography, pynacl @@ -31,12 +31,12 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.6 flask-mail==0.9.1 -flask-migrate==2.1.1 +flask-migrate==2.3.0 flask-principal==0.4.0 flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 -flask==0.12 +flask==0.12.4 future==0.17.1 gunicorn==19.9.0 idna==2.7 # via cryptography, requests @@ -55,7 +55,7 @@ marshmallow==2.16.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.1.0 # via mock +pbr==5.1.1 # via mock pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap @@ -80,7 +80,7 @@ six==1.11.0 sqlalchemy-utils==0.33.6 sqlalchemy==1.2.13 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 -urllib3==1.24.1 # via requests +urllib3==1.24.1 # via botocore, requests vine==1.1.4 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0 From a3f96b96eeea3f084983aa8631f45ff6e20c5e15 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 15:16:09 -0800 Subject: [PATCH 013/110] Add fixture to failing function --- lemur/plugins/lemur_aws/tests/test_elb.py | 2 +- requirements-docs.txt | 4 ++-- requirements.in | 2 +- requirements.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_elb.py b/lemur/plugins/lemur_aws/tests/test_elb.py index e34b66de..7facc4dd 100644 --- a/lemur/plugins/lemur_aws/tests/test_elb.py +++ b/lemur/plugins/lemur_aws/tests/test_elb.py @@ -4,7 +4,7 @@ from moto import mock_sts, mock_elb @mock_sts() @mock_elb() -def test_get_all_elbs(app): +def test_get_all_elbs(app, aws_credentials): from lemur.plugins.lemur_aws.elb import get_all_elbs client = boto3.client('elb', region_name='us-east-1') diff --git a/requirements-docs.txt b/requirements-docs.txt index b18291d3..709feb23 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -33,12 +33,12 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.6 flask-mail==0.9.1 -flask-migrate==2.1.1 +flask-migrate==2.3.0 flask-principal==0.4.0 flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 -flask==0.12 +flask==0.12.4 future==0.17.1 gunicorn==19.9.0 idna==2.7 diff --git a/requirements.in b/requirements.in index 5c5e2d93..deb44218 100644 --- a/requirements.in +++ b/requirements.in @@ -19,7 +19,7 @@ Flask-Principal Flask-RESTful Flask-Script Flask-SQLAlchemy -Flask==0.12.4 +Flask Flask-Cors future gunicorn diff --git a/requirements.txt b/requirements.txt index 613979af..abb3c061 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ flask-principal==0.4.0 flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 -flask==0.12.4 +flask==1.0.2 future==0.17.1 gunicorn==19.9.0 idna==2.7 # via cryptography, requests From 08a2a2b0e566f4acadedd634c7905dea14393da2 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 7 Nov 2018 15:34:25 -0800 Subject: [PATCH 014/110] Optimize certificate filtering by name --- lemur/certificates/service.py | 5 +++-- lemur/database.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 8fc031c4..6deeecfa 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -20,6 +20,7 @@ from lemur.common.utils import generate_private_key, truthiness from lemur.destinations.models import Destination from lemur.domains.models import Domain from lemur.extensions import metrics, sentry, signals +from lemur.models import certificate_associations from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate from lemur.plugins.base import plugins @@ -332,13 +333,13 @@ def render(args): elif 'id' in terms: query = query.filter(Certificate.id == cast(terms[1], Integer)) elif 'name' in terms: - query = query.filter( + query = query.join(certificate_associations).join(Domain).filter( or_( Certificate.name.ilike(term), Certificate.domains.any(Domain.name.ilike(term)), Certificate.cn.ilike(term), ) - ) + ).group_by(Certificate.id) else: query = database.filter(query, Certificate, terms) diff --git a/lemur/database.py b/lemur/database.py index ad3899aa..5cfe340a 100644 --- a/lemur/database.py +++ b/lemur/database.py @@ -11,11 +11,11 @@ """ from inflection import underscore from sqlalchemy import exc, func -from sqlalchemy.sql import and_, or_ from sqlalchemy.orm import make_transient +from sqlalchemy.sql import and_, or_ -from lemur.extensions import db from lemur.exceptions import AttrNotFound, DuplicateError +from lemur.extensions import db def filter_none(kwargs): @@ -273,7 +273,7 @@ def get_count(q): :param q: :return: """ - count_q = q.statement.with_only_columns([func.count()]).order_by(None) + count_q = q.statement.with_only_columns([func.count()]).group_by(None).order_by(None) count = q.session.execute(count_q).scalar() return count From 1643650685a337d852e3895ac47b66f214748499 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 7 Nov 2018 16:02:04 -0800 Subject: [PATCH 015/110] Changing essential part of query --- lemur/certificates/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 6deeecfa..6f2e4d9d 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -336,7 +336,7 @@ def render(args): query = query.join(certificate_associations).join(Domain).filter( or_( Certificate.name.ilike(term), - Certificate.domains.any(Domain.name.ilike(term)), + Domain.name.ilike(term), Certificate.cn.ilike(term), ) ).group_by(Certificate.id) From 6f0005c78e21f08983bedb4482403b61e1d1bd41 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 9 Nov 2018 10:31:27 -0800 Subject: [PATCH 016/110] Avoid colliding LetsEncrypt jobs --- lemur/common/celery.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 69bd9ce1..13f3c46b 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -142,8 +142,7 @@ def fetch_all_pending_acme_certs(): for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': - if cert.last_updated == cert.date_created or datetime.now( - timezone.utc) - cert.last_updated > timedelta(minutes=5): + if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5): fetch_acme_cert.delay(cert.id) From a7a05e26bc7c2178820eeaf674ae6e55e6da695f Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 12 Nov 2018 09:52:11 -0800 Subject: [PATCH 017/110] Do not re-use CSR during certificate reissuance; Update requirement; Add more logging to celery handler --- lemur/certificates/models.py | 2 +- lemur/certificates/service.py | 3 +++ lemur/common/celery.py | 15 ++++++++++++++- lemur/plugins/lemur_aws/plugin.py | 2 +- requirements-dev.txt | 4 ++-- requirements-docs.txt | 4 ++-- requirements-tests.txt | 14 +++++++------- requirements.txt | 16 ++++++++-------- 8 files changed, 38 insertions(+), 22 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 7a1706f4..97794c38 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -367,7 +367,7 @@ def update_destinations(target, value, initiator): destination_plugin = plugins.get(value.plugin_name) status = FAILURE_METRIC_STATUS try: - if target.private_key: + if target.private_key or not destination_plugin.requires_key: destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options) status = SUCCESS_METRIC_STATUS except Exception as e: diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 8fc031c4..0470811c 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -539,6 +539,9 @@ def reissue_certificate(certificate, replace=None, user=None): """ primitives = get_certificate_primitives(certificate) + if primitives.get("csr"): + # We do not want to re-use the CSR when creating a certificate because this defeats the purpose of rotation. + del primitives["csr"] if not user: primitives['creator'] = certificate.user diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 13f3c46b..82977051 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -53,8 +53,10 @@ def fetch_acme_cert(id): id: an id of a PendingCertificate """ log_data = { - "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) + "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "message": "Resolving pending certificate {}".format(id) } + current_app.logger.debug(log_data) pending_certs = pending_certificate_service.get_pending_certs([id]) new = 0 failed = 0 @@ -138,11 +140,22 @@ def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" pending_certs = pending_certificate_service.get_unresolved_pending_certs() + log_data = { + "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "message": "Starting job." + } + + current_app.logger.debug(log_data) + # We only care about certs using the acme-issuer plugin for cert in pending_certs: cert_authority = get_authority(cert.authority_id) if cert_authority.plugin_name == 'acme-issuer': if datetime.now(timezone.utc) - cert.last_updated > timedelta(minutes=5): + log_data["message"] = "Triggering job for cert {}".format(cert.name) + log_data["cert_name"] = cert.name + log_data["cert_id"] = cert.id + current_app.logger.debug(log_data) fetch_acme_cert.delay(cert.id) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index d959cfdc..c563eac8 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -35,8 +35,8 @@ from flask import current_app from lemur.plugins import lemur_aws as aws -from lemur.plugins.lemur_aws import iam, s3, elb, ec2 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin +from lemur.plugins.lemur_aws import iam, s3, elb, ec2 def get_region_from_dns(dns): diff --git a/requirements-dev.txt b/requirements-dev.txt index 1ded25a2..4120ef26 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ importlib-metadata==0.6 # via pre-commit importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 -nodeenv==1.3.2 +nodeenv==1.3.3 pkginfo==1.4.2 # via twine pre-commit==1.12.0 pycodestyle==2.3.1 # via flake8 @@ -27,7 +27,7 @@ pygments==2.2.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine requests-toolbelt==0.8.0 # via twine -requests==2.20.0 # via requests-toolbelt, twine +requests==2.20.1 # via requests-toolbelt, twine six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.28.1 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index 709feb23..12cbf76b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -38,7 +38,7 @@ flask-principal==0.4.0 flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 -flask==0.12.4 +flask==1.0.2 future==0.17.1 gunicorn==19.9.0 idna==2.7 @@ -84,7 +84,7 @@ s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.2 -sphinx==1.8.1 +sphinx==1.8.2 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.6 diff --git a/requirements-tests.txt b/requirements-tests.txt index a483c3fc..38b2ea59 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,15 +8,15 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.37 # via moto +boto3==1.9.42 # via moto boto==2.49.0 # via moto -botocore==1.12.37 # via boto3, moto, s3transfer +botocore==1.12.42 # via boto3, moto, s3transfer certifi==2018.10.15 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask -coverage==4.5.1 -cryptography==2.3.1 # via moto +coverage==4.5.2 +cryptography==2.4.1 # via moto docker-pycreds==0.3.0 # via docker docker==3.5.1 # via moto docutils==0.14 # via botocore @@ -46,14 +46,14 @@ pycryptodome==3.7.0 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==3.10.0 +pytest==3.10.1 python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.7 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 -requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.2 # via moto +requests==2.20.1 # via aws-xray-sdk, docker, moto, requests-mock, responses +responses==0.10.3 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker diff --git a/requirements.txt b/requirements.txt index abb3c061..ef46f98c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.27.1 +acme==0.28.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.2 # via flask-migrate amqp==2.3.2 # via kombu @@ -15,21 +15,21 @@ asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko billiard==3.5.0.4 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.37 -botocore==1.12.37 +boto3==1.9.42 +botocore==1.12.42 celery[redis]==4.2.1 certifi==2018.10.15 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 -cryptography==2.3.1 +cryptography==2.4.1 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.6 +flask-cors==3.0.7 flask-mail==0.9.1 flask-migrate==2.3.0 flask-principal==0.4.0 @@ -57,7 +57,7 @@ ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.1 # via mock pem==18.2.0 -psycopg2==2.7.5 +psycopg2==2.7.6.1 pyasn1-modules==0.2.2 # via python-ldap pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi @@ -73,12 +73,12 @@ pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 redis==2.10.6 # via celery requests-toolbelt==0.8.0 # via acme -requests[security]==2.20.0 +requests[security]==2.20.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.13 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.14 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests vine==1.1.4 # via amqp From 92a771f5ed301e491774714a2183f43c15c55a97 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 13 Nov 2018 09:14:21 -0800 Subject: [PATCH 018/110] More accurate db count functionality --- lemur/database.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/lemur/database.py b/lemur/database.py index 5cfe340a..82fb0423 100644 --- a/lemur/database.py +++ b/lemur/database.py @@ -10,8 +10,8 @@ .. moduleauthor:: Kevin Glisson """ from inflection import underscore -from sqlalchemy import exc, func -from sqlalchemy.orm import make_transient +from sqlalchemy import exc, func, distinct +from sqlalchemy.orm import make_transient, lazyload from sqlalchemy.sql import and_, or_ from lemur.exceptions import AttrNotFound, DuplicateError @@ -273,7 +273,31 @@ def get_count(q): :param q: :return: """ - count_q = q.statement.with_only_columns([func.count()]).group_by(None).order_by(None) + disable_group_by = False + if len(q._entities) > 1: + # currently support only one entity + raise Exception('only one entity is supported for get_count, got: %s' % q) + entity = q._entities[0] + if hasattr(entity, 'column'): + # _ColumnEntity has column attr - on case: query(Model.column)... + col = entity.column + if q._group_by and q._distinct: + # which query can have both? + raise NotImplementedError + if q._group_by or q._distinct: + col = distinct(col) + if q._group_by: + # need to disable group_by and enable distinct - we can do this because we have only 1 entity + disable_group_by = True + count_func = func.count(col) + else: + # _MapperEntity doesn't have column attr - on case: query(Model)... + count_func = func.count() + if q._group_by and not disable_group_by: + count_func = count_func.over(None) + count_q = q.options(lazyload('*')).statement.with_only_columns([count_func]).order_by(None) + if disable_group_by: + count_q = count_q.group_by(None) count = q.session.execute(count_q).scalar() return count From 3ce8abe46e861726ad022bbbce19f6f00097f518 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 13 Nov 2018 14:33:17 -0800 Subject: [PATCH 019/110] Left outer join on domains tables to avoid missing results --- lemur/authorities/service.py | 2 ++ lemur/certificates/service.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 1d35ad49..024cb42a 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -178,6 +178,8 @@ def render(args): terms = filt.split(';') if 'active' in filt: query = query.filter(Authority.active == truthiness(terms[1])) + elif 'cn' in filt: + query = query.join(Authority.active == truthiness(terms[1])) else: query = database.filter(query, Authority, terms) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index eb8b5d37..20a59e11 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -333,7 +333,7 @@ def render(args): elif 'id' in terms: query = query.filter(Certificate.id == cast(terms[1], Integer)) elif 'name' in terms: - query = query.join(certificate_associations).join(Domain).filter( + query = query.outerjoin(certificate_associations).outerjoin(Domain).filter( or_( Certificate.name.ilike(term), Domain.name.ilike(term), From 61839f4acadd3dfac12b21bb0572d1ddc795a236 Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Mon, 19 Nov 2018 13:42:42 +0100 Subject: [PATCH 020/110] Add support for nested group membership in ldap authenticator --- lemur/auth/ldap.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index 398a5830..dc8f8941 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -41,7 +41,6 @@ class LdapPrincipal(): self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None) self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None) self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None) - self.ldap_attrs = ['memberOf'] self.ldap_client = None self.ldap_groups = None @@ -168,11 +167,21 @@ class LdapPrincipal(): except ldap.LDAPError as e: raise Exception("ldap error: {0}".format(e)) - lgroups = self.ldap_client.search_s(self.ldap_base_dn, - ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf'] - # lgroups is a list of utf-8 encoded strings - # convert to a single string of groups to allow matching - self.ldap_groups = b''.join(lgroups).decode('ascii') + # Lookup user DN, needed to search for group membership + userdn = self.ldap_client.search_s(self.ldap_base_dn, + ldap.SCOPE_SUBTREE, ldap_filter, + ['distinguishedName'])[0][1]['distinguishedName'][0] + userdn = userdn.decode('utf-8') + # Search all groups that have the userDN as a member + groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn) + lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn']) + + # Create a list of group CN's from the result + self.ldap_groups = [] + for group in lgroups: + (dn, values) = group + self.ldap_groups.append(values['cn'][0].decode('ascii')) + self.ldap_client.unbind() def _ldap_validate_conf(self): From da10913045082669a66b5b2e7f2508da477f9714 Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Tue, 20 Nov 2018 10:37:36 +0100 Subject: [PATCH 021/110] Only search nested group memberships when LDAP_IS_ACTIVE_DIRECTORY is True --- lemur/auth/ldap.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index dc8f8941..7eded060 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -41,6 +41,8 @@ class LdapPrincipal(): self.ldap_default_role = current_app.config.get("LEMUR_DEFAULT_ROLE", None) self.ldap_required_group = current_app.config.get("LDAP_REQUIRED_GROUP", None) self.ldap_groups_to_roles = current_app.config.get("LDAP_GROUPS_TO_ROLES", None) + self.ldap_is_active_directory = current_app.config.get("LDAP_IS_ACTIVE_DIRECTORY", False) + self.ldap_attrs = ['memberOf'] self.ldap_client = None self.ldap_groups = None @@ -167,20 +169,27 @@ class LdapPrincipal(): except ldap.LDAPError as e: raise Exception("ldap error: {0}".format(e)) - # Lookup user DN, needed to search for group membership - userdn = self.ldap_client.search_s(self.ldap_base_dn, - ldap.SCOPE_SUBTREE, ldap_filter, - ['distinguishedName'])[0][1]['distinguishedName'][0] - userdn = userdn.decode('utf-8') - # Search all groups that have the userDN as a member - groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn) - lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn']) + if self.ldap_is_active_directory: + # Lookup user DN, needed to search for group membership + userdn = self.ldap_client.search_s(self.ldap_base_dn, + ldap.SCOPE_SUBTREE, ldap_filter, + ['distinguishedName'])[0][1]['distinguishedName'][0] + userdn = userdn.decode('utf-8') + # Search all groups that have the userDN as a member + groupfilter = '(&(objectclass=group)(member:1.2.840.113556.1.4.1941:={0}))'.format(userdn) + lgroups = self.ldap_client.search_s(self.ldap_base_dn, ldap.SCOPE_SUBTREE, groupfilter, ['cn']) - # Create a list of group CN's from the result - self.ldap_groups = [] - for group in lgroups: - (dn, values) = group - self.ldap_groups.append(values['cn'][0].decode('ascii')) + # Create a list of group CN's from the result + self.ldap_groups = [] + for group in lgroups: + (dn, values) = group + self.ldap_groups.append(values['cn'][0].decode('ascii')) + else: + lgroups = self.ldap_client.search_s(self.ldap_base_dn, + ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs)[0][1]['memberOf'] + # lgroups is a list of utf-8 encoded strings + # convert to a single string of groups to allow matching + self.ldap_groups = b''.join(lgroups).decode('ascii') self.ldap_client.unbind() From 5fc5a058b65f574784372cafff7aa9e465ebd468 Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Tue, 20 Nov 2018 10:51:14 +0100 Subject: [PATCH 022/110] Add documentation for the LDAP_IS_ACTIVE_DIRECTORY setting --- docs/administration.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index eec01cc5..9d6c8d12 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -324,6 +324,7 @@ Here is an example LDAP configuration stanza you can add to your config. Adjust LDAP_CACERT_FILE = '/opt/lemur/trusted.pem' LDAP_REQUIRED_GROUP = 'certificate-management-access' LDAP_GROUPS_TO_ROLES = {'certificate-management-admin': 'admin', 'certificate-management-read-only': 'read-only'} + LDAP_IS_ACTIVE_DIRECTORY = True The lemur ldap module uses the `user principal name` (upn) of the authenticating user to bind. This is done once for each user at login time. The UPN is effectively the email address in AD/LDAP of the user. If the user doesn't provide the email address, it constructs one based on the username supplied (which should normally match the samAccountName) and the value provided by the config LDAP_EMAIL_DOMAIN. @@ -406,6 +407,17 @@ The following LDAP options are not required, however TLS is always recommended. LDAP_GROUPS_TO_ROLES = {'lemur_admins': 'admin', 'Lemur Team DL Group': 'team@example.com'} +.. data:: LDAP_IS_ACTIVE_DIRECTORY + :noindex: + + When set to True, nested group memberships are supported, by searching for groups with the member:1.2.840.113556.1.4.1941 attribute set to the user DN. + When set to False, the list of groups will be determined by the 'memberof' attribute of the LDAP user logging in. + + :: + + LDAP_IS_ACTIVE_DIRECTORY = False + + Authentication Providers ~~~~~~~~~~~~~~~~~~~~~~~~ From 2381d0a4bb1ca6f12feae5700b57ed75ba8c3bcc Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 28 Nov 2018 11:32:52 -0800 Subject: [PATCH 023/110] Add async call to create pending cert when needed --- lemur/certificates/service.py | 7 +++++++ requirements-dev.txt | 4 ++-- requirements-docs.txt | 26 +++++++++++++------------- requirements-tests.txt | 16 ++++++++-------- requirements.in | 1 + requirements.txt | 14 +++++++------- 6 files changed, 38 insertions(+), 30 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 20a59e11..3a99a5f9 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -276,6 +276,13 @@ def create(**kwargs): certificate_issued.send(certificate=cert, authority=cert.authority) metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer)) + if isinstance(cert, PendingCertificate): + # We need to refresh the pending certificate to avoid "Instance is not bound to a Session; " + # "attribute refresh operation cannot proceed" + pending_cert = database.session_query(PendingCertificate).get(cert.id) + from lemur.common.celery import fetch_acme_cert + fetch_acme_cert.delay(pending_cert.id) + return cert diff --git a/requirements-dev.txt b/requirements-dev.txt index 4120ef26..599a1af9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,7 +14,7 @@ docutils==0.14 # via readme-renderer flake8==3.5.0 identify==1.1.7 # via pre-commit idna==2.7 # via requests -importlib-metadata==0.6 # via pre-commit +importlib-metadata==0.7 # via pre-commit importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 @@ -23,7 +23,7 @@ pkginfo==1.4.2 # via twine pre-commit==1.12.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 -pygments==2.2.0 # via readme-renderer +pygments==2.3.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine requests-toolbelt==0.8.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index 12cbf76b..a9c7a979 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,10 +4,10 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.27.1 +acme==0.28.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.2 +alembic==1.0.5 amqp==2.3.2 aniso8601==4.0.1 arrow==0.12.1 @@ -17,23 +17,23 @@ babel==2.6.0 # via sphinx bcrypt==3.1.4 billiard==3.5.0.4 blinker==1.4 -boto3==1.9.37 -botocore==1.12.37 +boto3==1.9.53 +botocore==1.12.53 celery[redis]==4.2.1 certifi==2018.10.15 cffi==1.11.5 chardet==3.0.4 click==7.0 cloudflare==2.1.0 -cryptography==2.3.1 +cryptography==2.4.2 dnspython3==1.15.0 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.6 +flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.0 +flask-migrate==2.3.1 flask-principal==0.4.0 flask-restful==0.3.6 flask-script==2.0.6 @@ -61,11 +61,11 @@ packaging==18.0 # via sphinx paramiko==2.4.2 pbr==5.1.1 pem==18.2.0 -psycopg2==2.7.5 +psycopg2==2.7.6.1 pyasn1-modules==0.2.2 pyasn1==0.4.4 pycparser==2.19 -pygments==2.2.0 # via sphinx +pygments==2.3.0 # via sphinx pyjwt==1.6.4 pynacl==1.3.0 pyopenssl==18.0.0 @@ -76,9 +76,9 @@ python-editor==1.0.3 pytz==2018.7 pyyaml==3.13 raven[flask]==6.9.0 -redis==2.10.6 +redis==3.0.1 requests-toolbelt==0.8.0 -requests[security]==2.20.0 +requests[security]==2.20.1 retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 @@ -87,8 +87,8 @@ sphinx-rtd-theme==0.4.2 sphinx==1.8.2 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx -sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.13 +sqlalchemy-utils==0.33.8 +sqlalchemy==1.2.14 tabulate==0.8.2 urllib3==1.24.1 vine==1.1.4 diff --git a/requirements-tests.txt b/requirements-tests.txt index 38b2ea59..72dcbbdb 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,21 +8,21 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.42 # via moto +boto3==1.9.53 # via moto boto==2.49.0 # via moto -botocore==1.12.42 # via boto3, moto, s3transfer +botocore==1.12.53 # via boto3, moto, s3transfer certifi==2018.10.15 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 -cryptography==2.4.1 # via moto +cryptography==2.4.2 # via moto docker-pycreds==0.3.0 # via docker docker==3.5.1 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==0.9.2 +faker==1.0.0 flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.17.1 # via python-jose @@ -40,20 +40,20 @@ nose==1.3.7 pbr==5.1.1 # via mock pluggy==0.8.0 # via pytest py==1.7.0 # via pytest -pyaml==17.12.1 # via moto +pyaml==18.11.0 # via moto pycparser==2.19 # via cffi -pycryptodome==3.7.0 # via python-jose +pycryptodome==3.7.2 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==3.10.1 +pytest==4.0.1 python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.7 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.20.1 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.3 # via moto +responses==0.10.4 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker diff --git a/requirements.in b/requirements.in index deb44218..9824650b 100644 --- a/requirements.in +++ b/requirements.in @@ -36,6 +36,7 @@ pyjwt pyOpenSSL python_ldap raven[flask] +redis<3 # redis>=3 is not compatible with celery requests retrying six diff --git a/requirements.txt b/requirements.txt index ef46f98c..c881799e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # acme==0.28.0 alembic-autogenerate-enums==0.0.2 -alembic==1.0.2 # via flask-migrate +alembic==1.0.5 # via flask-migrate amqp==2.3.2 # via kombu aniso8601==4.0.1 # via flask-restful arrow==0.12.1 @@ -15,15 +15,15 @@ asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko billiard==3.5.0.4 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.42 -botocore==1.12.42 +boto3==1.9.53 +botocore==1.12.53 celery[redis]==4.2.1 certifi==2018.10.15 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 -cryptography==2.4.1 +cryptography==2.4.2 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore @@ -31,7 +31,7 @@ dyn==1.8.1 flask-bcrypt==0.7.1 flask-cors==3.0.7 flask-mail==0.9.1 -flask-migrate==2.3.0 +flask-migrate==2.3.1 flask-principal==0.4.0 flask-restful==0.3.6 flask-script==2.0.6 @@ -71,13 +71,13 @@ python-ldap==3.1.0 pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 -redis==2.10.6 # via celery +redis==2.10.6 requests-toolbelt==0.8.0 # via acme requests[security]==2.20.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.6 +sqlalchemy-utils==0.33.8 sqlalchemy==1.2.14 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests From e074a14ee9f932b88494b89ecbb3fc59c9b508cf Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 28 Nov 2018 14:27:03 -0800 Subject: [PATCH 024/110] unit test --- lemur/certificates/models.py | 1 - lemur/certificates/service.py | 3 ++- lemur/common/celery.py | 4 ++-- lemur/manage.py | 2 +- lemur/plugins/lemur_acme/plugin.py | 13 ++++++++++++- lemur/tests/conf.py | 5 ++--- lemur/tests/test_pending_certificates.py | 6 +++--- 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 97794c38..e2ac2cba 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -138,7 +138,6 @@ class Certificate(db.Model): logs = relationship('Log', backref='certificate') endpoints = relationship('Endpoint', backref='certificate') rotation_policy = relationship("RotationPolicy") - sensitive_fields = ('private_key',) def __init__(self, **kwargs): diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 3a99a5f9..d965192e 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -281,7 +281,8 @@ def create(**kwargs): # "attribute refresh operation cannot proceed" pending_cert = database.session_query(PendingCertificate).get(cert.id) from lemur.common.celery import fetch_acme_cert - fetch_acme_cert.delay(pending_cert.id) + if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False): + fetch_acme_cert.delay(pending_cert.id) return cert diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 82977051..b7f65886 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -25,8 +25,8 @@ flask_app = create_app() def make_celery(app): - celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'], - broker=app.config['CELERY_BROKER_URL']) + celery = Celery(app.import_name, backend=app.config.get('CELERY_RESULT_BACKEND'), + broker=app.config.get('CELERY_BROKER_URL')) celery.conf.update(app.config) TaskBase = celery.Task diff --git a/lemur/manage.py b/lemur/manage.py index 6b1e1013..b972e8a5 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -47,7 +47,7 @@ from lemur.logs.models import Log # noqa from lemur.endpoints.models import Endpoint # noqa from lemur.policies.models import RotationPolicy # noqa from lemur.pending_certificates.models import PendingCertificate # noqa - +from lemur.dns_providers.models import DnsProvider # noqa manager = Manager(create_app) manager.add_option('-c', '--config', dest='config') diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 62e647c4..53d11935 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -333,9 +333,12 @@ class ACMEIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) - self.acme = AcmeHandler() + self.acme = None def get_dns_provider(self, type): + if not self.acme: + self.acme = AcmeHandler() + provider_types = { 'cloudflare': cloudflare, 'dyn': dyn, @@ -347,12 +350,16 @@ class ACMEIssuerPlugin(IssuerPlugin): return provider def get_all_zones(self, dns_provider): + if not self.acme: + self.acme = AcmeHandler() dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) return dns_provider_plugin.get_zones(account_number=account_number) def get_ordered_certificate(self, pending_cert): + if not self.acme: + self.acme = AcmeHandler() acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) if pending_cert.dns_provider_id: @@ -388,6 +395,8 @@ class ACMEIssuerPlugin(IssuerPlugin): return cert def get_ordered_certificates(self, pending_certs): + if not self.acme: + self.acme = AcmeHandler() pending = [] certs = [] for pending_cert in pending_certs: @@ -470,6 +479,8 @@ class ACMEIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ + if not self.acme: + self.acme = AcmeHandler() authority = issuer_options.get('authority') create_immediately = issuer_options.get('create_immediately', False) acme_client, registration = self.acme.setup_acme_client(authority) diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index c2b5d83d..bbe155cd 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -1,7 +1,7 @@ - # This is just Python which means you can inherit and tweak settings import os + _basedir = os.path.abspath(os.path.dirname(__file__)) THREADS_PER_PAGE = 8 @@ -78,14 +78,12 @@ DIGICERT_API_KEY = 'api-key' DIGICERT_ORG_ID = 111111 DIGICERT_ROOT = "ROOT" - VERISIGN_URL = 'http://example.com' VERISIGN_PEM_PATH = '~/' VERISIGN_FIRST_NAME = 'Jim' VERISIGN_LAST_NAME = 'Bob' VERSIGN_EMAIL = 'jim@example.com' - ACME_AWS_ACCOUNT_NUMBER = '11111111111' ACME_PRIVATE_KEY = ''' @@ -180,6 +178,7 @@ ACME_URL = 'https://acme-v01.api.letsencrypt.org' ACME_EMAIL = 'jim@example.com' ACME_TEL = '4088675309' ACME_DIRECTORY_URL = 'https://acme-v01.api.letsencrypt.org' +ACME_DISABLE_AUTORESOLVE = True LDAP_AUTH = True LDAP_BIND_URI = 'ldap://localhost' diff --git a/lemur/tests/test_pending_certificates.py b/lemur/tests/test_pending_certificates.py index 567159e1..7accf7d9 100644 --- a/lemur/tests/test_pending_certificates.py +++ b/lemur/tests/test_pending_certificates.py @@ -2,11 +2,10 @@ import json import pytest +from lemur.pending_certificates.views import * # noqa from .vectors import CSR_STR, INTERMEDIATE_CERT_STR, VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, \ VALID_USER_HEADER_TOKEN, WILDCARD_CERT_STR -from lemur.pending_certificates.views import * # noqa - def test_increment_attempt(pending_certificate): from lemur.pending_certificates.service import increment_attempt @@ -17,7 +16,8 @@ def test_increment_attempt(pending_certificate): def test_create_pending_certificate(async_issuer_plugin, async_authority, user): from lemur.certificates.service import create - pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'], common_name='ACommonName') + pending_cert = create(authority=async_authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'], + common_name='ACommonName') assert pending_cert.external_id == '12345' From 39b76d18dce1511f5909d0fb67c375a14eeb5eff Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 28 Nov 2018 14:41:56 -0800 Subject: [PATCH 025/110] add countdown to async call --- lemur/certificates/service.py | 2 +- lemur/plugins/lemur_acme/plugin.py | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index d965192e..c9a2fa24 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -282,7 +282,7 @@ def create(**kwargs): pending_cert = database.session_query(PendingCertificate).get(cert.id) from lemur.common.celery import fetch_acme_cert if not current_app.config.get("ACME_DISABLE_AUTORESOLVE", False): - fetch_acme_cert.delay(pending_cert.id) + fetch_acme_cert.apply_async((pending_cert.id,), countdown=5) return cert diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 53d11935..62e647c4 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -333,12 +333,9 @@ class ACMEIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) - self.acme = None + self.acme = AcmeHandler() def get_dns_provider(self, type): - if not self.acme: - self.acme = AcmeHandler() - provider_types = { 'cloudflare': cloudflare, 'dyn': dyn, @@ -350,16 +347,12 @@ class ACMEIssuerPlugin(IssuerPlugin): return provider def get_all_zones(self, dns_provider): - if not self.acme: - self.acme = AcmeHandler() dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) return dns_provider_plugin.get_zones(account_number=account_number) def get_ordered_certificate(self, pending_cert): - if not self.acme: - self.acme = AcmeHandler() acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) if pending_cert.dns_provider_id: @@ -395,8 +388,6 @@ class ACMEIssuerPlugin(IssuerPlugin): return cert def get_ordered_certificates(self, pending_certs): - if not self.acme: - self.acme = AcmeHandler() pending = [] certs = [] for pending_cert in pending_certs: @@ -479,8 +470,6 @@ class ACMEIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ - if not self.acme: - self.acme = AcmeHandler() authority = issuer_options.get('authority') create_immediately = issuer_options.get('create_immediately', False) acme_client, registration = self.acme.setup_acme_client(authority) From a90154e0ae4f98b5ebf366478e2240b7b2cd2f31 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 29 Nov 2018 09:29:05 -0800 Subject: [PATCH 026/110] LetsEncrypt Celery Flow --- lemur/common/celery.py | 5 ++++- lemur/dns_providers/models.py | 3 ++- lemur/plugins/lemur_acme/plugin.py | 7 ++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b7f65886..f2a2f826 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -21,7 +21,10 @@ from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources -flask_app = create_app() +if current_app: + flask_app = current_app +else: + flask_app = create_app() def make_celery(app): diff --git a/lemur/dns_providers/models.py b/lemur/dns_providers/models.py index d48cd0d1..435a2398 100644 --- a/lemur/dns_providers/models.py +++ b/lemur/dns_providers/models.py @@ -23,7 +23,8 @@ class DnsProvider(db.Model): status = Column(String(length=128), nullable=True) options = Column(JSON, nullable=True) domains = Column(JSON, nullable=True) - certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id') + certificates = relationship("Certificate", backref='dns_provider', foreign_keys='Certificate.dns_provider_id', + lazy='dynamic') def __init__(self, name, description, provider_type, credentials): self.name = name diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 62e647c4..26ca8ffc 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -333,9 +333,10 @@ class ACMEIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) - self.acme = AcmeHandler() def get_dns_provider(self, type): + self.acme = AcmeHandler() + provider_types = { 'cloudflare': cloudflare, 'dyn': dyn, @@ -347,12 +348,14 @@ class ACMEIssuerPlugin(IssuerPlugin): return provider def get_all_zones(self, dns_provider): + self.acme = AcmeHandler() dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) return dns_provider_plugin.get_zones(account_number=account_number) def get_ordered_certificate(self, pending_cert): + self.acme = AcmeHandler() acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) if pending_cert.dns_provider_id: @@ -388,6 +391,7 @@ class ACMEIssuerPlugin(IssuerPlugin): return cert def get_ordered_certificates(self, pending_certs): + self.acme = AcmeHandler() pending = [] certs = [] for pending_cert in pending_certs: @@ -470,6 +474,7 @@ class ACMEIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ + self.acme = AcmeHandler() authority = issuer_options.get('authority') create_immediately = issuer_options.get('create_immediately', False) acme_client, registration = self.acme.setup_acme_client(authority) From 2a235fb0e24d31bcd0e09be5518c67dcb41ecd2f Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 30 Nov 2018 12:44:52 -0800 Subject: [PATCH 027/110] Prefer DNS provider with longest matching zone --- lemur/plugins/lemur_acme/plugin.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 26ca8ffc..66295ed2 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -215,12 +215,18 @@ class AcmeHandler(object): :return: dns_providers: List of DNS providers that have the correct zone. """ self.dns_providers_for_domain[domain] = [] + match_length = 0 for dns_provider in self.all_dns_providers: if not dns_provider.domains: continue for name in dns_provider.domains: if domain.endswith("." + name): - self.dns_providers_for_domain[domain].append(dns_provider) + if len(name) > match_length: + self.dns_providers_for_domain[domain] = [dns_provider] + match_length = len(name) + elif len(name) == match_length: + self.dns_providers_for_domain[domain].append(dns_provider) + return self.dns_providers_for_domain def finalize_authorizations(self, acme_client, authorizations): From e0ac7497348183d19e82afd4019651540ab922d6 Mon Sep 17 00:00:00 2001 From: Ronald Moesbergen Date: Thu, 6 Dec 2018 16:47:53 +0100 Subject: [PATCH 028/110] When parsing SAN's, ignore unknown san_types, because in some cases they can contain unparsable/serializable values, resulting in a TypeError(repr(o) + " is not JSON serializable") --- lemur/common/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/common/fields.py b/lemur/common/fields.py index 9a0198e9..5ab0c6f0 100644 --- a/lemur/common/fields.py +++ b/lemur/common/fields.py @@ -350,6 +350,7 @@ class SubjectAlternativeNameExtension(Field): value = value.dotted_string else: current_app.logger.warning('Unknown SubAltName type: {name}'.format(name=name)) + continue general_names.append({'nameType': name_type, 'value': value}) From c32e20b6fc44b08961dc1971ba0db9a6892eb846 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 6 Dec 2018 12:25:43 -0800 Subject: [PATCH 029/110] Fix notifications - Ensure that notifcation e-mails are sent appropriately --- lemur/notifications/messaging.py | 35 +++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index ddf8d875..cd88ebc8 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -8,24 +8,21 @@ .. moduleauthor:: Kevin Glisson """ -from itertools import groupby from collections import defaultdict +from datetime import timedelta +from itertools import groupby import arrow -from datetime import timedelta from flask import current_app - from sqlalchemy import and_ from lemur import database +from lemur.certificates.models import Certificate +from lemur.certificates.schemas import certificate_notification_output_schema +from lemur.common.utils import windowed_query from lemur.constants import FAILURE_METRIC_STATUS, SUCCESS_METRIC_STATUS from lemur.extensions import metrics, sentry -from lemur.common.utils import windowed_query - -from lemur.certificates.schemas import certificate_notification_output_schema -from lemur.certificates.models import Certificate from lemur.pending_certificates.schemas import pending_certificate_output_schema - from lemur.plugins import plugins from lemur.plugins.utils import get_plugin_option @@ -74,10 +71,11 @@ def get_eligible_certificates(exclude=None): notification_groups = [] for certificate in items: - notification = needs_notification(certificate) + notifications = needs_notification(certificate) - if notification: - notification_groups.append((notification, certificate)) + if notifications: + for notification in notifications: + notification_groups.append((notification, certificate)) # group by notification for notification, items in groupby(notification_groups, lambda x: x[0].label): @@ -133,11 +131,21 @@ def send_expiration_notifications(exclude): notification_data.append(cert_data) security_data.append(cert_data) + notification_recipient = get_plugin_option('recipients', notification.options) + if notification_recipient: + notification_recipient = notification_recipient.split(",") + if send_notification('expiration', notification_data, [owner], notification): success += 1 else: failure += 1 + if notification_recipient and owner != notification_recipient and security_email != notification_recipient: + if send_notification('expiration', notification_data, notification_recipient, notification): + success += 1 + else: + failure += 1 + if send_notification('expiration', security_data, security_email, notification): success += 1 else: @@ -228,6 +236,8 @@ def needs_notification(certificate): now = arrow.utcnow() days = (certificate.not_after - now).days + notifications = [] + for notification in certificate.notifications: if not notification.active or not notification.options: return @@ -248,4 +258,5 @@ def needs_notification(certificate): raise Exception("Invalid base unit for expiration interval: {0}".format(unit)) if days == interval: - return notification + notifications.append(notification) + return notifications From da87135e028e444ca991b3a603b84a697254cc6e Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 6 Dec 2018 12:28:22 -0800 Subject: [PATCH 030/110] update reqs --- requirements-dev.txt | 2 +- requirements-docs.txt | 2 +- requirements-tests.txt | 10 +++++----- requirements.txt | 16 ++++++++-------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 599a1af9..d74b07f9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ aspy.yaml==1.1.1 # via pre-commit bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit -certifi==2018.10.15 # via requests +certifi==2018.11.29 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer diff --git a/requirements-docs.txt b/requirements-docs.txt index a9c7a979..35ca4322 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -76,7 +76,7 @@ python-editor==1.0.3 pytz==2018.7 pyyaml==3.13 raven[flask]==6.9.0 -redis==3.0.1 +redis==2.10.6 requests-toolbelt==0.8.0 requests[security]==2.20.1 retrying==1.3.3 diff --git a/requirements-tests.txt b/requirements-tests.txt index 72dcbbdb..e328b38a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,17 +8,17 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.53 # via moto +boto3==1.9.60 # via moto boto==2.49.0 # via moto -botocore==1.12.53 # via boto3, moto, s3transfer -certifi==2018.10.15 # via requests +botocore==1.12.60 # via boto3, moto, s3transfer +certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 cryptography==2.4.2 # via moto -docker-pycreds==0.3.0 # via docker -docker==3.5.1 # via moto +docker-pycreds==0.4.0 # via docker +docker==3.6.0 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 diff --git a/requirements.txt b/requirements.txt index c881799e..fadcfe4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.28.0 +acme==0.29.1 alembic-autogenerate-enums==0.0.2 alembic==1.0.5 # via flask-migrate amqp==2.3.2 # via kombu @@ -13,12 +13,12 @@ arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko -billiard==3.5.0.4 # via celery +billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.53 -botocore==1.12.53 +boto3==1.9.60 +botocore==1.12.60 celery[redis]==4.2.1 -certifi==2018.10.15 +certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask @@ -46,7 +46,7 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.1 # via celery +kombu==4.2.2 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako @@ -61,7 +61,7 @@ psycopg2==2.7.6.1 pyasn1-modules==0.2.2 # via python-ldap pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi -pyjwt==1.6.4 +pyjwt==1.7.0 pynacl==1.3.0 # via paramiko pyopenssl==18.0.0 pyrfc3339==1.1 # via acme @@ -77,7 +77,7 @@ requests[security]==2.20.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.8 +sqlalchemy-utils==0.33.9 sqlalchemy==1.2.14 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests From 437d918cf795516142d5f7c6f0f385b2d4716bfa Mon Sep 17 00:00:00 2001 From: Wesley Hartford Date: Mon, 10 Dec 2018 12:04:16 -0800 Subject: [PATCH 031/110] Fix textarea and validation on destination page The destination configuration page did not previously support a textarea input as was supported on most other pages. The validation of string inputs was not being performed. This commit addresses both of those issues and corrects the validation expressions for the AWS and S3 destination plugins so that they continue to function. The SFTP destination plugin does not have any string validation. The Kubernetes plugin does not work at all as far as I can tell; there will be another PR in the coming days to address that. --- lemur/plugins/lemur_aws/plugin.py | 7 +++---- .../angular/destinations/destination/destination.tpl.html | 6 +++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index c563eac8..1c2607a5 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -163,7 +163,7 @@ class AWSDestinationPlugin(DestinationPlugin): 'name': 'accountNumber', 'type': 'str', 'required': True, - 'validation': '/^[0-9]{12,12}$/', + 'validation': '[0-9]{12}', 'helpMessage': 'Must be a valid AWS account number!', }, { @@ -279,14 +279,14 @@ class S3DestinationPlugin(ExportDestinationPlugin): 'name': 'bucket', 'type': 'str', 'required': True, - 'validation': '/^$|\s+/', + 'validation': '[0-9a-z.-]{3,63}', 'helpMessage': 'Must be a valid S3 bucket name!', }, { 'name': 'accountNumber', 'type': 'str', 'required': True, - 'validation': '/^[0-9]{12,12}$/', + 'validation': '[0-9]{12}', 'helpMessage': 'A valid AWS account number with permission to access S3', }, { @@ -308,7 +308,6 @@ class S3DestinationPlugin(ExportDestinationPlugin): 'name': 'prefix', 'type': 'str', 'required': False, - 'validation': '/^$|\s+/', 'helpMessage': 'Must be a valid S3 object prefix!', } ] diff --git a/lemur/static/app/angular/destinations/destination/destination.tpl.html b/lemur/static/app/angular/destinations/destination/destination.tpl.html index 1d240dbb..f2771b49 100644 --- a/lemur/static/app/angular/destinations/destination/destination.tpl.html +++ b/lemur/static/app/angular/destinations/destination/destination.tpl.html @@ -47,7 +47,9 @@ - + +
+

{{ item.helpMessage }}

From 060c78fd91241af1a638b570b769db133a9e7f04 Mon Sep 17 00:00:00 2001 From: Wesley Hartford Date: Mon, 10 Dec 2018 15:33:04 -0800 Subject: [PATCH 032/110] Fix Kubernetes Destination Plugin The Kubernetes plugin was broken. There were two major issues: * The server certificate was entered in a string input making it impossible (as far as I know) to enter a valid PEM certificate. * The base64 encoding calls were passing strings where bytes were expected. The fix to the first issue depends on #2218 and a change in the options structure. I've also included some improved input validation and logging. --- lemur/plugins/lemur_kubernetes/plugin.py | 92 +++++++++++++++--------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index ee466596..a640a677 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -11,12 +11,14 @@ .. moduleauthor:: Mikhail Khodorovskiy """ import base64 -import os -import urllib -import requests import itertools +import os -from lemur.certificates.models import Certificate +import requests +from flask import current_app + +from lemur.common.defaults import common_name +from lemur.common.utils import parse_certificate from lemur.plugins.bases import DestinationPlugin DEFAULT_API_VERSION = 'v1' @@ -26,21 +28,32 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): # _resolve_uri(k8s_base_uri, namespace, kind, name, api_ver=DEFAULT_API_VERSION) url = _resolve_uri(k8s_base_uri, namespace, kind) + current_app.logger.debug("K8S POST request URL: %s", url) create_resp = k8s_api.post(url, json=data) + current_app.logger.debug("K8S POST response: %s", create_resp) if 200 <= create_resp.status_code <= 299: return None - elif create_resp.json()['reason'] != 'AlreadyExists': - return create_resp.content + else: + json = create_resp.json() + if 'reason' in json: + if json['reason'] != 'AlreadyExists': + return create_resp.content + else: + return create_resp.content - update_resp = k8s_api.put(_resolve_uri(k8s_base_uri, namespace, kind, name), json=data) + url = _resolve_uri(k8s_base_uri, namespace, kind, name) + current_app.logger.debug("K8S PUT request URL: %s", url) + + update_resp = k8s_api.put(url, json=data) + current_app.logger.debug("K8S PUT response: %s", update_resp) if not 200 <= update_resp.status_code <= 299: return update_resp.content - return + return None def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,): @@ -61,6 +74,12 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V ])) +# Performs Base64 encoding of string to string using the base64.b64encode() function +# which encodes bytes to bytes. +def base64encode(string): + return base64.b64encode(string.encode()).decode() + + class KubernetesDestinationPlugin(DestinationPlugin): title = 'Kubernetes' slug = 'kubernetes-destination' @@ -74,28 +93,28 @@ class KubernetesDestinationPlugin(DestinationPlugin): 'name': 'kubernetesURL', 'type': 'str', 'required': True, - 'validation': '@(https?|http)://(-\.)?([^\s/?\.#-]+\.?)+(/[^\s]*)?$@iS', + 'validation': 'https?://[a-zA-Z0-9.-]+(?::[0-9]+)?', 'helpMessage': 'Must be a valid Kubernetes server URL!', }, { 'name': 'kubernetesAuthToken', 'type': 'str', 'required': True, - 'validation': '/^$|\s+/', + 'validation': '[0-9a-zA-Z-_.]+', 'helpMessage': 'Must be a valid Kubernetes server Token!', }, { 'name': 'kubernetesServerCertificate', - 'type': 'str', + 'type': 'textarea', 'required': True, - 'validation': '/^$|\s+/', + 'validation': '-----BEGIN CERTIFICATE-----[a-zA-Z0-9/+\\s\\r\\n]+-----END CERTIFICATE-----', 'helpMessage': 'Must be a valid Kubernetes server Certificate!', }, { 'name': 'kubernetesNamespace', 'type': 'str', 'required': True, - 'validation': '/^$|\s+/', + 'validation': '[a-z0-9]([-a-z0-9]*[a-z0-9])?', 'helpMessage': 'Must be a valid Kubernetes Namespace!', }, @@ -106,33 +125,38 @@ class KubernetesDestinationPlugin(DestinationPlugin): def upload(self, name, body, private_key, cert_chain, options, **kwargs): - k8_bearer = self.get_option('kubernetesAuthToken', options) - k8_cert = self.get_option('kubernetesServerCertificate', options) - k8_namespace = self.get_option('kubernetesNamespace', options) - k8_base_uri = self.get_option('kubernetesURL', options) + try: + k8_bearer = self.get_option('kubernetesAuthToken', options) + k8_cert = self.get_option('kubernetesServerCertificate', options) + k8_namespace = self.get_option('kubernetesNamespace', options) + k8_base_uri = self.get_option('kubernetesURL', options) - k8s_api = K8sSession(k8_bearer, k8_cert) + k8s_api = K8sSession(k8_bearer, k8_cert) - cert = Certificate(body=body) + cn = common_name(parse_certificate(body)) - # in the future once runtime properties can be passed-in - use passed-in secret name - secret_name = 'certs-' + urllib.quote_plus(cert.name) + # in the future once runtime properties can be passed-in - use passed-in secret name + secret_name = 'certs-' + cn - err = ensure_resource(k8s_api, k8s_base_uri=k8_base_uri, namespace=k8_namespace, kind="secret", name=secret_name, data={ - 'apiVersion': 'v1', - 'kind': 'Secret', - 'metadata': { - 'name': secret_name, - }, - 'data': { - 'combined.pem': base64.b64encode(body + private_key), - 'ca.crt': base64.b64encode(cert_chain), - 'service.key': base64.b64encode(private_key), - 'service.crt': base64.b64encode(body), - } - }) + err = ensure_resource(k8s_api, k8s_base_uri=k8_base_uri, namespace=k8_namespace, kind="secret", name=secret_name, data={ + 'apiVersion': 'v1', + 'kind': 'Secret', + 'metadata': { + 'name': secret_name, + }, + 'data': { + 'combined.pem': base64encode('%s\n%s' % (body, private_key)), + 'ca.crt': base64encode(cert_chain), + 'service.key': base64encode(private_key), + 'service.crt': base64encode(body), + } + }) + except Exception as e: + current_app.logger.exception("Exception in upload") + raise e if err is not None: + current_app.logger.debug("Error deploying resource: %s", err) raise Exception("Error uploading secret: " + err) From a50d80992c41ff5ba4a6ac6248212fd2b5719c4f Mon Sep 17 00:00:00 2001 From: sirferl Date: Wed, 12 Dec 2018 12:45:48 +0100 Subject: [PATCH 033/110] updated query to ignore empty parameters --- lemur/certificates/cli.py | 52 +++++++++++++++------------------------ 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 7a46138c..c4a95187 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -238,17 +238,7 @@ def reissue(old_certificate_name, commit): if not old_cert: for certificate in get_all_pending_reissue(): - try: - request_reissue(certificate, commit) - except Exception as e: - sentry.captureException() - current_app.logger.exception( - "Error reissuing certificate: {}".format(certificate.name), exc_info=True) - print( - "[!] Failed to reissue certificates. Reason: {}".format( - e - ) - ) + request_reissue(certificate, commit) else: request_reissue(old_cert, commit) @@ -275,30 +265,31 @@ def query(fqdns, issuer, owner, expired): table = [] q = database.session_query(Certificate) + if issuer: + sub_query = database.session_query(Authority.id) \ + .filter(Authority.name.ilike('%{0}%'.format(issuer))) \ + .subquery() - sub_query = database.session_query(Authority.id) \ - .filter(Authority.name.ilike('%{0}%'.format(issuer))) \ - .subquery() - - q = q.filter( - or_( - Certificate.issuer.ilike('%{0}%'.format(issuer)), - Certificate.authority_id.in_(sub_query) + q = q.filter( + or_( + Certificate.issuer.ilike('%{0}%'.format(issuer)), + Certificate.authority_id.in_(sub_query) + ) ) - ) - - q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner))) + if owner: + q = q.filter(Certificate.owner.ilike('%{0}%'.format(owner))) if not expired: q = q.filter(Certificate.expired == False) # noqa - for f in fqdns.split(','): - q = q.filter( - or_( - Certificate.cn.ilike('%{0}%'.format(f)), - Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f))) + if fqdns: + for f in fqdns.split(','): + q = q.filter( + or_( + Certificate.cn.ilike('%{0}%'.format(f)), + Certificate.domains.any(Domain.name.ilike('%{0}%'.format(f))) + ) ) - ) for c in q.all(): table.append([c.id, c.name, c.owner, c.issuer]) @@ -373,10 +364,7 @@ def check_revoked(): else: status = verify_string(cert.body, "") - if status is None: - cert.status = 'unknown' - else: - cert.status = 'valid' if status else 'revoked' + cert.status = 'valid' if status else 'revoked' except Exception as e: sentry.captureException() From bc621c14680b42ca3d98ebc083b973f3210621b9 Mon Sep 17 00:00:00 2001 From: Wesley Hartford Date: Wed, 12 Dec 2018 13:25:36 -0800 Subject: [PATCH 034/110] Improve the Kubernetes Destination plugin The plugin now supports loading details from local files rather than requiring them to be entered through the UI. This is especially relaent when Lemur is deployed on Kubernetes as the certificate, token, and current namespace will be injected into the pod. The location these details are injected are the defaults if no configuration details are supplied. The plugin now supports deploying the secret in three different formats: * Full - matches the formate used by the plugin prior to these changes. * TLS - creates a secret of type kubernetes.io/tls and includes the certificate chain and private key, this format is used by many kubernetes features. * Certificate - creates a secret containing only the certificate chain, suitable for use as trust authority where private keys should _NOT_ be deployed. The deployed secret can now have a name set through the configuration options; the setting allows the insertion of the placeholder '{common_name}' which will be replaced by the certificate's common name value. Debug level logging has been added. --- lemur/plugins/lemur_kubernetes/plugin.py | 187 ++++++++++++++++++----- 1 file changed, 149 insertions(+), 38 deletions(-) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index a640a677..25ce8757 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -25,7 +25,6 @@ DEFAULT_API_VERSION = 'v1' def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): - # _resolve_uri(k8s_base_uri, namespace, kind, name, api_ver=DEFAULT_API_VERSION) url = _resolve_uri(k8s_base_uri, namespace, kind) current_app.logger.debug("K8S POST request URL: %s", url) @@ -56,11 +55,12 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): return None -def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,): +def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION): api_group = 'api' if '/' in api_ver: api_group = 'apis' - return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + ('/' + namespace if namespace else '') + return '{base}/{api_group}/{api_ver}/namespaces'.format(base=k8s_base_uri, api_group=api_group, api_ver=api_ver) + ( + '/' + namespace if namespace else '') def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_VERSION): @@ -80,6 +80,35 @@ def base64encode(string): return base64.b64encode(string.encode()).decode() +def build_secret(secret_format, secret_name, body, private_key, cert_chain): + secret = { + 'apiVersion': 'v1', + 'kind': 'Secret', + 'type': 'Opaque', + 'metadata': { + 'name': secret_name, + } + } + if secret_format == 'Full': + secret['data'] = { + 'combined.pem': base64encode('%s\n%s' % (body, private_key)), + 'ca.crt': base64encode(cert_chain), + 'service.key': base64encode(private_key), + 'service.crt': base64encode(body), + } + if secret_format == 'TLS': + secret['type'] = 'kubernetes.io/tls' + secret['data'] = { + 'tls.crt': base64encode(cert_chain), + 'tls.key': base64encode(private_key) + } + if secret_format == 'Certificate': + secret['data'] = { + 'tls.crt': base64encode(cert_chain), + } + return secret + + class KubernetesDestinationPlugin(DestinationPlugin): title = 'Kubernetes' slug = 'kubernetes-destination' @@ -89,35 +118,81 @@ class KubernetesDestinationPlugin(DestinationPlugin): author_url = 'https://github.com/mik373/lemur' options = [ + { + 'name': 'secretNameFormat', + 'type': 'str', + 'required': False, + # Validation is difficult. This regex is used by kubectl to validate secret names: + # [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)* + # Allowing the insertion of "{common_name}" (or any other such placeholder} + # at any point in the string proved very challenging and had a tendency to + # cause my browser to hang. The specified expression will allow any valid string + # but will also accept many invalid strings. + 'validation': '(?:[a-z0-9.-]|\\{common_name\\})+', + 'helpMessage': 'Must be a valid secret name, possibly including "{common_name}"', + 'default': '{common_name}' + }, { 'name': 'kubernetesURL', 'type': 'str', - 'required': True, + 'required': False, 'validation': 'https?://[a-zA-Z0-9.-]+(?::[0-9]+)?', 'helpMessage': 'Must be a valid Kubernetes server URL!', + 'default': 'https://kubernetes.default' }, { 'name': 'kubernetesAuthToken', 'type': 'str', - 'required': True, + 'required': False, 'validation': '[0-9a-zA-Z-_.]+', 'helpMessage': 'Must be a valid Kubernetes server Token!', }, + { + 'name': 'kubernetesAuthTokenFile', + 'type': 'str', + 'required': False, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/var/run/secrets/kubernetes.io/serviceaccount/token' + }, { 'name': 'kubernetesServerCertificate', 'type': 'textarea', - 'required': True, + 'required': False, 'validation': '-----BEGIN CERTIFICATE-----[a-zA-Z0-9/+\\s\\r\\n]+-----END CERTIFICATE-----', 'helpMessage': 'Must be a valid Kubernetes server Certificate!', }, + { + 'name': 'kubernetesServerCertificateFile', + 'type': 'str', + 'required': False, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' + }, { 'name': 'kubernetesNamespace', 'type': 'str', - 'required': True, + 'required': False, 'validation': '[a-z0-9]([-a-z0-9]*[a-z0-9])?', 'helpMessage': 'Must be a valid Kubernetes Namespace!', }, - + { + 'name': 'kubernetesNamespaceFile', + 'type': 'str', + 'required': False, + 'validation': '(/[^/]+)+', + 'helpMessage': 'Must be a valid file path!', + 'default': '/var/run/secrets/kubernetes.io/serviceaccount/namespace' + }, + { + 'name': 'secretFormat', + 'type': 'select', + 'required': True, + 'available': ['Full', 'TLS', 'Certificate'], + 'helpMessage': 'The type of Secret to create.', + 'default': 'Full' + } ] def __init__(self, *args, **kwargs): @@ -126,31 +201,31 @@ class KubernetesDestinationPlugin(DestinationPlugin): def upload(self, name, body, private_key, cert_chain, options, **kwargs): try: - k8_bearer = self.get_option('kubernetesAuthToken', options) - k8_cert = self.get_option('kubernetesServerCertificate', options) - k8_namespace = self.get_option('kubernetesNamespace', options) k8_base_uri = self.get_option('kubernetesURL', options) + secret_format = self.get_option('secretFormat', options) - k8s_api = K8sSession(k8_bearer, k8_cert) + k8s_api = K8sSession( + self.k8s_bearer(options), + self.k8s_cert(options) + ) cn = common_name(parse_certificate(body)) - # in the future once runtime properties can be passed-in - use passed-in secret name - secret_name = 'certs-' + cn + secret_name_format = self.get_option('secretNameFormat', options) + + secret_name = secret_name_format.format(common_name=cn) + + secret = build_secret(secret_format, secret_name, body, private_key, cert_chain) + + err = ensure_resource( + k8s_api, + k8s_base_uri=k8_base_uri, + namespace=self.k8s_namespace(options), + kind="secret", + name=secret_name, + data=secret + ) - err = ensure_resource(k8s_api, k8s_base_uri=k8_base_uri, namespace=k8_namespace, kind="secret", name=secret_name, data={ - 'apiVersion': 'v1', - 'kind': 'Secret', - 'metadata': { - 'name': secret_name, - }, - 'data': { - 'combined.pem': base64encode('%s\n%s' % (body, private_key)), - 'ca.crt': base64encode(cert_chain), - 'service.key': base64encode(private_key), - 'service.crt': base64encode(body), - } - }) except Exception as e: current_app.logger.exception("Exception in upload") raise e @@ -159,27 +234,63 @@ class KubernetesDestinationPlugin(DestinationPlugin): current_app.logger.debug("Error deploying resource: %s", err) raise Exception("Error uploading secret: " + err) + def k8s_bearer(self, options): + bearer = self.get_option('kubernetesAuthToken', options) + if not bearer: + bearer_file = self.get_option('kubernetesAuthTokenFile', options) + with open(bearer_file, "r") as file: + bearer = file.readline() + if bearer: + current_app.logger.debug("Using token read from %s", bearer_file) + else: + raise Exception("Unable to locate token in options or from %s", bearer_file) + else: + current_app.logger.debug("Using token from options") + return bearer + + def k8s_cert(self, options): + cert_file = self.get_option('kubernetesServerCertificateFile', options) + cert = self.get_option('kubernetesServerCertificate', options) + if cert: + cert_file = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert') + with open(cert_file, "w") as text_file: + text_file.write(cert) + current_app.logger.debug("Using certificate from options") + else: + current_app.logger.debug("Using certificate from %s", cert_file) + return cert_file + + def k8s_namespace(self, options): + namespace = self.get_option('kubernetesNamespace', options) + if not namespace: + namespace_file = self.get_option('kubernetesNamespaceFile', options) + with open(namespace_file, "r") as file: + namespace = file.readline() + if namespace: + current_app.logger.debug("Using namespace %s from %s", namespace, namespace_file) + else: + raise Exception("Unable to locate namespace in options or from %s", namespace_file) + else: + current_app.logger.debug("Using namespace %s from options", namespace) + return namespace + class K8sSession(requests.Session): - def __init__(self, bearer, cert): + def __init__(self, bearer, cert_file): super(K8sSession, self).__init__() self.headers.update({ 'Authorization': 'Bearer %s' % bearer }) - k8_ca = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'k8.cert') + self.verify = cert_file - with open(k8_ca, "w") as text_file: - text_file.write(cert) - - self.verify = k8_ca - - def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=30, allow_redirects=True, proxies=None, - hooks=None, stream=None, verify=None, cert=None, json=None): + def request(self, method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, + timeout=30, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, + json=None): """ This method overrides the default timeout to be 10s. """ - return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, - verify, cert, json) + return super(K8sSession, self).request(method, url, params, data, headers, cookies, files, auth, timeout, + allow_redirects, proxies, hooks, stream, verify, cert, json) From b35d494f2d6d6fc54498e88192555ad580764e47 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 18 Dec 2018 12:29:12 -0800 Subject: [PATCH 035/110] Update requirements --- requirements-dev.txt | 9 ++++----- requirements-docs.txt | 18 +++++++++--------- requirements-tests.txt | 16 ++++++++-------- requirements.txt | 18 +++++++++--------- 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d74b07f9..7b427b20 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,9 +13,8 @@ chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 identify==1.1.7 # via pre-commit -idna==2.7 # via requests +idna==2.8 # via requests importlib-metadata==0.7 # via pre-commit -importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 @@ -23,12 +22,12 @@ pkginfo==1.4.2 # via twine pre-commit==1.12.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 -pygments==2.3.0 # via readme-renderer +pygments==2.3.1 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine requests-toolbelt==0.8.0 # via twine -requests==2.20.1 # via requests-toolbelt, twine -six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer +requests==2.21.0 # via requests-toolbelt, twine +six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.28.1 # via twine twine==1.12.1 diff --git a/requirements-docs.txt b/requirements-docs.txt index 35ca4322..3f036915 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.28.0 +acme==0.29.1 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.5 @@ -15,12 +15,12 @@ asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 -billiard==3.5.0.4 +billiard==3.5.0.5 blinker==1.4 -boto3==1.9.53 -botocore==1.12.53 +boto3==1.9.60 +botocore==1.12.60 celery[redis]==4.2.1 -certifi==2018.10.15 +certifi==2018.11.29 cffi==1.11.5 chardet==3.0.4 click==7.0 @@ -49,7 +49,7 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.1 +kombu==4.2.2 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 @@ -65,8 +65,8 @@ psycopg2==2.7.6.1 pyasn1-modules==0.2.2 pyasn1==0.4.4 pycparser==2.19 -pygments==2.3.0 # via sphinx -pyjwt==1.6.4 +pygments==2.3.1 # via sphinx +pyjwt==1.7.0 pynacl==1.3.0 pyopenssl==18.0.0 pyparsing==2.3.0 # via packaging @@ -87,7 +87,7 @@ sphinx-rtd-theme==0.4.2 sphinx==1.8.2 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx -sqlalchemy-utils==0.33.8 +sqlalchemy-utils==0.33.9 sqlalchemy==1.2.14 tabulate==0.8.2 urllib3==1.24.1 diff --git a/requirements-tests.txt b/requirements-tests.txt index e328b38a..59c626f7 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.60 # via moto +boto3==1.9.67 # via moto boto==2.49.0 # via moto -botocore==1.12.60 # via boto3, moto, s3transfer +botocore==1.12.67 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -22,11 +22,11 @@ docker==3.6.0 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==1.0.0 +faker==1.0.1 flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.17.1 # via python-jose -idna==2.7 # via cryptography, requests +idna==2.8 # via cryptography, requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore @@ -46,16 +46,16 @@ pycryptodome==3.7.2 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==4.0.1 +pytest==4.0.2 python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.7 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 -requests==2.20.1 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.4 # via moto +requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses +responses==0.10.5 # via moto s3transfer==0.1.13 # via boto3 -six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client +six==1.12.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker diff --git a/requirements.txt b/requirements.txt index fadcfe4b..7ee9a167 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,11 +12,11 @@ aniso8601==4.0.1 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 -bcrypt==3.1.4 # via flask-bcrypt, paramiko +bcrypt==3.1.5 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.60 -botocore==1.12.60 +boto3==1.9.67 +botocore==1.12.67 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl @@ -33,13 +33,13 @@ flask-cors==3.0.7 flask-mail==0.9.1 flask-migrate==2.3.1 flask-principal==0.4.0 -flask-restful==0.3.6 +flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -idna==2.7 # via cryptography, requests +idna==2.8 # via cryptography, requests inflection==0.3.1 itsdangerous==1.1.0 # via flask jinja2==2.10 @@ -61,7 +61,7 @@ psycopg2==2.7.6.1 pyasn1-modules==0.2.2 # via python-ldap pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi -pyjwt==1.7.0 +pyjwt==1.7.1 pynacl==1.3.0 # via paramiko pyopenssl==18.0.0 pyrfc3339==1.1 # via acme @@ -73,12 +73,12 @@ pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 redis==2.10.6 requests-toolbelt==0.8.0 # via acme -requests[security]==2.20.1 +requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 # via boto3 -six==1.11.0 +six==1.12.0 sqlalchemy-utils==0.33.9 -sqlalchemy==1.2.14 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.15 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests vine==1.1.4 # via amqp From e7313da03e5d234b8829c7981d654a6e04dfb6b3 Mon Sep 17 00:00:00 2001 From: Wesley Hartford Date: Tue, 18 Dec 2018 22:24:48 -0500 Subject: [PATCH 036/110] Minor changes for code review suggestions. --- lemur/plugins/lemur_kubernetes/plugin.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index a640a677..4601592a 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -35,14 +35,8 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): if 200 <= create_resp.status_code <= 299: return None - - else: - json = create_resp.json() - if 'reason' in json: - if json['reason'] != 'AlreadyExists': - return create_resp.content - else: - return create_resp.content + elif create_resp.json().get('reason', '') != 'AlreadyExists': + return create_resp.content url = _resolve_uri(k8s_base_uri, namespace, kind, name) current_app.logger.debug("K8S PUT request URL: %s", url) @@ -53,7 +47,7 @@ def ensure_resource(k8s_api, k8s_base_uri, namespace, kind, name, data): if not 200 <= update_resp.status_code <= 299: return update_resp.content - return None + return def _resolve_ns(k8s_base_uri, namespace, api_ver=DEFAULT_API_VERSION,): @@ -152,8 +146,8 @@ class KubernetesDestinationPlugin(DestinationPlugin): } }) except Exception as e: - current_app.logger.exception("Exception in upload") - raise e + current_app.logger.exception("Exception in upload: {}".format(e), exc_info=True) + raise if err is not None: current_app.logger.debug("Error deploying resource: %s", err) From fbf48316b1c39bdbbbb6a6a673be51770462edd2 Mon Sep 17 00:00:00 2001 From: Wesley Hartford Date: Tue, 18 Dec 2018 22:43:32 -0500 Subject: [PATCH 037/110] Minor changes for code review suggestions. --- lemur/plugins/lemur_kubernetes/plugin.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index b111ac2b..30b864eb 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -197,20 +197,14 @@ class KubernetesDestinationPlugin(DestinationPlugin): try: k8_base_uri = self.get_option('kubernetesURL', options) secret_format = self.get_option('secretFormat', options) - k8s_api = K8sSession( self.k8s_bearer(options), self.k8s_cert(options) ) - cn = common_name(parse_certificate(body)) - secret_name_format = self.get_option('secretNameFormat', options) - secret_name = secret_name_format.format(common_name=cn) - secret = build_secret(secret_format, secret_name, body, private_key, cert_chain) - err = ensure_resource( k8s_api, k8s_base_uri=k8_base_uri, @@ -225,7 +219,7 @@ class KubernetesDestinationPlugin(DestinationPlugin): raise if err is not None: - current_app.logger.debug("Error deploying resource: %s", err) + current_app.logger.error("Error deploying resource: %s", err) raise Exception("Error uploading secret: " + err) def k8s_bearer(self, options): From 0f2e30cdae07f154c5b5809dc6e5ccee1c5e2158 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Fri, 21 Dec 2018 12:06:52 +0200 Subject: [PATCH 038/110] Deduplicate rows before notification associations unique constraint migration --- lemur/migrations/versions/449c3d5c7299_.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lemur/migrations/versions/449c3d5c7299_.py b/lemur/migrations/versions/449c3d5c7299_.py index 1dcb7ab5..0bc30db1 100644 --- a/lemur/migrations/versions/449c3d5c7299_.py +++ b/lemur/migrations/versions/449c3d5c7299_.py @@ -21,6 +21,14 @@ COLUMNS = ["notification_id", "certificate_id"] def upgrade(): + connection = op.get_bind() + # Delete duplicate entries + connection.execute("""\ + DELETE FROM certificate_notification_associations WHERE ctid NOT IN ( + -- Select the first tuple ID for each (notification_id, certificate_id) combination and keep that + SELECT min(ctid) FROM certificate_notification_associations GROUP BY notification_id, certificate_id + ) + """) op.create_unique_constraint(CONSTRAINT_NAME, TABLE, COLUMNS) From 72f6fdb17d3ad0bba4796fdc668db739246aa36b Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Wed, 19 Dec 2018 17:59:48 +0200 Subject: [PATCH 039/110] Properly handle Unicode in issuer name sanitization If the point of sanitization is to get rid of all non-alphanumeric characters then Unicode characters should probably be forbidden too. We can re-use the same sanitization function as used for cert 'name' --- lemur/common/defaults.py | 38 +++++++++++++++++------------------- lemur/tests/conftest.py | 7 ++++++- lemur/tests/test_defaults.py | 32 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index e9bbc6e6..72e863c1 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -7,18 +7,21 @@ from lemur.extensions import sentry from lemur.constants import SAN_NAMING_TEMPLATE, DEFAULT_NAMING_TEMPLATE -def text_to_slug(value): - """Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters.""" +def text_to_slug(value, joiner='-'): + """ + Normalize a string to a "slug" value, stripping character accents and removing non-alphanum characters. + A series of non-alphanumeric characters is replaced with the joiner character. + """ # Strip all character accents: decompose Unicode characters and then drop combining chars. value = ''.join(c for c in unicodedata.normalize('NFKD', value) if not unicodedata.combining(c)) - # Replace all remaining non-alphanumeric characters with '-'. Multiple characters get collapsed into a single dash. - # Except, keep 'xn--' used in IDNA domain names as is. - value = re.sub(r'[^A-Za-z0-9.]+(? Date: Fri, 21 Dec 2018 12:33:47 -0800 Subject: [PATCH 040/110] Update plugin.py --- lemur/plugins/lemur_kubernetes/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index 8de155c3..30b864eb 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -73,6 +73,7 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V def base64encode(string): return base64.b64encode(string.encode()).decode() + def build_secret(secret_format, secret_name, body, private_key, cert_chain): secret = { 'apiVersion': 'v1', @@ -101,6 +102,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain): } return secret + class KubernetesDestinationPlugin(DestinationPlugin): title = 'Kubernetes' slug = 'kubernetes-destination' From 4ec8490c558de6de64098d778d91cc1f79035caf Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 00:04:13 +0100 Subject: [PATCH 041/110] Create Dockerfile --- docker/Dockerfile | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..60aa473e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,66 @@ +FROM alpine:3.8 as builder + +ARG VERSION + +ENV VERSION master +#ENV VERSION 0.7.0 + +RUN apk --update add python3 + +RUN apk --update add --virtual build-dependencies \ + git \ + tar \ + curl \ + python3-dev \ + npm \ + bash \ + musl-dev \ + gcc \ + autoconf \ + automake \ + make \ + nasm \ + zlib-dev \ + postgresql-dev \ + libressl-dev \ + libffi-dev \ + cyrus-sasl-dev \ + openldap-dev + +#RUN git clone https://github.com/Netflix/lemur + +RUN mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 + +RUN ls -lha /opt/lemur/ + +WORKDIR /opt/lemur + +RUN pip3 install --upgrade pip + +RUN npm install --unsafe-perm +RUN pip3 install setuptools +RUN pip3 install -e . +RUN node_modules/.bin/gulp build +RUN node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) + +RUN apk del build-dependencies + +##################### + +RUN apk add --update libldap postgresql-client bash nginx supervisor + +RUN mkdir -p /run/nginx/ + +WORKDIR / + +COPY entrypoint / + +RUN chmod +x /entrypoint + +COPY lemur.py /root/.lemur/lemur.conf.py +COPY supervisor.conf / +COPY default.conf /etc/nginx/conf.d/ + +ENTRYPOINT ["/entrypoint"] + +CMD ["/usr/bin/supervisord","-c","supervisor.conf"] From fc6caecc0bbf93bf8b7614111ecb757f2a6eca51 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:37:09 +0100 Subject: [PATCH 042/110] Update Dockerfile --- docker/Dockerfile | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 60aa473e..54b517b8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,17 @@ FROM alpine:3.8 as builder -ARG VERSION - ENV VERSION master #ENV VERSION 0.7.0 +ENV uid 1337 +ENV gid 1337 +ENV user lemur +ENV group lemur + +#RUN adduser -D -S -u ${uid} ${user} -G ${group} + +RUN addgroup -S ${group} -g ${gid} && adduser -D -S ${user} -G ${group} -u ${uid} + RUN apk --update add python3 RUN apk --update add --virtual build-dependencies \ @@ -35,19 +42,29 @@ RUN ls -lha /opt/lemur/ WORKDIR /opt/lemur +RUN npm install --unsafe-perm + RUN pip3 install --upgrade pip -RUN npm install --unsafe-perm RUN pip3 install setuptools RUN pip3 install -e . + +#RUN node_modules/.bin/gulp build --urlContextPath=/arnold/foo + RUN node_modules/.bin/gulp build + +#RUN node_modules/.bin/gulp build -h + RUN node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) RUN apk del build-dependencies + ##################### -RUN apk add --update libldap postgresql-client bash nginx supervisor +RUN apk add --update libldap postgresql-client bash nginx supervisor curl + +#RUN python3 /opt/lemur/lemur/manage.py reset_password -u lemur RUN mkdir -p /run/nginx/ @@ -57,10 +74,18 @@ COPY entrypoint / RUN chmod +x /entrypoint -COPY lemur.py /root/.lemur/lemur.conf.py +#RUN mkdir -p /conf + +COPY lemur.py /conf/lemur.conf.py + COPY supervisor.conf / COPY default.conf /etc/nginx/conf.d/ +HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ + CMD curl --fail http://localhost:80/api/1/healthcheck |grep -q ok || exit 1 + ENTRYPOINT ["/entrypoint"] +#CMD ["python3","/lemur/lemur/manage.py","start","-b","0.0.0.0:8000"] + CMD ["/usr/bin/supervisord","-c","supervisor.conf"] From 7eb6617a2801bfccbe290898e64d16b7aba345be Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:37:30 +0100 Subject: [PATCH 043/110] Create supervisor.conf --- docker/supervisor.conf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docker/supervisor.conf diff --git a/docker/supervisor.conf b/docker/supervisor.conf new file mode 100644 index 00000000..e04e4002 --- /dev/null +++ b/docker/supervisor.conf @@ -0,0 +1,31 @@ +[supervisord] +nodaemon=true +user=root +logfile=/dev/stdout +logfile_maxbytes=0 +pidfile = /tmp/supervisord.pid + + +[program:lemur] +command=python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py start -b 0.0.0.0:8000 +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:nginx] +command=nginx -g "daemon off;" +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:dcron] +command=crond -f +user=root +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 From c25c703723a2eca125230570aa6ce406aa508d85 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:37:46 +0100 Subject: [PATCH 044/110] Create entrypoint --- docker/entrypoint | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docker/entrypoint diff --git a/docker/entrypoint b/docker/entrypoint new file mode 100644 index 00000000..386cdc08 --- /dev/null +++ b/docker/entrypoint @@ -0,0 +1,32 @@ +#!/bin/sh + +#echo $POSTGRES_USER +#echo $POSTGRES_PASSWORD +#echo $POSTGRES_HOST +#echo $POSTGRES_PORT +#echo $POSTGRES_DB + +export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" + +#echo $SQLALCHEMY_DATABASE_URI + +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;;' +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' + +# echo "from django.contrib.auth.models import User; User.objects.create_superuser('ronald', 'koko', 'koko')" | python /opt/lemur/lemur/manage.py shell + + +echo "running init" +python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init -p password +echo "done" + + +cron="${custom_cron:-"*/5 * * * *"}" + +echo "${cron} /opt/check/exec.sh" >> /etc/crontabs/root + +#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py notify expirations +#*/15 * * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py source sync -s all +#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py certificate check_revoked + +exec "$@" From 6d5782b44c832bfe5858cc5caab1c3b7d2315ae3 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:38:05 +0100 Subject: [PATCH 045/110] Create lemur.conf.py --- docker/lemur.conf.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docker/lemur.conf.py diff --git a/docker/lemur.conf.py b/docker/lemur.conf.py new file mode 100644 index 00000000..753b39af --- /dev/null +++ b/docker/lemur.conf.py @@ -0,0 +1,31 @@ +import os +_basedir = os.path.abspath(os.path.dirname(__file__)) + +CORS = os.environ.get("CORS") == "True" +debug = os.environ.get("DEBUG") == "True" + +SECRET_KEY = repr(os.environ.get('SECRET_KEY','Hrs8kCDNPuT9vtshsSWzlrYW+d+PrAXvg/HwbRE6M3vzSJTTrA/ZEw==')) + +LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET','YVKT6nNHnWRWk28Lra1OPxMvHTqg1ZXvAcO7bkVNSbrEuDQPABM0VQ==')) +LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS','Ls-qg9j3EMFHyGB_NL0GcQLI6622n9pSyGM_Pu0GdCo=')) + +LEMUR_WHITELISTED_DOMAINS = [] + +LEMUR_EMAIL = '' +LEMUR_SECURITY_TEAM_EMAIL = [] + + +LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY','')) +LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE','')) +LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION','')) +LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION','')) +LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT','')) + +ACTIVE_PROVIDERS = [] + +METRIC_PROVIDERS = [] + +LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) +LOG_FILE = str(os.environ.get('LOG_FILE','lemur.log')) + +SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') From 5567bb2eaafc5678bb14d88508e371cad1efd188 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:43:04 +0100 Subject: [PATCH 046/110] Update Dockerfile --- docker/Dockerfile | 80 +++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 55 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 54b517b8..d665da0e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,22 @@ -FROM alpine:3.8 as builder +FROM alpine:3.8 +ARG VERSION ENV VERSION master -#ENV VERSION 0.7.0 ENV uid 1337 ENV gid 1337 ENV user lemur ENV group lemur -#RUN adduser -D -S -u ${uid} ${user} -G ${group} +COPY entrypoint / +COPY lemur.conf.py /conf/lemur.conf.py +COPY supervisor.conf / +COPY default.conf /etc/nginx/conf.d/ -RUN addgroup -S ${group} -g ${gid} && adduser -D -S ${user} -G ${group} -u ${uid} - -RUN apk --update add python3 - -RUN apk --update add --virtual build-dependencies \ +RUN addgroup -S ${group} -g ${gid} && \ + adduser -D -S ${user} -G ${group} -u ${uid} && \ + apk --update add python3 libldap postgresql-client bash nginx supervisor curl && \ + apk --update add --virtual build-dependencies \ git \ tar \ curl \ @@ -32,60 +34,28 @@ RUN apk --update add --virtual build-dependencies \ libressl-dev \ libffi-dev \ cyrus-sasl-dev \ - openldap-dev - -#RUN git clone https://github.com/Netflix/lemur - -RUN mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 - -RUN ls -lha /opt/lemur/ - + openldap-dev && \ + mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ + pip3 install --upgrade pip && \ + pip3 install --upgrade setuptools && \ + chmod +x /entrypoint && \ + mkdir -p /run/nginx/ + WORKDIR /opt/lemur -RUN npm install --unsafe-perm - -RUN pip3 install --upgrade pip - -RUN pip3 install setuptools -RUN pip3 install -e . - -#RUN node_modules/.bin/gulp build --urlContextPath=/arnold/foo - -RUN node_modules/.bin/gulp build - -#RUN node_modules/.bin/gulp build -h - -RUN node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) - -RUN apk del build-dependencies - - -##################### - -RUN apk add --update libldap postgresql-client bash nginx supervisor curl - -#RUN python3 /opt/lemur/lemur/manage.py reset_password -u lemur - -RUN mkdir -p /run/nginx/ +RUN npm install --unsafe-perm && \ + pip3 install -e . && \ + node_modules/.bin/gulp build && \ + node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \ + apk del build-dependencies WORKDIR / -COPY entrypoint / - -RUN chmod +x /entrypoint - -#RUN mkdir -p /conf - -COPY lemur.py /conf/lemur.conf.py - -COPY supervisor.conf / -COPY default.conf /etc/nginx/conf.d/ - HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ - CMD curl --fail http://localhost:80/api/1/healthcheck |grep -q ok || exit 1 + CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1 + +USER lemur ENTRYPOINT ["/entrypoint"] -#CMD ["python3","/lemur/lemur/manage.py","start","-b","0.0.0.0:8000"] - CMD ["/usr/bin/supervisord","-c","supervisor.conf"] From 390157168546c2c0b32f69eba7ff786eee55448e Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:44:05 +0100 Subject: [PATCH 047/110] Update Dockerfile --- docker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d665da0e..0953b230 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,6 +8,7 @@ ENV gid 1337 ENV user lemur ENV group lemur + COPY entrypoint / COPY lemur.conf.py /conf/lemur.conf.py COPY supervisor.conf / @@ -39,7 +40,8 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ - mkdir -p /run/nginx/ + mkdir -p /run/nginx/ && \ + chown -R $user:$group /opt/lemur/ WORKDIR /opt/lemur From d8377ffc57c6a9e281223a72a775e0024d5b09bd Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:44:27 +0100 Subject: [PATCH 048/110] Update supervisor.conf --- docker/supervisor.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index e04e4002..b6355b6c 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -5,10 +5,9 @@ logfile=/dev/stdout logfile_maxbytes=0 pidfile = /tmp/supervisord.pid - [program:lemur] -command=python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py start -b 0.0.0.0:8000 -user=root +command=python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 +user=lemur stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 stderr_logfile=/dev/stderr From 4edda34e2dfb6868db4aa7053daea029a3cbcca2 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:47:27 +0100 Subject: [PATCH 049/110] Update entrypoint --- docker/entrypoint | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index 386cdc08..a3b4e20c 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,32 +1,20 @@ #!/bin/sh -#echo $POSTGRES_USER -#echo $POSTGRES_PASSWORD -#echo $POSTGRES_HOST -#echo $POSTGRES_PORT -#echo $POSTGRES_DB - export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" -#echo $SQLALCHEMY_DATABASE_URI - PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;;' PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' -# echo "from django.contrib.auth.models import User; User.objects.create_superuser('ronald', 'koko', 'koko')" | python /opt/lemur/lemur/manage.py shell - - -echo "running init" +echo "Running init" python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init -p password -echo "done" +echo "Done" +cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" +cron_sync="${CRON_SYNC:-"*/15 * * * *"}" +cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" -cron="${custom_cron:-"*/5 * * * *"}" - -echo "${cron} /opt/check/exec.sh" >> /etc/crontabs/root - -#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py notify expirations -#*/15 * * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py source sync -s all -#0 22 * * * lemur export LEMUR_CONF=/Users/me/.lemur/lemur.conf.py; python3 /opt/lemur/lemur/manage.py certificate check_revoked +echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root +echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root +echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root exec "$@" From ce634bfd08d91069699a3f1f208cf5899ab3f4f3 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 21:49:03 +0100 Subject: [PATCH 050/110] Create default.conf --- docker/default.conf | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docker/default.conf diff --git a/docker/default.conf b/docker/default.conf new file mode 100644 index 00000000..d71a93d3 --- /dev/null +++ b/docker/default.conf @@ -0,0 +1,26 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 80; + access_log /dev/stdout; + error_log /dev/stderr; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} From f8008e8614cdc35f62f42de00ba1c356b29999f0 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:01:28 +0100 Subject: [PATCH 051/110] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0953b230..0befdc57 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -56,7 +56,7 @@ WORKDIR / HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1 -USER lemur +USER root ENTRYPOINT ["/entrypoint"] From 58296cff5aa3b0d75a353a9c95c735678db2a4b6 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:25:11 +0100 Subject: [PATCH 052/110] Update entrypoint --- docker/entrypoint | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index a3b4e20c..eced8695 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -2,19 +2,28 @@ export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" -PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;;' +PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' + +echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' +echo "Done" echo "Running init" -python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init -p password +python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init +echo "Done" + +echo "Creating user" +echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell echo "Done" cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" +echo "Populating crontab" echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root +echo "Done" exec "$@" From 60b84a29b515639bf076a60d5e345adea5f84aaa Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:28:02 +0100 Subject: [PATCH 053/110] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0befdc57..e3bb4552 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ ENV group lemur COPY entrypoint / -COPY lemur.conf.py /conf/lemur.conf.py +COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / COPY default.conf /etc/nginx/conf.d/ From 692671a5431d2db17d2cf8d8f7b1c0503f0ed604 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:43:55 +0100 Subject: [PATCH 054/110] Update entrypoint --- docker/entrypoint | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index eced8695..2b275e60 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -8,13 +8,21 @@ echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo "Done" +# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then +# echo "Creating config" +# https://github.com/Netflix/lemur/issues/2257 +# python3 /opt/lemur/lemur/manage.py create_config +# echo "Done" +# fi + echo "Running init" python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init echo "Done" -echo "Creating user" -echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell -echo "Done" +# echo "Creating user" +# https://github.com/Netflix/lemur/issues/ +# echo "something that will create user" | python3 /opt/lemur/lemur/manage.py shell +# echo "Done" cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" From a4ce379bced46a095f95c29c03ff9aae832afa05 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:46:41 +0100 Subject: [PATCH 055/110] Update lemur.conf.py --- docker/lemur.conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/lemur.conf.py b/docker/lemur.conf.py index 753b39af..a5f7e8b6 100644 --- a/docker/lemur.conf.py +++ b/docker/lemur.conf.py @@ -26,6 +26,6 @@ ACTIVE_PROVIDERS = [] METRIC_PROVIDERS = [] LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG')) -LOG_FILE = str(os.environ.get('LOG_FILE','lemur.log')) +LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log')) SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur') From 2ae6c3a7147bcd23175932ac7bcd057d99ed48b2 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:48:28 +0100 Subject: [PATCH 056/110] Update Dockerfile --- docker/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e3bb4552..c2cc805f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,12 +36,15 @@ RUN addgroup -S ${group} -g ${gid} && \ libffi-dev \ cyrus-sasl-dev \ openldap-dev && \ - mkdir -p /opt/lemur && curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ + mkdir -p /opt/lemur /home/lemur/.lemur/ && \ + curl -sSL https://github.com/Netflix/lemur/archive/$VERSION.tar.gz | tar xz -C /opt/lemur --strip-components=1 && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ mkdir -p /run/nginx/ && \ - chown -R $user:$group /opt/lemur/ + touch /home/lemur/.lemur/lemur.log && \ + chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ + ln -s /home/lemur/.lemur/lemur.log /dev/stdout WORKDIR /opt/lemur From 7348fd37e86e5276cfe67a31f8693deafdf672d3 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:50:22 +0100 Subject: [PATCH 057/110] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c2cc805f..8305cdd5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,7 +44,7 @@ RUN addgroup -S ${group} -g ${gid} && \ mkdir -p /run/nginx/ && \ touch /home/lemur/.lemur/lemur.log && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ - ln -s /home/lemur/.lemur/lemur.log /dev/stdout + ln -s /dev/stdout /home/lemur/.lemur/lemur.log WORKDIR /opt/lemur From 97f6cdccfcd84848f9ca1f2de8df9bf03645010a Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 22:58:06 +0100 Subject: [PATCH 058/110] Update Dockerfile --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8305cdd5..d3d0d78b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,7 +42,6 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ mkdir -p /run/nginx/ && \ - touch /home/lemur/.lemur/lemur.log && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ ln -s /dev/stdout /home/lemur/.lemur/lemur.log From d5d42415013f52322f54d632cf11474ad356af7f Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:20:29 +0100 Subject: [PATCH 059/110] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 2b275e60..3604fce5 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -16,7 +16,7 @@ echo "Done" # fi echo "Running init" -python3 /opt/lemur/lemur/manage.py -c /conf/lemur.conf.py init +su lemur -c "python3 /opt/lemur/lemur/manage.py init" echo "Done" # echo "Creating user" From abd29f8462211f1f48e7b34991fc6ebc671973b1 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:53:39 +0100 Subject: [PATCH 060/110] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 3604fce5..0b39bfed 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,6 +1,6 @@ #!/bin/sh -export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" +echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB' >> /etc/environment PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' From ba20c0742083a4de25a319ad0387a8e40c604a0e Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:54:31 +0100 Subject: [PATCH 061/110] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 0b39bfed..3604fce5 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,6 +1,6 @@ #!/bin/sh -echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB' >> /etc/environment +export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' From e488c0ddcf8c4ff4c7a126e661673758c0132ea8 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Sun, 30 Dec 2018 23:57:14 +0100 Subject: [PATCH 062/110] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d3d0d78b..546e325e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,7 +16,7 @@ COPY default.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ - apk --update add python3 libldap postgresql-client bash nginx supervisor curl && \ + apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata bash && \ apk --update add --virtual build-dependencies \ git \ tar \ From aefdead50a95b35a7b852f5e7cd1a4b7befe3e67 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 00:04:58 +0100 Subject: [PATCH 063/110] Update entrypoint --- docker/entrypoint | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/entrypoint b/docker/entrypoint index 3604fce5..d0d8ab8b 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -17,6 +17,7 @@ echo "Done" echo "Running init" su lemur -c "python3 /opt/lemur/lemur/manage.py init" +#export LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py ; python3 /opt/lemur/lemur/manage.py init echo "Done" # echo "Creating user" From 25c4672845088e1324caa23e577796b5cd763842 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 10:41:19 +0100 Subject: [PATCH 064/110] Update supervisor.conf --- docker/supervisor.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index b6355b6c..311d997f 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -6,7 +6,7 @@ logfile_maxbytes=0 pidfile = /tmp/supervisord.pid [program:lemur] -command=python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 +command=/usr/bin/python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 user=lemur stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 @@ -14,7 +14,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:nginx] -command=nginx -g "daemon off;" +command=/usr/sbin/nginx -g "daemon off;" user=root stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 @@ -22,7 +22,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:dcron] -command=crond -f +command=/usr/sbin/crond -f user=root stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 From 239acb5f95a2b0fc6a4e7ffeb4bb514f6f3ac401 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 12:49:21 +0100 Subject: [PATCH 065/110] Update supervisor.conf --- docker/supervisor.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index 311d997f..185b07d1 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -1,4 +1,5 @@ [supervisord] +environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py nodaemon=true user=root logfile=/dev/stdout @@ -6,8 +7,9 @@ logfile_maxbytes=0 pidfile = /tmp/supervisord.pid [program:lemur] -command=/usr/bin/python3 /opt/lemur/lemur/manage.py start -b 0.0.0.0:8000 +command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000 user=lemur +directory=/opt/lemur/lemur stdout_logfile=/dev/stdout stdout_logfile_maxbytes = 0 stderr_logfile=/dev/stderr @@ -21,7 +23,7 @@ stdout_logfile_maxbytes = 0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:dcron] +[program:cron] command=/usr/sbin/crond -f user=root stdout_logfile=/dev/stdout From ca6f2b782b03f8c1f8a65a1b73507108d6a222de Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 12:52:07 +0100 Subject: [PATCH 066/110] Update supervisor.conf --- docker/supervisor.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/supervisor.conf b/docker/supervisor.conf index 185b07d1..fed01581 100644 --- a/docker/supervisor.conf +++ b/docker/supervisor.conf @@ -1,5 +1,4 @@ [supervisord] -environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py nodaemon=true user=root logfile=/dev/stdout @@ -7,6 +6,7 @@ logfile_maxbytes=0 pidfile = /tmp/supervisord.pid [program:lemur] +environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000 user=lemur directory=/opt/lemur/lemur From c94557f2edd8ddb006618e8095532c090aa1c10c Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 13:21:13 +0100 Subject: [PATCH 067/110] Update entrypoint --- docker/entrypoint | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/entrypoint b/docker/entrypoint index d0d8ab8b..dce3773d 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -8,6 +8,11 @@ echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo "Done" + +# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then +# openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE +# fi + # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" # https://github.com/Netflix/lemur/issues/2257 From 666f180482b17a578925566d118401d1390e63ae Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 13:21:30 +0100 Subject: [PATCH 068/110] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 546e325e..d2ae56a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,7 +16,7 @@ COPY default.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ - apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata bash && \ + apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \ apk --update add --virtual build-dependencies \ git \ tar \ From d6a374130cb033929c4c834b690af7a6d4fef229 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 13:33:58 +0100 Subject: [PATCH 069/110] Update entrypoint --- docker/entrypoint | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index dce3773d..82fe1780 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -9,9 +9,9 @@ PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTG echo "Done" -# if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then -# openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE -# fi +if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then + openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" +fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" From 341756d7c0fde73c58e9970393067fc1d79b74de Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 14:07:56 +0100 Subject: [PATCH 070/110] Update entrypoint --- docker/entrypoint | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index 82fe1780..1c895b16 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -8,10 +8,12 @@ echo "Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo "Done" - -if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" -fi +if [ -z ${SKIP_SSL} ]; then + if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then + openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" + fi + cp default.conf default_ssl.conf +then # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" From 6b1d2bfb60578dabbc390a64b0f7efc74834b475 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 14:55:13 +0100 Subject: [PATCH 071/110] Create default-ssl.conf --- docker/default-ssl.conf | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docker/default-ssl.conf diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf new file mode 100644 index 00000000..8b791c45 --- /dev/null +++ b/docker/default-ssl.conf @@ -0,0 +1,31 @@ +add_header X-Frame-Options DENY; +add_header X-Content-Type-Options nosniff; +add_header X-XSS-Protection "1; mode=block"; + +server { + listen 443; + server_name _; + access_log /dev/stdout; + error_log /dev/stderr; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers HIGH:!aNULL:!MD5; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} From 542e9539199d4c3a51c77ee9911e28cff7afcf90 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Wed, 20 Jun 2018 18:42:34 +0300 Subject: [PATCH 072/110] Check that stored private keys match certificates This is done in two places: * Certificate import validator -- throws validation errors. * Certificate model constructor -- to ensure integrity of Lemur's data even when issuer plugins or other code paths have bugs. --- lemur/certificates/models.py | 14 ++++- lemur/certificates/schemas.py | 26 ++++++++-- lemur/common/utils.py | 15 ++++++ lemur/common/validators.py | 33 ++++++------ lemur/tests/conftest.py | 9 +++- lemur/tests/factories.py | 5 ++ lemur/tests/test_certificates.py | 88 ++++++++++++++++++++++++++++++-- lemur/tests/test_validators.py | 22 ++++++-- 8 files changed, 181 insertions(+), 31 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index e2ac2cba..3eaba746 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -19,7 +19,7 @@ from sqlalchemy.sql.expression import case, extract from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property -from lemur.common import defaults, utils +from lemur.common import defaults, utils, validators from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS from lemur.database import db from lemur.domains.models import Domain @@ -186,6 +186,18 @@ class Certificate(db.Model): for domain in defaults.domains(cert): self.domains.append(Domain(name=domain)) + # Check integrity before saving anything into the database. + # For user-facing API calls, validation should also be done in schema validators. + self.check_integrity() + + def check_integrity(self): + """ + Integrity checks: Does the cert have a matching private key? + """ + if self.private_key: + validators.verify_private_key_match(utils.parse_private_key(self.private_key), self.parsed_cert, + error_class=AssertionError) + @cached_property def parsed_cert(self): assert self.body, "Certificate body not set" diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index bf18eac9..6b457086 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -10,7 +10,7 @@ from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError from lemur.authorities.schemas import AuthorityNestedOutputSchema -from lemur.common import validators, missing +from lemur.common import missing, utils, validators from lemur.common.fields import ArrowDateTime, Hex from lemur.common.schema import LemurInputSchema, LemurOutputSchema from lemur.constants import CERTIFICATE_KEY_TYPES @@ -242,8 +242,8 @@ class CertificateUploadInputSchema(CertificateCreationSchema): authority = fields.Nested(AssociatedAuthoritySchema, required=False) notify = fields.Boolean(missing=True) external_id = fields.String(missing=None, allow_none=True) - private_key = fields.String(validate=validators.private_key) - body = fields.String(required=True, validate=validators.public_certificate) + private_key = fields.String() + body = fields.String(required=True) chain = fields.String(validate=validators.public_certificate, missing=None, allow_none=True) # TODO this could be multiple certificates @@ -258,6 +258,26 @@ class CertificateUploadInputSchema(CertificateCreationSchema): if not data.get('private_key'): raise ValidationError('Destinations require private key.') + @validates_schema + def validate_cert_private_key(self, data): + cert = None + key = None + if data.get('body'): + try: + cert = utils.parse_certificate(data['body']) + except ValueError: + raise ValidationError("Public certificate presented is not valid.", field_names=['body']) + + if data.get('private_key'): + try: + key = utils.parse_private_key(data['private_key']) + except ValueError: + raise ValidationError("Private key presented is not valid.", field_names=['private_key']) + + if cert and key: + # Throws ValidationError + validators.verify_private_key_match(key, cert) + class CertificateExportInputSchema(LemurInputSchema): plugin = fields.Nested(PluginInputSchema) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 7ea9d7f2..62e59d69 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -13,6 +13,7 @@ import sqlalchemy from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa, ec +from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask_restful.reqparse import RequestParser from sqlalchemy import and_, func @@ -52,6 +53,20 @@ def parse_certificate(body): return x509.load_pem_x509_certificate(body, default_backend()) +def parse_private_key(private_key): + """ + Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm). + + Raises ValueError for an invalid string. + + :param private_key: String containing PEM private key + """ + if isinstance(private_key, str): + private_key = private_key.encode('utf8') + + return load_pem_private_key(private_key, password=None, backend=default_backend()) + + def parse_csr(csr): """ Helper function that parses a CSR. diff --git a/lemur/common/validators.py b/lemur/common/validators.py index 47a94a30..90169553 100644 --- a/lemur/common/validators.py +++ b/lemur/common/validators.py @@ -2,14 +2,12 @@ import re from cryptography import x509 from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization from cryptography.x509 import NameOID from flask import current_app from marshmallow.exceptions import ValidationError from lemur.auth.permissions import SensitiveDomainPermission from lemur.common.utils import parse_certificate, is_weekend -from lemur.domains import service as domain_service def public_certificate(body): @@ -26,22 +24,6 @@ def public_certificate(body): raise ValidationError('Public certificate presented is not valid.') -def private_key(key): - """ - User to validate that a given string is a RSA private key - - :param key: - :return: :raise ValueError: - """ - try: - if isinstance(key, bytes): - serialization.load_pem_private_key(key, None, backend=default_backend()) - else: - serialization.load_pem_private_key(key.encode('utf-8'), None, backend=default_backend()) - except Exception: - raise ValidationError('Private key presented is not valid.') - - def common_name(value): """If the common name could be a domain name, apply domain validation rules.""" # Common name could be a domain name, or a human-readable name of the subject (often used in CA names or client @@ -66,6 +48,9 @@ def sensitive_domain(domain): raise ValidationError('Domain {0} does not match whitelisted domain patterns. ' 'Contact an administrator to issue the certificate.'.format(domain)) + # Avoid circular import. + from lemur.domains import service as domain_service + if any(d.sensitive for d in domain_service.get_by_name(domain)): raise ValidationError('Domain {0} has been marked as sensitive. ' 'Contact an administrator to issue the certificate.'.format(domain)) @@ -141,3 +126,15 @@ def dates(data): raise ValidationError('Validity end must not be after {0}'.format(data['authority'].authority_certificate.not_after)) return data + + +def verify_private_key_match(key, cert, error_class=ValidationError): + """ + Checks that the supplied private key matches the certificate. + + :param cert: Parsed certificate + :param key: Parsed private key + :param error_class: Exception class to raise on error + """ + if key.public_key().public_numbers() != cert.public_key().public_numbers(): + raise error_class("Private key does not match certificate.") diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index d292e6d6..9a48eb94 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -15,7 +15,7 @@ from lemur.tests.vectors import SAN_CERT_KEY, INTERMEDIATE_KEY from .factories import ApiKeyFactory, AuthorityFactory, NotificationFactory, DestinationFactory, \ CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, \ - RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory + RotationPolicyFactory, PendingCertificateFactory, AsyncAuthorityFactory, CryptoAuthorityFactory def pytest_runtest_setup(item): @@ -91,6 +91,13 @@ def authority(session): return a +@pytest.fixture +def crypto_authority(session): + a = CryptoAuthorityFactory() + session.commit() + return a + + @pytest.fixture def async_authority(session): a = AsyncAuthorityFactory() diff --git a/lemur/tests/factories.py b/lemur/tests/factories.py index cae2c354..3717c64d 100644 --- a/lemur/tests/factories.py +++ b/lemur/tests/factories.py @@ -168,6 +168,11 @@ class AsyncAuthorityFactory(AuthorityFactory): authority_certificate = SubFactory(CertificateFactory) +class CryptoAuthorityFactory(AuthorityFactory): + """Authority factory based on 'cryptography' plugin.""" + plugin = {'slug': 'cryptography-issuer'} + + class DestinationFactory(BaseFactory): """Destination factory.""" plugin_name = 'test-destination' diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 87416a7a..a1df1c0d 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -18,7 +18,7 @@ from lemur.domains.models import Domain from lemur.tests.vectors import VALID_ADMIN_API_TOKEN, VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKEN, CSR_STR, \ - INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY + INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY, ROOTCA_KEY, ROOTCA_CERT_STR def test_get_or_increase_name(session, certificate): @@ -365,6 +365,85 @@ def test_certificate_sensitive_name(client, authority, session, logged_in_user): assert errors['common_name'][0].startswith("Domain sensitive.example.com has been marked as sensitive") +def test_certificate_upload_schema_ok(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'name': 'Jane', + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'privateKey': SAN_CERT_KEY, + 'chain': INTERMEDIATE_CERT_STR, + 'external_id': '1234', + } + data, errors = CertificateUploadInputSchema().load(data) + assert not errors + + +def test_certificate_upload_schema_minimal(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + } + data, errors = CertificateUploadInputSchema().load(data) + assert not errors + + +def test_certificate_upload_schema_long_chain(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'chain': INTERMEDIATE_CERT_STR + '\n' + ROOTCA_CERT_STR + } + data, errors = CertificateUploadInputSchema().load(data) + assert not errors + + +def test_certificate_upload_schema_invalid_body(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': 'Hereby I certify that this is a valid body', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'body': ['Public certificate presented is not valid.']} + + +def test_certificate_upload_schema_invalid_pkey(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'owner': 'pwner@example.com', + 'body': SAN_CERT_STR, + 'privateKey': 'Look at me Im a private key!!111', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'private_key': ['Private key presented is not valid.']} + + +def test_certificate_upload_schema_invalid_chain(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'body': SAN_CERT_STR, + 'chain': 'CHAINSAW', + 'owner': 'pwner@example.com', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'chain': ['Public certificate presented is not valid.']} + + +def test_certificate_upload_schema_wrong_pkey(client): + from lemur.certificates.schemas import CertificateUploadInputSchema + data = { + 'body': SAN_CERT_STR, + 'privateKey': ROOTCA_KEY, + 'chain': INTERMEDIATE_CERT_STR, + 'owner': 'pwner@example.com', + } + data, errors = CertificateUploadInputSchema().load(data) + assert errors == {'_schema': ['Private key does not match certificate.']} + + def test_create_basic_csr(client): csr_config = dict( common_name='example.com', @@ -462,8 +541,11 @@ def test_create_certificate(issuer_plugin, authority, user): assert cert.name == 'ACustomName1' -def test_reissue_certificate(issuer_plugin, authority, certificate): +def test_reissue_certificate(issuer_plugin, crypto_authority, certificate, logged_in_user): from lemur.certificates.service import reissue_certificate + + # test-authority would return a mismatching private key, so use 'cryptography-issuer' plugin instead. + certificate.authority = crypto_authority new_cert = reissue_certificate(certificate) assert new_cert @@ -487,7 +569,7 @@ def test_import(user): assert str(cert.not_after) == '2047-12-31T22:00:00+00:00' assert str(cert.not_before) == '2017-12-31T22:00:00+00:00' assert cert.issuer == 'LemurTrustUnittestsClass1CA2018' - assert cert.name == 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333-2' + assert cert.name.startswith('SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231') cert = import_certificate(body=SAN_CERT_STR, chain=INTERMEDIATE_CERT_STR, private_key=SAN_CERT_KEY, owner='joe@example.com', name='ACustomName2', creator=user['user']) assert cert.name == 'ACustomName2' diff --git a/lemur/tests/test_validators.py b/lemur/tests/test_validators.py index 815b7c9d..c3d5357d 100644 --- a/lemur/tests/test_validators.py +++ b/lemur/tests/test_validators.py @@ -1,16 +1,28 @@ -import pytest from datetime import datetime -from .vectors import SAN_CERT_KEY + +import pytest from marshmallow.exceptions import ValidationError +from lemur.common.utils import parse_private_key +from lemur.common.validators import verify_private_key_match +from lemur.tests.vectors import INTERMEDIATE_CERT, SAN_CERT, SAN_CERT_KEY + def test_private_key(session): - from lemur.common.validators import private_key + parse_private_key(SAN_CERT_KEY) - private_key(SAN_CERT_KEY) + with pytest.raises(ValueError): + parse_private_key('invalid_private_key') + + +def test_validate_private_key(session): + key = parse_private_key(SAN_CERT_KEY) + + verify_private_key_match(key, SAN_CERT) with pytest.raises(ValidationError): - private_key('invalid_private_key') + # Wrong key for certificate + verify_private_key_match(key, INTERMEDIATE_CERT) def test_sub_alt_type(session): From 7fb0631ff025ebd09b7f95a8c68b90010cd32e23 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 15:37:19 +0100 Subject: [PATCH 073/110] Update entrypoint --- docker/entrypoint | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index 1c895b16..ebfa9bfa 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,18 +1,27 @@ #!/bin/sh -export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" +if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];the + echo " # Vars not set" + exit 1 +fi + +export POSTGRES_PORT="${POSTGRES_PORT:-5432}" + +echo 'export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"' >> /etc/profile + +source /etc/profile PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;' -echo "Create Postgres trgm extension" +echo " # Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' -echo "Done" +echo " # Done" if [ -z ${SKIP_SSL} ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=FAKE" + openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=FAKE/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi - cp default.conf default_ssl.conf + mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf then # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then @@ -22,10 +31,9 @@ then # echo "Done" # fi -echo "Running init" +echo " # Running init" su lemur -c "python3 /opt/lemur/lemur/manage.py init" -#export LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py ; python3 /opt/lemur/lemur/manage.py init -echo "Done" +echo " # Done" # echo "Creating user" # https://github.com/Netflix/lemur/issues/ @@ -36,10 +44,10 @@ cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" -echo "Populating crontab" +echo " # Populating crontab" echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root -echo "Done" +echo " # Done" exec "$@" From 728be37de9a969f164de3f750efece77e9c43938 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 15:37:48 +0100 Subject: [PATCH 074/110] Update Dockerfile --- docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d2ae56a3..b105b1fb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,6 +13,7 @@ COPY entrypoint / COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / COPY default.conf /etc/nginx/conf.d/ +COPY default-ssl.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ @@ -41,7 +42,7 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade pip && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ - mkdir -p /run/nginx/ && \ + mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ ln -s /dev/stdout /home/lemur/.lemur/lemur.log From 4faedf3e5b8280161169c488e89337fcc3ee2683 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 16:58:51 +0100 Subject: [PATCH 075/110] Update entrypoint --- docker/entrypoint | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index ebfa9bfa..f97e2cdb 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,6 +1,6 @@ #!/bin/sh -if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];the +if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then echo " # Vars not set" exit 1 fi @@ -22,7 +22,7 @@ if [ -z ${SKIP_SSL} ]; then openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=FAKE/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf -then +fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then # echo "Creating config" From 809ca0fcfe28198aae8b28f521fd0a2ee88b5494 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 17:13:31 +0100 Subject: [PATCH 076/110] Update Dockerfile --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b105b1fb..8ebb5241 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,8 +43,7 @@ RUN addgroup -S ${group} -g ${gid} && \ pip3 install --upgrade setuptools && \ chmod +x /entrypoint && \ mkdir -p /run/nginx/ /etc/nginx/ssl/ && \ - chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \ - ln -s /dev/stdout /home/lemur/.lemur/lemur.log + chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ WORKDIR /opt/lemur From 628aaf2748a46fc302fc73a61149ec4c2c9629a5 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 17:36:52 +0100 Subject: [PATCH 077/110] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index f97e2cdb..b2850963 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -19,7 +19,7 @@ echo " # Done" if [ -z ${SKIP_SSL} ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=FAKE/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" + openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf fi From c0f6e5a134274a3fa329645738755c29a27e2e04 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 18:03:39 +0100 Subject: [PATCH 078/110] Update default-ssl.conf --- docker/default-ssl.conf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf index 8b791c45..2235b88d 100644 --- a/docker/default-ssl.conf +++ b/docker/default-ssl.conf @@ -2,6 +2,30 @@ add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; +server { + listen 80; + server_name _; + access_log /dev/stdout; + error_log /dev/stderr; + + location /api { + proxy_pass http://127.0.0.1:8000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + root /opt/lemur/lemur/static/dist; + include mime.types; + index index.html; + } + +} + server { listen 443; server_name _; From 918af0873f8ba4102b0a5283f4c2f140e7a2508b Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 18:35:17 +0100 Subject: [PATCH 079/110] Update default-ssl.conf --- docker/default-ssl.conf | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf index 2235b88d..8b791c45 100644 --- a/docker/default-ssl.conf +++ b/docker/default-ssl.conf @@ -2,30 +2,6 @@ add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; -server { - listen 80; - server_name _; - access_log /dev/stdout; - error_log /dev/stderr; - - location /api { - proxy_pass http://127.0.0.1:8000; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - proxy_redirect off; - proxy_buffering off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location / { - root /opt/lemur/lemur/static/dist; - include mime.types; - index index.html; - } - -} - server { listen 443; server_name _; From ff0dbdcc5a1b1f2fefcb2fceab3dd6f695ab0dff Mon Sep 17 00:00:00 2001 From: Lukas M Date: Mon, 31 Dec 2018 18:36:02 +0100 Subject: [PATCH 080/110] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index b2850963..565c0fd6 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -1,7 +1,7 @@ #!/bin/sh if [ -z "${POSTGRES_USER}" ] || [ -z "${POSTGRES_PASSWORD}" ] || [ -z "${POSTGRES_HOST}" ] || [ -z "${POSTGRES_DB}" ];then - echo " # Vars not set" + echo "Database vars not set" exit 1 fi From 3cc63c6618846bc1e15b56458c8ce5aeca247641 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:05:45 +0100 Subject: [PATCH 081/110] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index 565c0fd6..d7ace70a 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -17,7 +17,7 @@ echo " # Create Postgres trgm extension" PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION pg_trgm;' echo " # Done" -if [ -z ${SKIP_SSL} ]; then +if [ -z "${SKIP_SSL}" ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi From 0d0c295f82705a8173a4530f3b9393898bfe9c37 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:33:49 +0100 Subject: [PATCH 082/110] Update entrypoint --- docker/entrypoint | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index d7ace70a..18ab0da5 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -42,12 +42,12 @@ echo " # Done" cron_notify="${CRON_NOTIFY:-"0 22 * * *"}" cron_sync="${CRON_SYNC:-"*/15 * * * *"}" -cron_check_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" +cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}" echo " # Populating crontab" -echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" >> /etc/crontabs/root -echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" >> /etc/crontabs/root -echo "${cron_check_revoked} lemur /opt/lemur/lemur/manage.py certificate check_revoked" >> /etc/crontabs/root +echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify +echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync +echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked echo " # Done" exec "$@" From bb4b781d246297e298143c9153e10088d0d8660d Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:46:56 +0100 Subject: [PATCH 083/110] Update entrypoint --- docker/entrypoint | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/entrypoint b/docker/entrypoint index 18ab0da5..ad1d310c 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -22,6 +22,7 @@ if [ -z "${SKIP_SSL}" ]; then openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf + mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a fi # if [ ! -f /home/lemur/.lemur/lemur.conf.py ]; then From 28382ce728d25c190d5dca14d88a65d69d0c6802 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:48:42 +0100 Subject: [PATCH 084/110] Update default-ssl.conf --- docker/default-ssl.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/default-ssl.conf b/docker/default-ssl.conf index 8b791c45..86c770df 100644 --- a/docker/default-ssl.conf +++ b/docker/default-ssl.conf @@ -2,6 +2,12 @@ add_header X-Frame-Options DENY; add_header X-Content-Type-Options nosniff; add_header X-XSS-Protection "1; mode=block"; +server { + listen 80; + server_name _; + return 301 https://$host$request_uri; +} + server { listen 443; server_name _; From 4570fcf7fa07cd42b249e67926f1a4bfc5e24990 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:49:24 +0100 Subject: [PATCH 085/110] Rename docker/default-ssl.conf to docker/nginx/default-ssl.conf --- docker/{ => nginx}/default-ssl.conf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{ => nginx}/default-ssl.conf (100%) diff --git a/docker/default-ssl.conf b/docker/nginx/default-ssl.conf similarity index 100% rename from docker/default-ssl.conf rename to docker/nginx/default-ssl.conf From 248c0d226f827e0c612450baacf27100670079ad Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:49:36 +0100 Subject: [PATCH 086/110] Rename docker/default.conf to docker/nginx/default.conf --- docker/{ => nginx}/default.conf | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{ => nginx}/default.conf (100%) diff --git a/docker/default.conf b/docker/nginx/default.conf similarity index 100% rename from docker/default.conf rename to docker/nginx/default.conf From 949ebfa2850f02f1e2f875706192fe9dddb8f299 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:49:49 +0100 Subject: [PATCH 087/110] Update Dockerfile --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8ebb5241..7fa61700 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,8 +12,8 @@ ENV group lemur COPY entrypoint / COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / -COPY default.conf /etc/nginx/conf.d/ -COPY default-ssl.conf /etc/nginx/conf.d/ +COPY nginx/default.conf /etc/nginx/conf.d/ +COPY nginx/default-ssl.conf /etc/nginx/conf.d/ RUN addgroup -S ${group} -g ${gid} && \ adduser -D -S ${user} -G ${group} -u ${uid} && \ From 6c1129c946a4b47bf966e9c003335122995dc6c6 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:50:14 +0100 Subject: [PATCH 088/110] Rename docker/lemur.conf.py to docker/src/lemur.conf.py --- docker/{ => src}/lemur.conf.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{ => src}/lemur.conf.py (100%) diff --git a/docker/lemur.conf.py b/docker/src/lemur.conf.py similarity index 100% rename from docker/lemur.conf.py rename to docker/src/lemur.conf.py From 125a885742a19c0eb2f821007d168b0b22b98f45 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 11:50:48 +0100 Subject: [PATCH 089/110] Update Dockerfile --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7fa61700..f7d1caf7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,9 +8,8 @@ ENV gid 1337 ENV user lemur ENV group lemur - COPY entrypoint / -COPY lemur.conf.py /home/lemur/.lemur/lemur.conf.py +COPY src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py COPY supervisor.conf / COPY nginx/default.conf /etc/nginx/conf.d/ COPY nginx/default-ssl.conf /etc/nginx/conf.d/ From 7cbdc09055a04c747b2ab190b7e4d5b3e2144761 Mon Sep 17 00:00:00 2001 From: Lukas M Date: Tue, 1 Jan 2019 12:09:06 +0100 Subject: [PATCH 090/110] Update entrypoint --- docker/entrypoint | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint b/docker/entrypoint index ad1d310c..6077167a 100644 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -19,7 +19,7 @@ echo " # Done" if [ -z "${SKIP_SSL}" ]; then if [ ! -f /etc/nginx/ssl/server.crt ] && [ ! -f /etc/nginx/ssl/server.key ]; then - openssl req -x509 -newkey rsa:4096 -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" + openssl req -x509 -newkey rsa:4096 -nodes -keyout /etc/nginx/ssl/server.key -out /etc/nginx/ssl/server.crt -days 365 -subj "/C=US/ST=FAKE/L=FAKE/O=FAKE/OU=FAKE/CN=FAKE" fi mv /etc/nginx/conf.d/default-ssl.conf.a /etc/nginx/conf.d/default-ssl.conf mv /etc/nginx/conf.d/default.conf /etc/nginx/conf.d/default.conf.a From 3ac5361cb2b22775c7bd2f2fe5989c919919d9af Mon Sep 17 00:00:00 2001 From: bby-bishopclark <30503374+bby-bishopclark@users.noreply.github.com> Date: Thu, 3 Jan 2019 07:58:42 -0800 Subject: [PATCH 091/110] Update index.rst Simple English gaffes noticed while perusing docs -- Setup vs set up, it's vs English, etc. --- docs/quickstart/index.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 70ca1312..adeadd7c 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -22,7 +22,7 @@ Some basic prerequisites which you'll need in order to run Lemur: Installing Build Dependencies ----------------------------- -If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build it's dependencies: +If installing Lemur on a bare Ubuntu OS you will need to grab the following packages so that Lemur can correctly build its dependencies: .. code-block:: bash @@ -117,7 +117,7 @@ Simply run: .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command. -You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to setup in your environment. +You can specify ``-c`` or ``--config`` to any Lemur command to specify the current environment you are working in. Lemur will also look under the environmental variable ``LEMUR_CONF`` should that be easier to set up in your environment. Update your configuration @@ -144,7 +144,7 @@ Before Lemur will run you need to fill in a few required variables in the config LEMUR_DEFAULT_ORGANIZATION LEMUR_DEFAULT_ORGANIZATIONAL_UNIT -Setup Postgres +Set Up Postgres -------------- For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on. @@ -193,10 +193,10 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. -Setup a Reverse Proxy +Set Up a Reverse Proxy --------------------- -By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need setup a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. +By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. Proxying with Nginx From faa91ef2a71aac12cbf68910e172dd7beec96ad5 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 8 Jan 2019 09:47:46 -0800 Subject: [PATCH 092/110] Update requirements with Kombu fix --- requirements-dev.txt | 16 +++++++++------- requirements-docs.txt | 40 ++++++++++++++++++++-------------------- requirements-tests.txt | 10 +++++----- requirements.in | 1 + requirements.txt | 24 ++++++++++++------------ 5 files changed, 47 insertions(+), 44 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b427b20..e9e47ed5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,18 +8,19 @@ aspy.yaml==1.1.1 # via pre-commit bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.11.29 # via requests -cfgv==1.1.0 # via pre-commit +cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.1.7 # via pre-commit +identify==1.1.8 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.7 # via pre-commit +importlib-metadata==0.8 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 -pkginfo==1.4.2 # via twine -pre-commit==1.12.0 +pkginfo==1.5.0 # via twine +pre-commit==1.13.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer @@ -29,8 +30,9 @@ requests-toolbelt==0.8.0 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.28.1 # via twine +tqdm==4.29.0 # via twine twine==1.12.1 urllib3==1.24.1 # via requests -virtualenv==16.1.0 # via pre-commit +virtualenv==16.2.0 # via pre-commit webencodings==0.5.1 # via bleach +zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index 3f036915..bb1fe767 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,21 +4,21 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.29.1 +acme==0.30.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.5 amqp==2.3.2 aniso8601==4.0.1 -arrow==0.12.1 +arrow==0.13.0 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx -bcrypt==3.1.4 +bcrypt==3.1.5 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.60 -botocore==1.12.60 +boto3==1.9.75 +botocore==1.12.75 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 @@ -35,13 +35,13 @@ flask-cors==3.0.7 flask-mail==0.9.1 flask-migrate==2.3.1 flask-principal==0.4.0 -flask-restful==0.3.6 +flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -idna==2.7 +idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 @@ -49,12 +49,12 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.2 +kombu==4.2.1 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 marshmallow-sqlalchemy==0.15.0 -marshmallow==2.16.3 +marshmallow==2.17.0 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==18.0 # via sphinx @@ -62,35 +62,35 @@ paramiko==2.4.2 pbr==5.1.1 pem==18.2.0 psycopg2==2.7.6.1 -pyasn1-modules==0.2.2 -pyasn1==0.4.4 +pyasn1-modules==0.2.3 +pyasn1==0.4.5 pycparser==2.19 pygments==2.3.1 # via sphinx -pyjwt==1.7.0 +pyjwt==1.7.1 pynacl==1.3.0 pyopenssl==18.0.0 pyparsing==2.3.0 # via packaging pyrfc3339==1.1 python-dateutil==2.7.5 python-editor==1.0.3 -pytz==2018.7 +pytz==2018.9 pyyaml==3.13 -raven[flask]==6.9.0 +raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.8.0 -requests[security]==2.20.1 +requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 -six==1.11.0 +six==1.12.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.2 -sphinx==1.8.2 +sphinx==1.8.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx -sqlalchemy-utils==0.33.9 -sqlalchemy==1.2.14 +sqlalchemy-utils==0.33.10 +sqlalchemy==1.2.15 tabulate==0.8.2 urllib3==1.24.1 -vine==1.1.4 +vine==1.2.0 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 59c626f7..a11de6ec 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.67 # via moto +boto3==1.9.75 # via moto boto==2.49.0 # via moto -botocore==1.12.67 # via boto3, moto, s3transfer +botocore==1.12.75 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -34,7 +34,7 @@ jsondiff==1.1.1 # via moto jsonpickle==1.0 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==4.3.0 # via pytest +more-itertools==5.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.1 # via mock @@ -46,10 +46,10 @@ pycryptodome==3.7.2 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==4.0.2 +pytest==4.1.0 python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.7 # via moto +pytz==2018.9 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.21.0 # via aws-xray-sdk, docker, moto, requests-mock, responses diff --git a/requirements.in b/requirements.in index 9824650b..e427c9a2 100644 --- a/requirements.in +++ b/requirements.in @@ -25,6 +25,7 @@ future gunicorn inflection jinja2 +kombu<=4.2.2 # Kombu 4.2.2 breaks requirements lockfile marshmallow-sqlalchemy marshmallow diff --git a/requirements.txt b/requirements.txt index 7ee9a167..e3918631 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,19 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.29.1 +acme==0.30.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.5 # via flask-migrate amqp==2.3.2 # via kombu aniso8601==4.0.1 # via flask-restful -arrow==0.12.1 +arrow==0.13.0 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.5 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.67 -botocore==1.12.67 +boto3==1.9.75 +botocore==1.12.75 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl @@ -46,20 +46,20 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.2 # via celery +kombu==4.2.1 lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako marshmallow-sqlalchemy==0.15.0 -marshmallow==2.16.3 +marshmallow==2.17.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.1 # via mock pem==18.2.0 psycopg2==2.7.6.1 -pyasn1-modules==0.2.2 # via python-ldap -pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap +pyasn1-modules==0.2.3 # via python-ldap +pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi pyjwt==1.7.1 pynacl==1.3.0 # via paramiko @@ -68,19 +68,19 @@ pyrfc3339==1.1 # via acme python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339 +pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare -raven[flask]==6.9.0 +raven[flask]==6.10.0 redis==2.10.6 requests-toolbelt==0.8.0 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.12.0 -sqlalchemy-utils==0.33.9 +sqlalchemy-utils==0.33.10 sqlalchemy==1.2.15 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests -vine==1.1.4 # via amqp +vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0 From c95fde702376cd99d8cdb4d8b1bbaf89f0913666 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 8 Jan 2019 09:55:53 -0800 Subject: [PATCH 093/110] Better fix for kombu is to unpin it and modify makefile --- Makefile | 2 +- requirements-docs.txt | 2 +- requirements.in | 1 - requirements.txt | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 19a69236..f859f554 100644 --- a/Makefile +++ b/Makefile @@ -113,10 +113,10 @@ endif @echo "--> Updating Python requirements" pip install --upgrade pip pip install --upgrade pip-tools + pip-compile --output-file requirements.txt requirements.in -U --no-index pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index - pip-compile --output-file requirements.txt requirements.in -U --no-index @echo "--> Done updating Python requirements" @echo "--> Removing python-ldap from requirements-docs.txt" grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt diff --git a/requirements-docs.txt b/requirements-docs.txt index bb1fe767..19ebb0ea 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -49,7 +49,7 @@ jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 jsonlines==1.2.0 -kombu==4.2.1 +kombu==4.2.2.post1 lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 diff --git a/requirements.in b/requirements.in index e427c9a2..9824650b 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,6 @@ future gunicorn inflection jinja2 -kombu<=4.2.2 # Kombu 4.2.2 breaks requirements lockfile marshmallow-sqlalchemy marshmallow diff --git a/requirements.txt b/requirements.txt index e3918631..59871284 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.2.1 +kombu==4.2.2.post1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako From 3ee12cc50be99bb9e6b9b074f606468d9e2aa742 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 10 Jan 2019 09:26:15 -0800 Subject: [PATCH 094/110] Update requirements --- requirements-dev.txt | 7 +++---- requirements-docs.txt | 6 +++--- requirements-tests.txt | 8 ++++---- requirements.txt | 6 +++--- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e9e47ed5..21156588 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,8 +5,7 @@ # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # aspy.yaml==1.1.1 # via pre-commit -bleach==3.0.2 # via readme-renderer -cached-property==1.5.1 # via pre-commit +bleach==3.1.0 # via readme-renderer certifi==2018.11.29 # via requests cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests @@ -19,8 +18,8 @@ importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 -pkginfo==1.5.0 # via twine -pre-commit==1.13.0 +pkginfo==1.5.0.1 # via twine +pre-commit==1.14.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer diff --git a/requirements-docs.txt b/requirements-docs.txt index 19ebb0ea..f3182456 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -9,7 +9,7 @@ alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.5 amqp==2.3.2 -aniso8601==4.0.1 +aniso8601==4.1.0 arrow==0.13.0 asn1crypto==0.24.0 asyncpool==1.0 @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.5 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.75 -botocore==1.12.75 +boto3==1.9.76 +botocore==1.12.76 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 diff --git a/requirements-tests.txt b/requirements-tests.txt index a11de6ec..490d74d1 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.75 # via moto +boto3==1.9.76 # via moto boto==2.49.0 # via moto -botocore==1.12.75 # via boto3, moto, s3transfer +botocore==1.12.76 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -38,7 +38,7 @@ more-itertools==5.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.1 # via mock -pluggy==0.8.0 # via pytest +pluggy==0.8.1 # via pytest py==1.7.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi @@ -60,5 +60,5 @@ text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask -wrapt==1.10.11 # via aws-xray-sdk +wrapt==1.11.0 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.txt b/requirements.txt index 59871284..bc72db0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,15 +8,15 @@ acme==0.30.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.5 # via flask-migrate amqp==2.3.2 # via kombu -aniso8601==4.0.1 # via flask-restful +aniso8601==4.1.0 # via flask-restful arrow==0.13.0 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.5 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.75 -botocore==1.12.75 +boto3==1.9.76 +botocore==1.12.76 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl From 0e02e6da799af16120b9ddb54c7542a68aa4365f Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 11 Jan 2019 11:13:43 -0800 Subject: [PATCH 095/110] Be more forgiving to throttling --- lemur/plugins/lemur_aws/elb.py | 18 +++++++++--------- lemur/plugins/lemur_aws/iam.py | 8 ++++---- lemur/plugins/lemur_aws/sts.py | 16 +++++++++++++--- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/lemur/plugins/lemur_aws/elb.py b/lemur/plugins/lemur_aws/elb.py index 4c4ce97f..b4391dd8 100644 --- a/lemur/plugins/lemur_aws/elb.py +++ b/lemur/plugins/lemur_aws/elb.py @@ -95,7 +95,7 @@ def get_all_elbs_v2(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs): """ Get a listener ARN from an endpoint. @@ -113,7 +113,7 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_elbs(**kwargs): """ Fetches one page elb objects for a given account and region. @@ -123,7 +123,7 @@ def get_elbs(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_elbs_v2(**kwargs): """ Fetches one page of elb objects for a given account and region. @@ -136,7 +136,7 @@ def get_elbs_v2(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_listeners_v2(**kwargs): """ Fetches one page of listener objects for a given elb arn. @@ -149,7 +149,7 @@ def describe_listeners_v2(**kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs): """ Fetching all policies currently associated with an ELB. @@ -161,7 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_ssl_policies_v2(policy_names, **kwargs): """ Fetching all policies currently associated with an ELB. @@ -173,7 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_load_balancer_types(policies, **kwargs): """ Describe the policies with policy details. @@ -185,7 +185,7 @@ def describe_load_balancer_types(policies, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def attach_certificate(name, port, certificate_id, **kwargs): """ Attaches a certificate to a listener, throws exception @@ -205,7 +205,7 @@ def attach_certificate(name, port, certificate_id, **kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def attach_certificate_v2(listener_arn, port, certificates, **kwargs): """ Attaches a certificate to a listener, throws exception diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index b2a07798..7010c909 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -52,7 +52,7 @@ def create_arn_from_cert(account_number, region, certificate_name): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): """ Upload a certificate to AWS @@ -95,7 +95,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def delete_cert(cert_name, **kwargs): """ Delete a certificate from AWS @@ -112,7 +112,7 @@ def delete_cert(cert_name, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_certificate(name, **kwargs): """ Retrieves an SSL certificate. @@ -126,7 +126,7 @@ def get_certificate(name, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_certificates(**kwargs): """ Fetches one page of certificate objects for a given account. diff --git a/lemur/plugins/lemur_aws/sts.py b/lemur/plugins/lemur_aws/sts.py index 001ea2c8..6253ad7a 100644 --- a/lemur/plugins/lemur_aws/sts.py +++ b/lemur/plugins/lemur_aws/sts.py @@ -9,14 +9,22 @@ from functools import wraps import boto3 +from botocore.config import Config from flask import current_app +config = Config( + retries=dict( + max_attempts=20 + ) +) + + def sts_client(service, service_type='client'): def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): - sts = boto3.client('sts') + sts = boto3.client('sts', config=config) arn = 'arn:aws:iam::{0}:role/{1}'.format( kwargs.pop('account_number'), current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur') @@ -31,7 +39,8 @@ def sts_client(service, service_type='client'): region_name=kwargs.pop('region', 'us-east-1'), aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], - aws_session_token=role['Credentials']['SessionToken'] + aws_session_token=role['Credentials']['SessionToken'], + config=config ) kwargs['client'] = client elif service_type == 'resource': @@ -40,7 +49,8 @@ def sts_client(service, service_type='client'): region_name=kwargs.pop('region', 'us-east-1'), aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], - aws_session_token=role['Credentials']['SessionToken'] + aws_session_token=role['Credentials']['SessionToken'], + config=config ) kwargs['resource'] = resource return f(*args, **kwargs) From c4e6e7c59bae61855ea1e0ea514fc8da5566b962 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 14 Jan 2019 08:02:27 -0800 Subject: [PATCH 096/110] Optimize DB cert filtering --- lemur/certificates/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index c9a2fa24..e4503324 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -307,7 +307,7 @@ def render(args): if filt: terms = filt.split(';') - term = '%{0}%'.format(terms[1]) + term = '{0}%'.format(terms[1]) # Exact matches for quotes. Only applies to name, issuer, and cn if terms[1].startswith('"') and terms[1].endswith('"'): term = terms[1][1:-1] From 31a86687e72e02883a9d80abe345b5b5b64d2667 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 14 Jan 2019 09:20:02 -0800 Subject: [PATCH 097/110] Reduce the expense of joins --- lemur/certificates/service.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index e4503324..1b203260 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -20,7 +20,6 @@ from lemur.common.utils import generate_private_key, truthiness from lemur.destinations.models import Destination from lemur.domains.models import Domain from lemur.extensions import metrics, sentry, signals -from lemur.models import certificate_associations from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate from lemur.plugins.base import plugins @@ -341,13 +340,13 @@ def render(args): elif 'id' in terms: query = query.filter(Certificate.id == cast(terms[1], Integer)) elif 'name' in terms: - query = query.outerjoin(certificate_associations).outerjoin(Domain).filter( + query = query.filter( or_( Certificate.name.ilike(term), - Domain.name.ilike(term), + Certificate.domains.any(Domain.name.ilike(term)), Certificate.cn.ilike(term), ) - ).group_by(Certificate.id) + ) else: query = database.filter(query, Certificate, terms) From 3567a768d5a0e281d41d7c3f8bdb73bf3c6a7728 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 14 Jan 2019 13:35:55 -0800 Subject: [PATCH 098/110] Compare certificate hashes to determine if Lemur already has a synced certificate --- lemur/common/utils.py | 11 +++++++++++ lemur/sources/service.py | 5 +++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 62e59d69..0504c958 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -12,6 +12,7 @@ import string import sqlalchemy from cryptography import x509 from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import rsa, ec from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask_restful.reqparse import RequestParser @@ -226,3 +227,13 @@ def truthiness(s): """If input string resembles something truthy then return True, else False.""" return s.lower() in ('true', 'yes', 'on', 't', '1') + + +def find_matching_certificates_by_hash(cert, matching_certs): + """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs), + determine if any of the certificate hashes match and return the matches.""" + matching = [] + for c in matching_certs: + if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(hashes.SHA256()): + matching.append(c) + return matching diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 227f1bce..55d2ee62 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.utils import parse_certificate +from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate from lemur.common.defaults import serial from lemur.plugins.base import plugins @@ -126,7 +126,8 @@ def sync_certificates(source, user): if not exists: cert = parse_certificate(certificate['body']) - exists = certificate_service.get_by_serial(serial(cert)) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) if not certificate.get('owner'): certificate['owner'] = user.email From d3284a4006a87940ca485adea33957c116176c02 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 14 Jan 2019 17:52:06 -0800 Subject: [PATCH 099/110] adjusting the query to filter authorities based on matching CN --- lemur/authorities/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 024cb42a..41c381e3 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -15,6 +15,7 @@ from lemur import database from lemur.common.utils import truthiness from lemur.extensions import metrics from lemur.authorities.models import Authority +from lemur.certificates.models import Certificate from lemur.roles import service as role_service from lemur.certificates.service import upload @@ -179,7 +180,12 @@ def render(args): if 'active' in filt: query = query.filter(Authority.active == truthiness(terms[1])) elif 'cn' in filt: - query = query.join(Authority.active == truthiness(terms[1])) + term = '%{0}%'.format(terms[1]) + sub_query = database.session_query(Certificate.root_authority_id) \ + .filter(Certificate.cn.ilike(term)) \ + .subquery() + + query = query.filter(Authority.id.in_(sub_query)) else: query = database.filter(query, Authority, terms) From 7f88c24e8374f669ba1e12c3d5ff06892de3b04c Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 17 Jan 2019 14:56:04 -0800 Subject: [PATCH 100/110] Fix LetsEncrypt Dyn flow for duplicate CN/SAN --- lemur/common/utils.py | 11 +++++++++++ lemur/plugins/lemur_acme/dyn.py | 8 ++++++-- lemur/sources/service.py | 5 +++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 62e59d69..f26f07df 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -12,6 +12,7 @@ import string import sqlalchemy from cryptography import x509 from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import rsa, ec from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask_restful.reqparse import RequestParser @@ -226,3 +227,13 @@ def truthiness(s): """If input string resembles something truthy then return True, else False.""" return s.lower() in ('true', 'yes', 'on', 't', '1') + + +def find_matching_certificates_by_hash(cert, matching_certs): + """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs), + determine if any of the certificate hashes match and return the matches.""" + matching = [] + for c in matching_certs: + if parse_certificate(c).fingerprint(hashes.SHA256()) == cert.body.fingerprint(hashes.SHA256()): + matching.append(c) + return matching diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 9bab3a65..5d419f7f 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -5,7 +5,7 @@ import dns.exception import dns.name import dns.query import dns.resolver -from dyn.tm.errors import DynectCreateError +from dyn.tm.errors import DynectCreateError, DynectGetError from dyn.tm.session import DynectSession from dyn.tm.zones import Node, Zone, get_all_zones from flask import current_app @@ -119,7 +119,11 @@ def delete_txt_record(change_id, account_number, domain, token): zone = Zone(zone_name) node = Node(zone_name, fqdn) - all_txt_records = node.get_all_records_by_type('TXT') + try: + all_txt_records = node.get_all_records_by_type('TXT') + except DynectGetError: + # No Text Records remain or host is not in the zone anymore because all records have been deleted. + return for txt_record in all_txt_records: if txt_record.txtdata == ("{}".format(token)): current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn)) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 227f1bce..55d2ee62 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.utils import parse_certificate +from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate from lemur.common.defaults import serial from lemur.plugins.base import plugins @@ -126,7 +126,8 @@ def sync_certificates(source, user): if not exists: cert = parse_certificate(certificate['body']) - exists = certificate_service.get_by_serial(serial(cert)) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) if not certificate.get('owner'): certificate['owner'] = user.email From d689f5cda3aad42ff7b6363c40332160ca3a395a Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 17 Jan 2019 14:59:57 -0800 Subject: [PATCH 101/110] Fix LetsEncrypt for duplicate CN/SAN --- requirements-dev.txt | 5 ++--- requirements-docs.txt | 18 +++++++++--------- requirements-tests.txt | 8 ++++---- requirements.txt | 16 ++++++++-------- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 21156588..c1f55581 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,12 +14,11 @@ flake8==3.5.0 identify==1.1.8 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit -importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.14.0 +pre-commit==1.14.2 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer @@ -29,7 +28,7 @@ requests-toolbelt==0.8.0 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.29.0 # via twine +tqdm==4.29.1 # via twine twine==1.12.1 urllib3==1.24.1 # via requests virtualenv==16.2.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index f3182456..a7df5395 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -7,18 +7,18 @@ acme==0.30.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.5 -amqp==2.3.2 +alembic==1.0.6 +amqp==2.4.0 aniso8601==4.1.0 arrow==0.13.0 asn1crypto==0.24.0 asyncpool==1.0 babel==2.6.0 # via sphinx -bcrypt==3.1.5 +bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.76 -botocore==1.12.76 +boto3==1.9.80 +botocore==1.12.80 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 @@ -54,7 +54,7 @@ lockfile==0.12.2 mako==1.0.7 markupsafe==1.1.0 marshmallow-sqlalchemy==0.15.0 -marshmallow==2.17.0 +marshmallow==2.18.0 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==18.0 # via sphinx @@ -69,7 +69,7 @@ pygments==2.3.1 # via sphinx pyjwt==1.7.1 pynacl==1.3.0 pyopenssl==18.0.0 -pyparsing==2.3.0 # via packaging +pyparsing==2.3.1 # via packaging pyrfc3339==1.1 python-dateutil==2.7.5 python-editor==1.0.3 @@ -87,8 +87,8 @@ sphinx-rtd-theme==0.4.2 sphinx==1.8.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx -sqlalchemy-utils==0.33.10 -sqlalchemy==1.2.15 +sqlalchemy-utils==0.33.11 +sqlalchemy==1.2.16 tabulate==0.8.2 urllib3==1.24.1 vine==1.2.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 490d74d1..2d54dce6 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.76 # via moto +boto3==1.9.80 # via moto boto==2.49.0 # via moto -botocore==1.12.76 # via boto3, moto, s3transfer +botocore==1.12.80 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -18,7 +18,7 @@ click==7.0 # via flask coverage==4.5.2 cryptography==2.4.2 # via moto docker-pycreds==0.4.0 # via docker -docker==3.6.0 # via moto +docker==3.7.0 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 @@ -46,7 +46,7 @@ pycryptodome==3.7.2 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==4.1.0 +pytest==4.1.1 python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.9 # via moto diff --git a/requirements.txt b/requirements.txt index bc72db0a..79268c8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,17 +6,17 @@ # acme==0.30.0 alembic-autogenerate-enums==0.0.2 -alembic==1.0.5 # via flask-migrate -amqp==2.3.2 # via kombu +alembic==1.0.6 # via flask-migrate +amqp==2.4.0 # via kombu aniso8601==4.1.0 # via flask-restful arrow==0.13.0 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 -bcrypt==3.1.5 # via flask-bcrypt, paramiko +bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.76 -botocore==1.12.76 +boto3==1.9.80 +botocore==1.12.80 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl @@ -51,7 +51,7 @@ lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.1.0 # via jinja2, mako marshmallow-sqlalchemy==0.15.0 -marshmallow==2.17.0 +marshmallow==2.18.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 @@ -77,8 +77,8 @@ requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.12.0 -sqlalchemy-utils==0.33.10 -sqlalchemy==1.2.15 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.33.11 +sqlalchemy==1.2.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp From 4b893ab5b49b622a1634ef54e7323b219390bf0f Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Fri, 5 Jan 2018 13:08:07 +0200 Subject: [PATCH 102/110] Expose full certificate RFC 4514 Distinguished Name string Using rfc4514_string() method added in cryptography version 2.5. --- lemur/certificates/models.py | 4 ++++ lemur/certificates/schemas.py | 1 + lemur/static/app/angular/certificates/view/view.tpl.html | 2 ++ lemur/tests/test_certificates.py | 6 ++++++ requirements.txt | 2 +- 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 3eaba746..34305cc2 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -227,6 +227,10 @@ class Certificate(db.Model): def location(self): return defaults.location(self.parsed_cert) + @property + def distinguished_name(self): + return self.parsed_cert.subject.rfc4514_string() + @property def key_type(self): if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey): diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 6b457086..946bd541 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -206,6 +206,7 @@ class CertificateOutputSchema(LemurOutputSchema): cn = fields.String() common_name = fields.String(attribute='cn') + distinguished_name = fields.String() not_after = fields.DateTime() validity_end = ArrowDateTime(attribute='not_after') diff --git a/lemur/static/app/angular/certificates/view/view.tpl.html b/lemur/static/app/angular/certificates/view/view.tpl.html index ba17ffa6..28b4e08e 100644 --- a/lemur/static/app/angular/certificates/view/view.tpl.html +++ b/lemur/static/app/angular/certificates/view/view.tpl.html @@ -83,6 +83,8 @@
+
Distinguished Name
+
{{ certificate.distinguishedName }}
Certificate Authority
{{ certificate.authority ? certificate.authority.name : "Imported" }} ({{ certificate.issuer }})
Serial
diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index a1df1c0d..db2d27cf 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -619,6 +619,12 @@ def test_certificate_get_body(client): response_body = client.get(api.url_for(Certificates, certificate_id=1), headers=VALID_USER_HEADER_TOKEN).json assert response_body['serial'] == '211983098819107449768450703123665283596' assert response_body['serialHex'] == '9F7A75B39DAE4C3F9524C68B06DA6A0C' + assert response_body['distinguishedName'] == ('CN=LemurTrust Unittests Class 1 CA 2018,' + 'O=LemurTrust Enterprises Ltd,' + 'OU=Unittesting Operations Center,' + 'C=EE,' + 'ST=N/A,' + 'L=Earth') @pytest.mark.parametrize("token,status", [ diff --git a/requirements.txt b/requirements.txt index 79268c8a..d700de42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 -cryptography==2.4.2 +cryptography==2.5 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore From a9724e73830be5c6ee00f6cd81bf2aff6865b071 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 24 Jan 2019 17:23:40 -0800 Subject: [PATCH 103/110] Resolving the 2 years error from UI during cert creation: Though a CA would accept two year validity, we were getting error for being beyond 2 years. This is because our current conversion is just current date plus 2 years, 1/25/2019 + 2 years ==> 1/25/2019 This is more strictly seen two years and 1 day extra, violating the 2 year's limit. --- lemur/common/missing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/common/missing.py b/lemur/common/missing.py index a4bbba77..508019b2 100644 --- a/lemur/common/missing.py +++ b/lemur/common/missing.py @@ -16,6 +16,9 @@ def convert_validity_years(data): data['validity_start'] = now.isoformat() end = now.replace(years=+int(data['validity_years'])) + # some CAs want to see exactly two years validity, and not two years plus one day, as is the case currently + # 1/25/2019 + 2 years ==> 1/25/2019 (two years and 1 day extra, violating the 2 year's limit) + end = end.replace(days=-1) if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True): if is_weekend(end): end = end.replace(days=-2) From c47fa0f9a23689f0fce6e02364f12288bbf7c7db Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 24 Jan 2019 17:52:22 -0800 Subject: [PATCH 104/110] adjusting the tests to reflect on the new full year convert limit! --- lemur/tests/test_missing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/tests/test_missing.py b/lemur/tests/test_missing.py index 4f2c20c6..80a7df48 100644 --- a/lemur/tests/test_missing.py +++ b/lemur/tests/test_missing.py @@ -6,12 +6,12 @@ from freezegun import freeze_time def test_convert_validity_years(session): from lemur.common.missing import convert_validity_years - with freeze_time("2016-01-01"): + with freeze_time("2016-01-02"): data = convert_validity_years(dict(validity_years=2)) assert data['validity_start'] == arrow.utcnow().isoformat() - assert data['validity_end'] == arrow.utcnow().replace(years=+2).isoformat() + assert data['validity_end'] == arrow.utcnow().replace(years=+2, days=-1).isoformat() - with freeze_time("2015-01-10"): + with freeze_time("2015-01-11"): data = convert_validity_years(dict(validity_years=1)) - assert data['validity_end'] == arrow.utcnow().replace(years=+1, days=-2).isoformat() + assert data['validity_end'] == arrow.utcnow().replace(years=+1, days=-3).isoformat() From b4d1b80e04c6ead46635977fc9d21161718eb6e5 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 29 Jan 2019 10:13:44 -0500 Subject: [PATCH 105/110] Adding support for cfssl auth mode signing --- lemur/plugins/lemur_cfssl/plugin.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lemur/plugins/lemur_cfssl/plugin.py b/lemur/plugins/lemur_cfssl/plugin.py index 030f290a..ead633bc 100644 --- a/lemur/plugins/lemur_cfssl/plugin.py +++ b/lemur/plugins/lemur_cfssl/plugin.py @@ -10,6 +10,9 @@ import json import requests +import base64 +import hmac +import hashlib from flask import current_app @@ -48,6 +51,21 @@ class CfsslIssuerPlugin(IssuerPlugin): data = {'certificate_request': csr} data = json.dumps(data) + try: + hex_key = current_app.config.get('CFSSL_KEY') + key=bytes.fromhex(hex_key) + except: + #unable to find CFSSL_KEY in config, continue using normal sign method + pass + else: + data=data.encode() + + token = base64.b64encode(hmac.new(key,data,digestmod=hashlib.sha256).digest()) + data = base64.b64encode(data) + + data = json.dumps({'token': token.decode('utf-8'), 'request': data.decode('utf-8')}) + + url = "{0}{1}".format(current_app.config.get('CFSSL_URL'), '/api/v1/cfssl/authsign') response = self.session.post(url, data=data.encode(encoding='utf_8', errors='strict')) if response.status_code > 399: metrics.send('cfssl_create_certificate_failure', 'counter', 1) From 254a3079f2ceb7408b42d3ec9626cbf69d4abb7e Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 29 Jan 2019 11:01:55 -0500 Subject: [PATCH 106/110] fix whitespace --- lemur/plugins/lemur_cfssl/plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_cfssl/plugin.py b/lemur/plugins/lemur_cfssl/plugin.py index ead633bc..d2abc2aa 100644 --- a/lemur/plugins/lemur_cfssl/plugin.py +++ b/lemur/plugins/lemur_cfssl/plugin.py @@ -53,14 +53,14 @@ class CfsslIssuerPlugin(IssuerPlugin): try: hex_key = current_app.config.get('CFSSL_KEY') - key=bytes.fromhex(hex_key) + key = bytes.fromhex(hex_key) except: #unable to find CFSSL_KEY in config, continue using normal sign method pass else: - data=data.encode() + data = data.encode() - token = base64.b64encode(hmac.new(key,data,digestmod=hashlib.sha256).digest()) + token = base64.b64encode(hmac.new(key, data, digestmod=hashlib.sha256).digest()) data = base64.b64encode(data) data = json.dumps({'token': token.decode('utf-8'), 'request': data.decode('utf-8')}) From c68a9cf80acd651ad18fe48a6c7d0e0a43ef7f29 Mon Sep 17 00:00:00 2001 From: alwaysjolley Date: Tue, 29 Jan 2019 11:10:56 -0500 Subject: [PATCH 107/110] fixing linting issues --- lemur/plugins/lemur_cfssl/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_cfssl/plugin.py b/lemur/plugins/lemur_cfssl/plugin.py index d2abc2aa..4bfefc85 100644 --- a/lemur/plugins/lemur_cfssl/plugin.py +++ b/lemur/plugins/lemur_cfssl/plugin.py @@ -54,8 +54,8 @@ class CfsslIssuerPlugin(IssuerPlugin): try: hex_key = current_app.config.get('CFSSL_KEY') key = bytes.fromhex(hex_key) - except: - #unable to find CFSSL_KEY in config, continue using normal sign method + except (ValueError, NameError): + # unable to find CFSSL_KEY in config, continue using normal sign method pass else: data = data.encode() From d2317acfc550b35a1ad40b449c37d66e5f258cf8 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 29 Jan 2019 15:17:40 -0800 Subject: [PATCH 108/110] allowing create_user with noninteractive PW;updating reqs --- lemur/manage.py | 19 +++++++++++-------- requirements-dev.txt | 11 ++++++----- requirements-docs.txt | 24 ++++++++++++------------ requirements-tests.txt | 18 +++++++++--------- requirements.txt | 24 ++++++++++++------------ 5 files changed, 50 insertions(+), 46 deletions(-) diff --git a/lemur/manage.py b/lemur/manage.py index b972e8a5..184b9aa6 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -273,10 +273,11 @@ class CreateUser(Command): Option('-u', '--username', dest='username', required=True), Option('-e', '--email', dest='email', required=True), Option('-a', '--active', dest='active', default=True), - Option('-r', '--roles', dest='roles', action='append', default=[]) + Option('-r', '--roles', dest='roles', action='append', default=[]), + Option('-p', '--password', dest='password', default=None) ) - def run(self, username, email, active, roles): + def run(self, username, email, active, roles, password): role_objs = [] for r in roles: role_obj = role_service.get_by_name(r) @@ -286,14 +287,16 @@ class CreateUser(Command): sys.stderr.write("[!] Cannot find role {0}\n".format(r)) sys.exit(1) - password1 = prompt_pass("Password") - password2 = prompt_pass("Confirm Password") + if not password: + password1 = prompt_pass("Password") + password2 = prompt_pass("Confirm Password") + password = password1 - if password1 != password2: - sys.stderr.write("[!] Passwords do not match!\n") - sys.exit(1) + if password1 != password2: + sys.stderr.write("[!] Passwords do not match!\n") + sys.exit(1) - user_service.create(username, password1, email, active, None, role_objs) + user_service.create(username, password, email, active, None, role_objs) sys.stdout.write("[+] Created new user: {0}\n".format(username)) diff --git a/requirements-dev.txt b/requirements-dev.txt index c1f55581..ac35f3e9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,16 +4,17 @@ # # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # -aspy.yaml==1.1.1 # via pre-commit +aspy.yaml==1.1.2 # via pre-commit bleach==3.1.0 # via readme-renderer certifi==2018.11.29 # via requests cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.1.8 # via pre-commit +identify==1.2.1 # via pre-commit idna==2.8 # via requests importlib-metadata==0.8 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 @@ -24,13 +25,13 @@ pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit readme-renderer==24.0 # via twine -requests-toolbelt==0.8.0 # via twine +requests-toolbelt==0.9.0 # via twine requests==2.21.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.29.1 # via twine +tqdm==4.30.0 # via twine twine==1.12.1 urllib3==1.24.1 # via requests -virtualenv==16.2.0 # via pre-commit +virtualenv==16.3.0 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.3.3 # via importlib-metadata diff --git a/requirements-docs.txt b/requirements-docs.txt index a7df5395..15085766 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,10 +4,10 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.30.0 +acme==0.30.2 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.6 +alembic==1.0.7 amqp==2.4.0 aniso8601==4.1.0 arrow==0.13.0 @@ -17,15 +17,15 @@ babel==2.6.0 # via sphinx bcrypt==3.1.6 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.80 -botocore==1.12.80 +boto3==1.9.86 +botocore==1.12.86 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 chardet==3.0.4 click==7.0 cloudflare==2.1.0 -cryptography==2.4.2 +cryptography==2.5 dnspython3==1.15.0 dnspython==1.15.0 docutils==0.14 @@ -57,18 +57,18 @@ marshmallow-sqlalchemy==0.15.0 marshmallow==2.18.0 mock==2.0.0 ndg-httpsclient==0.5.1 -packaging==18.0 # via sphinx +packaging==19.0 # via sphinx paramiko==2.4.2 pbr==5.1.1 pem==18.2.0 -psycopg2==2.7.6.1 -pyasn1-modules==0.2.3 +psycopg2==2.7.7 +pyasn1-modules==0.2.4 pyasn1==0.4.5 pycparser==2.19 pygments==2.3.1 # via sphinx pyjwt==1.7.1 pynacl==1.3.0 -pyopenssl==18.0.0 +pyopenssl==19.0.0 pyparsing==2.3.1 # via packaging pyrfc3339==1.1 python-dateutil==2.7.5 @@ -77,7 +77,7 @@ pytz==2018.9 pyyaml==3.13 raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.8.0 +requests-toolbelt==0.9.0 requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 @@ -88,8 +88,8 @@ sphinx==1.8.3 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.16 -tabulate==0.8.2 +sqlalchemy==1.2.17 +tabulate==0.8.3 urllib3==1.24.1 vine==1.2.0 werkzeug==0.14.1 diff --git a/requirements-tests.txt b/requirements-tests.txt index 2d54dce6..c326e951 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,30 +8,30 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.80 # via moto +boto3==1.9.86 # via moto boto==2.49.0 # via moto -botocore==1.12.80 # via boto3, moto, s3transfer +botocore==1.12.86 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==7.0 # via flask coverage==4.5.2 -cryptography==2.4.2 # via moto +cryptography==2.5 # via moto docker-pycreds==0.4.0 # via docker docker==3.7.0 # via moto docutils==0.14 # via botocore ecdsa==0.13 # via python-jose factory-boy==2.11.1 -faker==1.0.1 +faker==1.0.2 flask==1.0.2 # via pytest-flask freezegun==0.3.11 future==0.17.1 # via python-jose -idna==2.8 # via cryptography, requests +idna==2.8 # via requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto -jsonpickle==1.0 # via aws-xray-sdk +jsonpickle==1.1 # via aws-xray-sdk markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==5.0.0 # via pytest @@ -42,8 +42,8 @@ pluggy==0.8.1 # via pytest py==1.7.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi -pycryptodome==3.7.2 # via python-jose -pyflakes==2.0.0 +pycryptodome==3.7.3 # via python-jose +pyflakes==2.1.0 pytest-flask==0.14.0 pytest-mock==1.10.0 pytest==4.1.1 @@ -60,5 +60,5 @@ text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask -wrapt==1.11.0 # via aws-xray-sdk +wrapt==1.11.1 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.txt b/requirements.txt index 79268c8a..c595e509 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,9 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.30.0 +acme==0.30.2 alembic-autogenerate-enums==0.0.2 -alembic==1.0.6 # via flask-migrate +alembic==1.0.7 # via flask-migrate amqp==2.4.0 # via kombu aniso8601==4.1.0 # via flask-restful arrow==0.13.0 @@ -15,15 +15,15 @@ asyncpool==1.0 bcrypt==3.1.6 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.80 -botocore==1.12.80 +boto3==1.9.86 +botocore==1.12.86 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.1.0 -cryptography==2.4.2 +cryptography==2.5 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore @@ -39,7 +39,7 @@ flask-sqlalchemy==2.3.2 flask==1.0.2 future==0.17.1 gunicorn==19.9.0 -idna==2.8 # via cryptography, requests +idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask jinja2==2.10 @@ -57,13 +57,13 @@ ndg-httpsclient==0.5.1 paramiko==2.4.2 pbr==5.1.1 # via mock pem==18.2.0 -psycopg2==2.7.6.1 -pyasn1-modules==0.2.3 # via python-ldap +psycopg2==2.7.7 +pyasn1-modules==0.2.4 # via python-ldap pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.19 # via cffi pyjwt==1.7.1 pynacl==1.3.0 # via paramiko -pyopenssl==18.0.0 +pyopenssl==19.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic @@ -72,14 +72,14 @@ pytz==2018.9 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.10.0 redis==2.10.6 -requests-toolbelt==0.8.0 # via acme +requests-toolbelt==0.9.0 # via acme requests[security]==2.21.0 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.12.0 sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.2 +sqlalchemy==1.2.17 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.3 urllib3==1.24.1 # via botocore, requests vine==1.2.0 # via amqp werkzeug==0.14.1 # via flask From 48ad20facaba794a8a14c249af5fb83f206b7006 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 29 Jan 2019 16:17:08 -0800 Subject: [PATCH 109/110] moving the 2 year validity issue to the Verisign plugin, and address it there --- lemur/common/missing.py | 4 +--- lemur/plugins/lemur_verisign/plugin.py | 15 ++++++++++++--- lemur/tests/test_missing.py | 8 ++++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lemur/common/missing.py b/lemur/common/missing.py index 508019b2..5c7dffac 100644 --- a/lemur/common/missing.py +++ b/lemur/common/missing.py @@ -16,9 +16,7 @@ def convert_validity_years(data): data['validity_start'] = now.isoformat() end = now.replace(years=+int(data['validity_years'])) - # some CAs want to see exactly two years validity, and not two years plus one day, as is the case currently - # 1/25/2019 + 2 years ==> 1/25/2019 (two years and 1 day extra, violating the 2 year's limit) - end = end.replace(days=-1) + if not current_app.config.get('LEMUR_ALLOW_WEEKEND_EXPIRATION', True): if is_weekend(end): end = end.replace(days=-2) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 3e672a43..3f16f997 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -111,10 +111,19 @@ def process_options(options): data['subject_alt_names'] = ",".join(get_additional_names(options)) + if options.get('validity_end') > arrow.utcnow().replace(years=2): + raise Exception("Verisign issued certificates cannot exceed two years in validity") + if options.get('validity_end'): - period = get_default_issuance(options) - data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY") - data['validityPeriod'] = period + # VeriSign (Symantec) only accepts strictly smaller than 2 year end date + if options.get('validity_end') < arrow.utcnow().replace(years=2).replace(days=-1): + period = get_default_issuance(options) + data['specificEndDate'] = options['validity_end'].format("MM/DD/YYYY") + data['validityPeriod'] = period + else: + # allowing Symantec website setting the end date, given the validity period + data['validityPeriod'] = str(get_default_issuance(options)) + options.pop('validity_end', None) elif options.get('validity_years'): if options['validity_years'] in [1, 2]: diff --git a/lemur/tests/test_missing.py b/lemur/tests/test_missing.py index 80a7df48..4f2c20c6 100644 --- a/lemur/tests/test_missing.py +++ b/lemur/tests/test_missing.py @@ -6,12 +6,12 @@ from freezegun import freeze_time def test_convert_validity_years(session): from lemur.common.missing import convert_validity_years - with freeze_time("2016-01-02"): + with freeze_time("2016-01-01"): data = convert_validity_years(dict(validity_years=2)) assert data['validity_start'] == arrow.utcnow().isoformat() - assert data['validity_end'] == arrow.utcnow().replace(years=+2, days=-1).isoformat() + assert data['validity_end'] == arrow.utcnow().replace(years=+2).isoformat() - with freeze_time("2015-01-11"): + with freeze_time("2015-01-10"): data = convert_validity_years(dict(validity_years=1)) - assert data['validity_end'] == arrow.utcnow().replace(years=+1, days=-3).isoformat() + assert data['validity_end'] == arrow.utcnow().replace(years=+1, days=-2).isoformat() From e24a94d798bd69a0110b1e5ddf532192621ca754 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Wed, 26 Dec 2018 19:49:56 +0200 Subject: [PATCH 110/110] Enforce that PEM strings (certs, keys, CSR) are internally passed as str, not bytes This was already true in most places but not 100%, leading to lots of redundant checks and conversions. --- lemur/certificates/service.py | 10 +--------- lemur/common/utils.py | 17 +++++++---------- lemur/plugins/lemur_aws/iam.py | 3 +-- lemur/plugins/lemur_cryptography/plugin.py | 11 ++++------- lemur/plugins/lemur_csr/plugin.py | 11 +++-------- lemur/plugins/lemur_java/plugin.py | 18 +++++------------- lemur/plugins/lemur_openssl/plugin.py | 11 +++-------- lemur/tests/conftest.py | 7 +++---- 8 files changed, 27 insertions(+), 61 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1b203260..0f37d70e 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -221,11 +221,6 @@ def upload(**kwargs): else: kwargs['roles'] = roles - if kwargs.get('private_key'): - private_key = kwargs['private_key'] - if not isinstance(private_key, bytes): - kwargs['private_key'] = private_key.encode('utf-8') - cert = Certificate(**kwargs) cert.authority = kwargs.get('authority') cert = database.create(cert) @@ -432,10 +427,7 @@ def create_csr(**csr_config): encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, # would like to use PKCS8 but AWS ELBs don't like it encryption_algorithm=serialization.NoEncryption() - ) - - if isinstance(private_key, bytes): - private_key = private_key.decode('utf-8') + ).decode('utf-8') csr = request.public_bytes( encoding=serialization.Encoding.PEM diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 0504c958..32271e89 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -48,24 +48,22 @@ def parse_certificate(body): :param body: :return: """ - if isinstance(body, str): - body = body.encode('utf-8') + assert isinstance(body, str) - return x509.load_pem_x509_certificate(body, default_backend()) + return x509.load_pem_x509_certificate(body.encode('utf-8'), default_backend()) def parse_private_key(private_key): """ Parses a PEM-format private key (RSA, DSA, ECDSA or any other supported algorithm). - Raises ValueError for an invalid string. + Raises ValueError for an invalid string. Raises AssertionError when passed value is not str-type. :param private_key: String containing PEM private key """ - if isinstance(private_key, str): - private_key = private_key.encode('utf8') + assert isinstance(private_key, str) - return load_pem_private_key(private_key, password=None, backend=default_backend()) + return load_pem_private_key(private_key.encode('utf8'), password=None, backend=default_backend()) def parse_csr(csr): @@ -75,10 +73,9 @@ def parse_csr(csr): :param csr: :return: """ - if isinstance(csr, str): - csr = csr.encode('utf-8') + assert isinstance(csr, str) - return x509.load_pem_x509_csr(csr, default_backend()) + return x509.load_pem_x509_csr(csr.encode('utf-8'), default_backend()) def get_authority_key(body): diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 7010c909..49816c2b 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -64,6 +64,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): :param path: :return: """ + assert isinstance(private_key, str) client = kwargs.pop('client') if not path or path == '/': @@ -72,8 +73,6 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): name = name + '-' + path.strip('/') try: - if isinstance(private_key, bytes): - private_key = private_key.decode("utf-8") if cert_chain: return client.upload_server_certificate( Path=path, diff --git a/lemur/plugins/lemur_cryptography/plugin.py b/lemur/plugins/lemur_cryptography/plugin.py index fe9d7bb3..97060391 100644 --- a/lemur/plugins/lemur_cryptography/plugin.py +++ b/lemur/plugins/lemur_cryptography/plugin.py @@ -14,6 +14,7 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization +from lemur.common.utils import parse_private_key from lemur.plugins.bases import IssuerPlugin from lemur.plugins import lemur_cryptography as cryptography_issuer @@ -40,7 +41,8 @@ def issue_certificate(csr, options, private_key=None): if options.get("authority"): # Issue certificate signed by an existing lemur_certificates authority issuer_subject = options['authority'].authority_certificate.subject - issuer_private_key = options['authority'].authority_certificate.private_key + assert private_key is None, "Private would be ignored, authority key used instead" + private_key = options['authority'].authority_certificate.private_key chain_cert_pem = options['authority'].authority_certificate.body authority_key_identifier_public = options['authority'].authority_certificate.public_key authority_key_identifier_subject = x509.SubjectKeyIdentifier.from_public_key(authority_key_identifier_public) @@ -52,7 +54,6 @@ def issue_certificate(csr, options, private_key=None): else: # Issue certificate that is self-signed (new lemur_certificates root authority) issuer_subject = csr.subject - issuer_private_key = private_key chain_cert_pem = "" authority_key_identifier_public = csr.public_key() authority_key_identifier_subject = None @@ -112,11 +113,7 @@ def issue_certificate(csr, options, private_key=None): # FIXME: Not implemented in lemur/schemas.py yet https://github.com/Netflix/lemur/issues/662 pass - private_key = serialization.load_pem_private_key( - bytes(str(issuer_private_key).encode('utf-8')), - password=None, - backend=default_backend() - ) + private_key = parse_private_key(private_key) cert = builder.sign(private_key, hashes.SHA256(), default_backend()) cert_pem = cert.public_bytes( diff --git a/lemur/plugins/lemur_csr/plugin.py b/lemur/plugins/lemur_csr/plugin.py index e06035d1..13f42084 100644 --- a/lemur/plugins/lemur_csr/plugin.py +++ b/lemur/plugins/lemur_csr/plugin.py @@ -38,14 +38,9 @@ def create_csr(cert, chain, csr_tmp, key): :param csr_tmp: :param key: """ - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') - - if isinstance(key, bytes): - key = key.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) + assert isinstance(key, str) with mktempfile() as key_tmp: with open(key_tmp, 'w') as f: diff --git a/lemur/plugins/lemur_java/plugin.py b/lemur/plugins/lemur_java/plugin.py index 151794da..5aab5342 100644 --- a/lemur/plugins/lemur_java/plugin.py +++ b/lemur/plugins/lemur_java/plugin.py @@ -59,11 +59,8 @@ def split_chain(chain): def create_truststore(cert, chain, jks_tmp, alias, passphrase): - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) with mktempfile() as cert_tmp: with open(cert_tmp, 'w') as f: @@ -98,14 +95,9 @@ def create_truststore(cert, chain, jks_tmp, alias, passphrase): def create_keystore(cert, chain, jks_tmp, key, alias, passphrase): - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') - - if isinstance(key, bytes): - key = key.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) + assert isinstance(key, str) # Create PKCS12 keystore from private key and public certificate with mktempfile() as cert_tmp: diff --git a/lemur/plugins/lemur_openssl/plugin.py b/lemur/plugins/lemur_openssl/plugin.py index d50b4e43..9ddce925 100644 --- a/lemur/plugins/lemur_openssl/plugin.py +++ b/lemur/plugins/lemur_openssl/plugin.py @@ -44,14 +44,9 @@ def create_pkcs12(cert, chain, p12_tmp, key, alias, passphrase): :param alias: :param passphrase: """ - if isinstance(cert, bytes): - cert = cert.decode('utf-8') - - if isinstance(chain, bytes): - chain = chain.decode('utf-8') - - if isinstance(key, bytes): - key = key.decode('utf-8') + assert isinstance(cert, str) + assert isinstance(chain, str) + assert isinstance(key, str) with mktempfile() as key_tmp: with open(key_tmp, 'w') as f: diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index 9a48eb94..3790358e 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -3,12 +3,11 @@ import os import datetime import pytest from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask import current_app from flask_principal import identity_changed, Identity from lemur import create_app +from lemur.common.utils import parse_private_key from lemur.database import db as _db from lemur.auth.service import create_token from lemur.tests.vectors import SAN_CERT_KEY, INTERMEDIATE_KEY @@ -235,12 +234,12 @@ def logged_in_admin(session, app): @pytest.fixture def private_key(): - return load_pem_private_key(SAN_CERT_KEY.encode(), password=None, backend=default_backend()) + return parse_private_key(SAN_CERT_KEY) @pytest.fixture def issuer_private_key(): - return load_pem_private_key(INTERMEDIATE_KEY.encode(), password=None, backend=default_backend()) + return parse_private_key(INTERMEDIATE_KEY) @pytest.fixture