From 301f099622db033330d50c42d6e24f39b2adb5bd Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Fri, 21 Aug 2020 09:56:46 +0200 Subject: [PATCH 01/34] Fix link for WES-entropy-client --- docs/production/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/production/index.rst b/docs/production/index.rst index 67e97dae..ccace703 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -49,7 +49,7 @@ The amount of effort you wish to expend ensuring that Lemur has good entropy to If you wish to generate more entropy for your system we would suggest you take a look at the following resources: -- `WES-entropy-client `_ +- `WES-entropy-client `_ - `haveged `_ For additional information about OpenSSL entropy issues: From 02d711282de1848025a5228f173699fbbf1c51a3 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 14 Sep 2020 18:12:33 -0700 Subject: [PATCH 02/34] New column key_type commenting conflicting property for now --- lemur/certificates/models.py | 3 +++ lemur/migrations/versions/434c29e40511_.py | 26 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 lemur/migrations/versions/434c29e40511_.py diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 675cecb4..af70c1e8 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -154,6 +154,7 @@ class Certificate(db.Model): Integer, ForeignKey("authorities.id", ondelete="CASCADE") ) rotation_policy_id = Column(Integer, ForeignKey("rotation_policies.id")) + key_type = Column(String(128)) notifications = relationship( "Notification", @@ -297,6 +298,7 @@ class Certificate(db.Model): def distinguished_name(self): return self.parsed_cert.subject.rfc4514_string() + """ @property def key_type(self): if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey): @@ -305,6 +307,7 @@ class Certificate(db.Model): ) elif isinstance(self.parsed_cert.public_key(), ec.EllipticCurvePublicKey): return get_key_type_from_ec_curve(self.parsed_cert.public_key().curve.name) + """ @property def validity_remaining(self): diff --git a/lemur/migrations/versions/434c29e40511_.py b/lemur/migrations/versions/434c29e40511_.py new file mode 100644 index 00000000..677be1d9 --- /dev/null +++ b/lemur/migrations/versions/434c29e40511_.py @@ -0,0 +1,26 @@ +"""empty message + +Revision ID: 434c29e40511 +Revises: 8323a5ea723a +Create Date: 2020-09-11 17:24:51.344585 + +""" + +# revision identifiers, used by Alembic. +revision = '434c29e40511' +down_revision = '8323a5ea723a' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('certificates', sa.Column('key_type', sa.String(length=128), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('certificates', 'key_type') + # ### end Alembic commands ### From 676562ffdea03035a4c35d1af41a01746f2f304c Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 14 Sep 2020 18:13:35 -0700 Subject: [PATCH 03/34] Match column type to db schema No functional change --- lemur/dns_providers/models.py | 2 +- lemur/migrations/env.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/dns_providers/models.py b/lemur/dns_providers/models.py index eb8cdff9..0e301e7b 100644 --- a/lemur/dns_providers/models.py +++ b/lemur/dns_providers/models.py @@ -12,7 +12,7 @@ class DnsProvider(db.Model): __tablename__ = "dns_providers" id = Column(Integer(), primary_key=True) name = Column(String(length=256), unique=True, nullable=True) - description = Column(Text(), nullable=True) + description = Column(String(length=1024), nullable=True) provider_type = Column(String(length=256), nullable=True) credentials = Column(Vault, nullable=True) api_endpoint = Column(String(length=256), nullable=True) diff --git a/lemur/migrations/env.py b/lemur/migrations/env.py index 008a9952..3acefc3a 100644 --- a/lemur/migrations/env.py +++ b/lemur/migrations/env.py @@ -67,7 +67,8 @@ def run_migrations_online(): context.configure( connection=connection, target_metadata=target_metadata, - **current_app.extensions["migrate"].configure_args + **current_app.extensions["migrate"].configure_args, + compare_type=True ) try: From f5228407c2495e9ba6d72d0b35b5a8164867e065 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 16:51:01 +0000 Subject: [PATCH 04/34] Bump pytest from 6.0.1 to 6.0.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.0.1...6.0.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index e9106767..57791eba 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -62,7 +62,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in -pytest==6.0.1 # via -r requirements-tests.in, pytest-flask, pytest-mock +pytest==6.0.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto From 51fbd6a8714786d43c3b5cb8b309e44d1b024ebe Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 17:04:29 +0000 Subject: [PATCH 05/34] Bump faker from 4.1.2 to 4.1.3 Bumps [faker](https://github.com/joke2k/faker) from 4.1.2 to 4.1.3. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.1.2...v4.1.3) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 57791eba..643dcebc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==3.0.1 # via -r requirements-tests.in -faker==4.1.2 # via -r requirements-tests.in, factory-boy +faker==4.1.3 # via -r requirements-tests.in, factory-boy fakeredis==1.4.3 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in From dc675311f07743e479001029f76e4eb443fbb3c9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 17:13:55 +0000 Subject: [PATCH 06/34] Bump coverage from 5.2.1 to 5.3 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.2.1 to 5.3. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.2.1...coverage-5.3) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 643dcebc..c18cb2a3 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -18,7 +18,7 @@ cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.2 # via black, flask -coverage==5.2.1 # via -r requirements-tests.in +coverage==5.3 # via -r requirements-tests.in cryptography==3.1 # via moto, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto From 1ceafc593a0c677a017c32b9e2cf1f7ebef1735e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 17:22:55 +0000 Subject: [PATCH 07/34] Bump moto from 1.3.14 to 1.3.16 Bumps [moto](https://github.com/spulec/moto) from 1.3.14 to 1.3.16. - [Release notes](https://github.com/spulec/moto/releases) - [Changelog](https://github.com/spulec/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/spulec/moto/compare/1.3.14...1.3.16) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index c18cb2a3..bb25b5e5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -19,11 +19,11 @@ cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.2 # via black, flask coverage==5.3 # via -r requirements-tests.in -cryptography==3.1 # via moto, sshpubkeys +cryptography==3.1 # via moto, python-jose, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto docutils==0.15.2 # via botocore -ecdsa==0.15 # via python-jose, sshpubkeys +ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.0.1 # via -r requirements-tests.in faker==4.1.3 # via -r requirements-tests.in, factory-boy fakeredis==1.4.3 # via -r requirements-tests.in @@ -43,10 +43,10 @@ jsonpatch==1.25 # via cfn-lint jsonpickle==1.4 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==3.2.0 # via aws-sam-translator, cfn-lint -markupsafe==1.1.1 # via jinja2 +markupsafe==1.1.1 # via jinja2, moto mock==4.0.2 # via moto -more-itertools==8.2.0 # via pytest -moto==1.3.14 # via -r requirements-tests.in +more-itertools==8.2.0 # via moto, pytest +moto==1.3.16 # via -r requirements-tests.in mypy-extensions==0.4.3 # via black networkx==2.4 # via cfn-lint nose==1.3.7 # via -r requirements-tests.in @@ -64,7 +64,7 @@ pytest-flask==1.0.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in pytest==6.0.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto -python-jose==3.1.0 # via moto +python-jose[cryptography]==3.1.0 # via moto pytz==2019.3 # via moto pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto redis==3.5.3 # via fakeredis @@ -88,7 +88,7 @@ websocket-client==0.57.0 # via docker werkzeug==1.0.1 # via flask, moto, pytest-flask wrapt==1.12.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto -zipp==3.1.0 # via importlib-metadata +zipp==3.1.0 # via importlib-metadata, moto # The following packages are considered to be unsafe in a requirements file: # setuptools From 8022efe32e5a4630e607b7d02ede6b2473378b6e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 17:35:46 +0000 Subject: [PATCH 08/34] Bump acme from 1.7.0 to 1.8.0 Bumps [acme](https://github.com/letsencrypt/letsencrypt) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/letsencrypt/letsencrypt/releases) - [Commits](https://github.com/letsencrypt/letsencrypt/compare/v1.7.0...v1.8.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 37d50804..3ee96dd7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.7.0 # via -r requirements.txt +acme==1.8.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate diff --git a/requirements.txt b/requirements.txt index 64e41b3c..fcb06cd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.7.0 # via -r requirements.in +acme==1.8.0 # via -r requirements.in alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu From f5e71bb431dce204c0e96054f00335956cd0fa5f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2020 17:45:52 +0000 Subject: [PATCH 09/34] Bump boto3 from 1.14.56 to 1.14.61 Bumps [boto3](https://github.com/boto/boto3) from 1.14.56 to 1.14.61. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.14.56...1.14.61) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 3ee96dd7..f3f417bf 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.14.56 # via -r requirements.txt -botocore==1.17.56 # via -r requirements.txt, boto3, s3transfer +boto3==1.14.61 # via -r requirements.txt +botocore==1.17.61 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index bb25b5e5..20453852 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.14.56 # via aws-sam-translator, moto +boto3==1.14.61 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.17.56 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.17.61 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index fcb06cd9..27a37a8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.14.56 # via -r requirements.in -botocore==1.17.56 # via -r requirements.in, boto3, s3transfer +boto3==1.14.61 # via -r requirements.in +botocore==1.17.61 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 980883cb8d968a9f9eafda54cc1b7387e7cc2f4a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 15 Sep 2020 11:39:29 -0700 Subject: [PATCH 10/34] Dependbot failed to merge this PR, so raising it manually Bump http-proxy from 1.16.2 to 1.18.1 https://github.com/Netflix/lemur/pull/3123#partial-pull-merging --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1a54eccc..c4105e01 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "gulp-uglify": "^2.0.0", "gulp-useref": "^3.1.2", "gulp-util": "^3.0.1", - "http-proxy": "~1.16.2", + "http-proxy": ">=1.18.1", "jshint-stylish": "^2.2.1", "karma": "^4.4.1", "karma-jasmine": "^1.1.0", From 5ae65c2c4dc4c8c609ede2725a35f59d1c24a555 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 15 Sep 2020 14:55:04 -0700 Subject: [PATCH 11/34] Remove unused import --- lemur/certificates/models.py | 2 -- lemur/dns_providers/models.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index af70c1e8..eb49fbd5 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -9,10 +9,8 @@ from datetime import timedelta import arrow from cryptography import x509 -from cryptography.hazmat.primitives.asymmetric import rsa, ec from flask import current_app from idna.core import InvalidCodepoint -from lemur.common.utils import get_key_type_from_ec_curve from sqlalchemy import ( event, Integer, diff --git a/lemur/dns_providers/models.py b/lemur/dns_providers/models.py index 0e301e7b..7ad51308 100644 --- a/lemur/dns_providers/models.py +++ b/lemur/dns_providers/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, Integer, String, text, Text +from sqlalchemy import Column, Integer, String, text from sqlalchemy.dialects.postgresql import JSON from sqlalchemy.orm import relationship from sqlalchemy_utils import ArrowType From 51549ae79597db068340d77be3eaf2d7b3bd8e51 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 15 Sep 2020 17:37:58 -0700 Subject: [PATCH 12/34] Adding comment for the property to be removed --- lemur/certificates/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index eb49fbd5..f71d2199 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -297,6 +297,7 @@ class Certificate(db.Model): return self.parsed_cert.subject.rfc4514_string() """ + # Commenting this property as key_type is now added as a column. This code can be removed in future. @property def key_type(self): if isinstance(self.parsed_cert.public_key(), rsa.RSAPublicKey): From 5d7ca8520a1480af31d14e7fbc8424ef9f504e76 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 16 Sep 2020 10:23:04 +0200 Subject: [PATCH 13/34] Add remark that the WES-entropy-client is now linked to a fork --- docs/production/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/production/index.rst b/docs/production/index.rst index ccace703..503cb581 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -52,6 +52,8 @@ If you wish to generate more entropy for your system we would suggest you take a - `WES-entropy-client `_ - `haveged `_ +The original *WES-entropy-client* repository by WhitewoodCrypto was removed, the link now points to a fork of it. + For additional information about OpenSSL entropy issues: - `Managing and Understanding Entropy Usage `_ From fae37932552b6f60a0046c01ccbdae2075eb655d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 11:09:32 -0700 Subject: [PATCH 14/34] entrrust plugin revised --- lemur/plugins/lemur_entrust/plugin.py | 90 ++++++++++++++++++--------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 315da8bd..3669d9d6 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -1,9 +1,12 @@ -from lemur.plugins.bases import IssuerPlugin, SourcePlugin + import arrow import requests import json -from lemur.plugins import lemur_entrust as ENTRUST +import sys from flask import current_app + +from lemur.plugins import lemur_entrust as entrust +from lemur.plugins.bases import IssuerPlugin, SourcePlugin from lemur.extensions import metrics from lemur.common.utils import validate_conf @@ -24,17 +27,17 @@ def determine_end_date(end_date): """ Determine appropriate end date :param end_date: - :return: validity_end + :return: validity_end as string """ # ENTRUST only allows 13 months of max certificate duration - max_validity_end = arrow.utcnow().shift(years=1, months=+1).format('YYYY-MM-DD') + max_validity_end = arrow.utcnow().shift(years=1, months=+1) if not end_date: end_date = max_validity_end if end_date > max_validity_end: end_date = max_validity_end - return end_date + return end_date.format('YYYY-MM-DD') def process_options(options): @@ -49,6 +52,9 @@ def process_options(options): # take the value as Cert product-type # else default to "STANDARD_SSL" authority = options.get("authority").name.upper() + # STANDARD_SSL (cn=domain, san=www.domain), + # ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]), + # WILDCARD_SSL (unlimited sans, and wildcard) product_type = current_app.config.get("ENTRUST_PRODUCT_{0}".format(authority), "STANDARD_SSL") if options.get("validity_end"): @@ -67,6 +73,7 @@ def process_options(options): "eku": "SERVER_AND_CLIENT_AUTH", "certType": product_type, "certExpiryDate": validity_end, + # "keyType": "RSA", Entrust complaining about this parameter "tracking": tracking_data } return data @@ -86,23 +93,31 @@ def handle_response(my_response): 404: "Unknown jobId", 429: "Too many requests" } + try: d = json.loads(my_response.content) - except Exception as e: + except ValueError: # catch an empty jason object here - d = {'errors': 'No detailled message'} + d = {'response': 'No detailed message'} s = my_response.status_code if s > 399: - raise Exception("ENTRUST error: {0}\n{1}".format(msg.get(s, s), d['errors'])) - current_app.logger.info("Response: {0}, {1} ".format(s, d)) + raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}") + + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Response", + "status": s, + "response": d + } + current_app.logger.info(log_data) return d class EntrustIssuerPlugin(IssuerPlugin): - title = "ENTRUST" + title = "Entrust" slug = "entrust-issuer" description = "Enables the creation of certificates by ENTRUST" - version = ENTRUST.VERSION + version = entrust.VERSION author = "sirferl" author_url = "https://github.com/sirferl/lemur" @@ -119,7 +134,6 @@ class EntrustIssuerPlugin(IssuerPlugin): "ENTRUST_NAME", "ENTRUST_EMAIL", "ENTRUST_PHONE", - "ENTRUST_ISSUING", ] validate_conf(current_app, required_vars) @@ -142,9 +156,12 @@ class EntrustIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ - current_app.logger.info( - "Requesting options: {0}".format(issuer_options) - ) + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Requesting options", + "options": issuer_options + } + current_app.logger.info(log_data) url = current_app.config.get("ENTRUST_URL") + "/certificates" @@ -161,31 +178,41 @@ class EntrustIssuerPlugin(IssuerPlugin): response_dict = handle_response(response) external_id = response_dict['trackingId'] cert = response_dict['endEntityCert'] - chain = response_dict['chainCerts'][1] - current_app.logger.info( - "Received Chain: {0}".format(chain) - ) + if len(response_dict['chainCerts']) < 2: + # certificate signed by CA directly, no ICA included ini the chain + chain = None + else: + chain = response_dict['chainCerts'][1] + + log_data["message"] = "Received Chain" + log_data["options"] = f"chain: {chain}" + current_app.logger.info(log_data) return cert, chain, external_id def revoke_certificate(self, certificate, comments): - """Revoke a Digicert certificate.""" + """Revoke an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") # make certificate revoke request - revoke_url = "{0}/certificates/{1}/revocations".format( - base_url, certificate.external_id - ) - metrics.send("entrust_revoke_certificate", "counter", 1) - if comments == '' or not comments: + revoke_url = f"{base_url}/certificates/{certificate.external_id}/revocations" + if not comments or comments == '': comments = "revoked via API" data = { - "crlReason": "superseded", + "crlReason": "superseded", # enum (keyCompromise, affiliationChanged, superseded, cessationOfOperation) "revocationComment": comments } response = self.session.post(revoke_url, json=data) + metrics.send("entrust_revoke_certificate", "counter", 1) + return handle_response(response) - data = handle_response(response) + def deactivate_certificate(self, certificate, comments): + """Deactivates an Entrust certificate.""" + base_url = current_app.config.get("ENTRUST_URL") + revoke_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" + response = self.session.post(revoke_url) + metrics.send("entrust_revoke_certificate", "counter", 1) + return handle_response(response) @staticmethod def create_authority(options): @@ -200,7 +227,8 @@ class EntrustIssuerPlugin(IssuerPlugin): entrust_root = current_app.config.get("ENTRUST_ROOT") entrust_issuing = current_app.config.get("ENTRUST_ISSUING") role = {"username": "", "password": "", "name": "entrust"} - current_app.logger.info("Creating Auth: {0} {1}".format(options, entrust_issuing)) + current_app.logger.info(f"Creating Auth: {options} {entrust_issuing}") + # body, chain, role return entrust_root, "", [role] def get_ordered_certificate(self, order_id): @@ -211,10 +239,10 @@ class EntrustIssuerPlugin(IssuerPlugin): class EntrustSourcePlugin(SourcePlugin): - title = "ENTRUST" + title = "Entrust" slug = "entrust-source" - description = "Enables the collecion of certificates" - version = ENTRUST.VERSION + description = "Enables the collection of certificates" + version = entrust.VERSION author = "sirferl" author_url = "https://github.com/sirferl/lemur" From 416f39222a7375c1a31462cc4c24b9ab025b1b5d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 17:02:19 -0700 Subject: [PATCH 15/34] testing --- lemur/plugins/lemur_entrust/tests/conftest.py | 1 + .../lemur_entrust/tests/test_entrust.py | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 lemur/plugins/lemur_entrust/tests/conftest.py create mode 100644 lemur/plugins/lemur_entrust/tests/test_entrust.py diff --git a/lemur/plugins/lemur_entrust/tests/conftest.py b/lemur/plugins/lemur_entrust/tests/conftest.py new file mode 100644 index 00000000..0e1cd89f --- /dev/null +++ b/lemur/plugins/lemur_entrust/tests/conftest.py @@ -0,0 +1 @@ +from lemur.tests.conftest import * # noqa diff --git a/lemur/plugins/lemur_entrust/tests/test_entrust.py b/lemur/plugins/lemur_entrust/tests/test_entrust.py new file mode 100644 index 00000000..b1cd4c83 --- /dev/null +++ b/lemur/plugins/lemur_entrust/tests/test_entrust.py @@ -0,0 +1,54 @@ +from unittest.mock import patch, Mock + +import arrow +from cryptography import x509 +from lemur.plugins.lemur_entrust import plugin + + +def config_mock(*args): + values = { + "ENTRUST_API_CERT": "-----BEGIN CERTIFICATE-----abc-----END CERTIFICATE-----", + "ENTRUST_API_KEY": False, + "ENTRUST_API_USER": "test", + "ENTRUST_API_PASS": "password", + "ENTRUST_URL": "http", + "ENTRUST_ROOT": None, + "ENTRUST_NAME": "test", + "ENTRUST_EMAIL": "test@lemur.net", + "ENTRUST_PHONE": "0123456", + "ENTRUST_PRODUCT_ENTRUST": "ADVANTAGE_SSL" + } + return values[args[0]] + + +@patch("lemur.plugins.lemur_entrust.plugin.current_app") +def test_process_options(mock_current_app, authority): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2020, 10, 7).format('YYYY-MM-DD')) + + authority.name = "Entrust" + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "organization": "Example, Inc.", + "organizational_unit": "Example Org", + "validity_end": arrow.get(2020, 10, 7), + "authority": authority, + } + + expected = { + "signingAlg": "SHA-2", + "eku": "SERVER_AND_CLIENT_AUTH", + "certType": "ADVANTAGE_SSL", + "certExpiryDate": arrow.get(2020, 10, 7).format('YYYY-MM-DD'), + "tracking": { + "requesterName": mock_current_app.config.get("ENTRUST_NAME"), + "requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"), + "requesterPhone": mock_current_app.config.get("ENTRUST_PHONE") + } + } + + assert expected == plugin.process_options(options) From edab32d9a13cbde2916424d923d1b2ab9aea4b0c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 17:03:22 -0700 Subject: [PATCH 16/34] setting the required entrust configs --- lemur/tests/conf.py | 48 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index af0c09ce..bf033421 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -1,9 +1,18 @@ # This is just Python which means you can inherit and tweak settings import os +import random +import string _basedir = os.path.abspath(os.path.dirname(__file__)) + +# generate random secrets for unittest +def get_random_secret(length): + input_ascii = string.ascii_letters + string.digits + return ''.join(random.choice(input_ascii) for i in range(length)) + + THREADS_PER_PAGE = 8 # General @@ -86,7 +95,6 @@ DIGICERT_CIS_API_KEY = "api-key" DIGICERT_CIS_ROOTS = {"root": "ROOT"} DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"} - VERISIGN_URL = "http://example.com" VERISIGN_PEM_PATH = "~/" VERISIGN_FIRST_NAME = "Jim" @@ -197,3 +205,41 @@ LDAP_REQUIRED_GROUP = "Lemur Access" LDAP_DEFAULT_ROLE = "role1" ALLOW_CERT_DELETION = True + +ENTRUST_API_CERT = "api-cert" +ENTRUST_API_KEY = get_random_secret(32) +ENTRUST_API_USER = "user" +ENTRUST_API_PASS = get_random_secret(32) +ENTRUST_URL = "https://api.entrust.net/enterprise/v2" +ENTRUST_ROOT = """ +-----BEGIN CERTIFICATE----- +MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC +VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50 +cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs +IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz +dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy +NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu +dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt +dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0 +aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T +RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN +cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW +wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1 +U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0 +jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP +BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN +BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/ +jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ +Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v +1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R +nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH +VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g== +-----END CERTIFICATE----- +""" +ENTRUST_NAME = "lemur" +ENTRUST_EMAIL = "lemur@example.com" +ENTRUST_PHONE = "123456" +ENTRUST_ISSUING = "" +ENTRUST_PRODUCT_ENTRUST = "ADVANTAGE_SSL" From cc855e27582a6a04db84006c53e1149dc3546f07 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Sep 2020 17:16:07 -0700 Subject: [PATCH 17/34] modern python style --- lemur/plugins/lemur_entrust/plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 3669d9d6..50b9d929 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -20,7 +20,7 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ - metrics.send("ENTRUST_status_code_{}".format(r.status_code), "counter", 1) + metrics.send("entrust_status_code", "counter", 1, metadata={"status_code": r.status_code}) def determine_end_date(end_date): @@ -55,7 +55,7 @@ def process_options(options): # STANDARD_SSL (cn=domain, san=www.domain), # ADVANTAGE_SSL (cn=domain, san=[www.domain, one_more_option]), # WILDCARD_SSL (unlimited sans, and wildcard) - product_type = current_app.config.get("ENTRUST_PRODUCT_{0}".format(authority), "STANDARD_SSL") + product_type = current_app.config.get(f"ENTRUST_PRODUCT_{authority}", "STANDARD_SSL") if options.get("validity_end"): validity_end = determine_end_date(options.get("validity_end")) @@ -173,7 +173,7 @@ class EntrustIssuerPlugin(IssuerPlugin): except requests.exceptions.Timeout: raise Exception("Timeout for POST") except requests.exceptions.RequestException as e: - raise Exception("Error for POST {0}".format(e)) + raise Exception(f"Error for POST {e}") response_dict = handle_response(response) external_id = response_dict['trackingId'] From 6dc4b9877bf11f90e94c80fad0843abfbea7ea4a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2020 06:11:00 +0000 Subject: [PATCH 18/34] Bump boto3 from 1.14.61 to 1.15.2 Bumps [boto3](https://github.com/boto/boto3) from 1.14.61 to 1.15.2. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.14.61...1.15.2) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 6 +++--- requirements-tests.txt | 5 ++--- requirements.txt | 5 ++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index f3f417bf..5c6fdf92 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.14.61 # via -r requirements.txt -botocore==1.17.61 # via -r requirements.txt, boto3, s3transfer +boto3==1.15.2 # via -r requirements.txt +botocore==1.18.2 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt @@ -29,7 +29,7 @@ cloudflare==2.8.13 # via -r requirements.txt cryptography==3.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 -docutils==0.15.2 # via -r requirements.txt, botocore, sphinx +docutils==0.15.2 # via sphinx dyn==1.8.1 # via -r requirements.txt flask-bcrypt==0.7.1 # via -r requirements.txt flask-cors==3.0.9 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 20453852..b2b51cd7 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.14.61 # via aws-sam-translator, moto +boto3==1.15.2 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.17.61 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.18.2 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto @@ -22,7 +22,6 @@ coverage==5.3 # via -r requirements-tests.in cryptography==3.1 # via moto, python-jose, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto -docutils==0.15.2 # via botocore ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.0.1 # via -r requirements-tests.in faker==4.1.3 # via -r requirements-tests.in, factory-boy diff --git a/requirements.txt b/requirements.txt index 27a37a8c..cab2a8ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.14.61 # via -r requirements.in -botocore==1.17.61 # via -r requirements.in, boto3, s3transfer +boto3==1.15.2 # via -r requirements.in +botocore==1.18.2 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in @@ -27,7 +27,6 @@ cloudflare==2.8.13 # via -r requirements.in cryptography==3.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 -docutils==0.15.2 # via botocore dyn==1.8.1 # via -r requirements.in flask-bcrypt==0.7.1 # via -r requirements.in flask-cors==3.0.9 # via -r requirements.in From 9f66c18e71641ca4070770a801af7ec4beb04f8b Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 22 Sep 2020 14:48:40 +0200 Subject: [PATCH 19/34] Add REDIS_HOST and REDIS_PORT to celery configuration documentation --- docs/production/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/production/index.rst b/docs/production/index.rst index 67e97dae..1fdd5dde 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -390,6 +390,9 @@ Here are the Celery configuration variables that should be set:: CELERY_IMPORTS = ('lemur.common.celery') CELERY_TIMEZONE = 'UTC' + REDIS_HOST="your_redis_url" + REDIS_PORT="6379" + Do not forget to import crontab module in your configuration file:: from celery.task.schedules import crontab From 8de9842092d073852907a0242503e941e2d7c0e3 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:22:45 -0700 Subject: [PATCH 20/34] Backfill the key_type column: DB Upgrade --- lemur/common/utils.py | 17 ++++ lemur/migrations/versions/c301c59688d2_.py | 108 +++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 lemur/migrations/versions/c301c59688d2_.py diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 01cc64ae..283d1eec 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -71,6 +71,23 @@ def parse_private_key(private_key): ) +def get_key_type_from_certificate(body): + """ + + Helper function to determine key type by pasrding given PEM certificate + + :param body: PEM string + :return: Key type string + """ + parsed_cert = parse_certificate(body) + if isinstance(parsed_cert.public_key(), rsa.RSAPublicKey): + return "RSA{key_size}".format( + key_size=parsed_cert.public_key().key_size + ) + elif isinstance(parsed_cert.public_key(), ec.EllipticCurvePublicKey): + return get_key_type_from_ec_curve(parsed_cert.public_key().curve.name) + + def split_pem(data): """ Split a string of several PEM payloads to a list of strings. diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py new file mode 100644 index 00000000..6bd94cfb --- /dev/null +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -0,0 +1,108 @@ +""" + +This upgrade of database updates the key_type information for certificates +that are either still valid or have expired in last 30 days. For RSA keys, +the algorithm is determined based on the key length. For rest of the keys, +the certificate body is parsed to determine the exact key type information. + +Each individual change is explicitly committed. The logs are added to file +named upgrade_logs in current working directory. If faced any issue while +running this upgrade, there is no harm in re-running the upgrade. Each run +processes only the keys for which key type information is not yet determined. +A successful end to end run will end up updating the Alembic Version to new +Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported +by Lemur. This could be a long running job depending upon the number of +keys it may process. + +Revision ID: c301c59688d2 +Revises: 434c29e40511 +Create Date: 2020-09-21 14:28:50.757998 + +""" + +# revision identifiers, used by Alembic. +revision = 'c301c59688d2' +down_revision = '434c29e40511' + +from alembic import op +from sqlalchemy.sql import text +from lemur.common import utils +import time + +log_file = open('upgrade_logs', 'a') + + +def upgrade(): + log_file.write("\n*** Starting new run ***\n") + start_time = time.time() + + # Update RSA keys using the key length information + update_key_type_rsa(1024) + update_key_type_rsa(2048) + update_key_type_rsa(4096) + + # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs. + update_key_type() + + log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time)) + log_file.close() + + +def downgrade(): + # Change key type column back to null + # Going back 32 days instead of 31 to make sure no certificates are skipped + stmt = text( + "update certificates set key_type=null where not_after > CURRENT_DATE - 32" + ) + op.execute(stmt) + + +""" + Helper methods performing updates for RSA and rest of the keys +""" + + +def update_key_type_rsa(bits): + log_file.write("Processing certificate with key type RSA %s\n" % bits) + + stmt = text( + "update certificates set key_type='RSA{0}' where bits={0} and not_after > CURRENT_DATE - 31 and key_type is null".format(bits) + ) + log_file.write("Query: %s\n" % stmt) + + start_time = time.time() + op.execute(stmt) + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def update_key_type(): + conn = op.get_bind() + start_time = time.time() + + # Loop through all certificates are valid today or expired in last 30 days + for cert_id, body in conn.execute( + text( + "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null") + ): + try: + cert_key_type = utils.get_key_type_from_certificate(body) + except ValueError: + log_file.write("Error in processing certificate. ID: %s\n" % cert_id) + else: + log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + stmt = text( + "update certificates set key_type=:key_type where id=:id" + ) + stmt = stmt.bindparams(key_type=cert_key_type, id=cert_id) + op.execute(stmt) + + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def commit(): + stmt = text("commit") + op.execute(stmt) From 9211178e77b1c78a17775c27e1caaee8d0e0756f Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:31:38 -0700 Subject: [PATCH 21/34] Added date-time and modified log file name --- lemur/migrations/versions/c301c59688d2_.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 6bd94cfb..2c91783f 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -28,12 +28,13 @@ from alembic import op from sqlalchemy.sql import text from lemur.common import utils import time +import datetime -log_file = open('upgrade_logs', 'a') +log_file = open('db_upgrade.log', 'a') def upgrade(): - log_file.write("\n*** Starting new run ***\n") + log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) start_time = time.time() # Update RSA keys using the key length information From 921e8d8236d3be4b1bf9cfa044fe581f0744a2f2 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:46:15 -0700 Subject: [PATCH 22/34] Add error message to the logs --- lemur/migrations/versions/c301c59688d2_.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 2c91783f..0dd46315 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -6,9 +6,12 @@ the algorithm is determined based on the key length. For rest of the keys, the certificate body is parsed to determine the exact key type information. Each individual change is explicitly committed. The logs are added to file -named upgrade_logs in current working directory. If faced any issue while -running this upgrade, there is no harm in re-running the upgrade. Each run -processes only the keys for which key type information is not yet determined. +named db_upgrade.log in current working directory. Any error encountered +while parsing a certificate will also be logged along with the certificate +ID. If faced any issue while running this upgrade, there is no harm in +re-running the upgrade. Each run processes only the keys for which key type +information is not yet determined. + A successful end to end run will end up updating the Alembic Version to new Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported by Lemur. This could be a long running job depending upon the number of @@ -89,8 +92,8 @@ def update_key_type(): ): try: cert_key_type = utils.get_key_type_from_certificate(body) - except ValueError: - log_file.write("Error in processing certificate. ID: %s\n" % cert_id) + except ValueError as e: + log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) else: log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) stmt = text( From f97e880fa662b308d43d562c729ac6c56ba634a0 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 11:06:11 +0200 Subject: [PATCH 23/34] REDIS_PORT as integer, add hint about multiple redis databases --- docs/production/index.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/production/index.rst b/docs/production/index.rst index ade54c48..55752a95 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -388,12 +388,16 @@ To enable celery support, you must also have configuration values that tell Cele Here are the Celery configuration variables that should be set:: CELERY_RESULT_BACKEND = 'redis://your_redis_url:6379' - CELERY_BROKER_URL = 'redis://your_redis_url:6379' + CELERY_BROKER_URL = 'redis://your_redis_url:6379/0' CELERY_IMPORTS = ('lemur.common.celery') CELERY_TIMEZONE = 'UTC' REDIS_HOST="your_redis_url" - REDIS_PORT="6379" + REDIS_PORT=6379 + REDIS_DB=0 + +Out of the box, every Redis instance supports 16 databases. The default database (`REDIS_DB`) is set to 0, however, you can use any of the databases from 0-15. Via `redis.conf` more databases can be supported. +In the `redis://` url, the database number needs to be added with a slash after the port. Do not forget to import crontab module in your configuration file:: From cad04885a09ff232ad49ed9ac96b2498070151bb Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 13:17:28 +0200 Subject: [PATCH 24/34] Add celery configuration options, to config documentation --- docs/administration.rst | 48 +++++++++++++++++++++++++++++++++++++++ docs/production/index.rst | 1 + 2 files changed, 49 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index a3225fc2..0cec16a0 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -328,6 +328,54 @@ Lemur supports sending certification expiration notifications through SES and SM LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2] +Celery Options +--------------- +To make use of automated tasks within lemur (e.g. syncing source/destinations, or reissuing ACME certificates), you +need to configure celery. See :ref:`Periodic Tasks ` for more in depth documentation. + +.. data:: CELERY_RESULT_BACKEND + :noindex: + + The url to your redis backend (needs to be in the format `redis://:/`) + +.. data:: CELERY_BROKER_URL + :noindex: + + The url to your redis broker (needs to be in the format `redis://:/`) + +.. data:: CELERY_IMPORTS + :noindex: + + The module that celery needs to import, in our case thats `lemur.common.celery` + +.. data:: CELERY_TIMEZONE + :noindex: + + The timezone for celery to work with + + +.. data:: CELERYBEAT_SCHEDULE + :noindex: + + This defines the schedule, with which the celery beat makes the worker run the specified tasks. + +Since the celery module, relies on the RedisHandler, the following options also need to be set. + +.. data:: REDIS_HOST + :noindex: + + Hostname of your redis instance + +.. data:: REDIS_PORT + :noindex: + + Port on which redis is running (default: 6379) + +.. data:: REDIS_DB + :noindex: + + Which redis database to be used, by default redis offers databases 0-15 (default: 0) + Authentication Options ---------------------- Lemur currently supports Basic Authentication, LDAP Authentication, Ping OAuth2, and Google out of the box. Additional flows can be added relatively easily. diff --git a/docs/production/index.rst b/docs/production/index.rst index 55752a95..e4dd2e84 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -315,6 +315,7 @@ It will start a shell from which you can start/stop/restart the service. You can read all errors that might occur from /tmp/lemur.log. +.. _PeriodicTasks: Periodic Tasks ============== From d5557c1533dd94a11c3edfed31562949c87a414e Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 09:58:28 -0700 Subject: [PATCH 25/34] Update index.rst adding insight about the default db --- docs/production/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/production/index.rst b/docs/production/index.rst index e4dd2e84..9f90c0cc 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -398,7 +398,7 @@ Here are the Celery configuration variables that should be set:: REDIS_DB=0 Out of the box, every Redis instance supports 16 databases. The default database (`REDIS_DB`) is set to 0, however, you can use any of the databases from 0-15. Via `redis.conf` more databases can be supported. -In the `redis://` url, the database number needs to be added with a slash after the port. +In the `redis://` url, the database number can be added with a slash after the port. (defaults to 0, if omitted) Do not forget to import crontab module in your configuration file:: From e3fa0726080b6a10434ff1a68871fe14998a133a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 10:17:30 -0700 Subject: [PATCH 26/34] Update c301c59688d2_.py language --- lemur/migrations/versions/c301c59688d2_.py | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 0dd46315..8f1941b2 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -1,21 +1,23 @@ """ -This upgrade of database updates the key_type information for certificates -that are either still valid or have expired in last 30 days. For RSA keys, -the algorithm is determined based on the key length. For rest of the keys, -the certificate body is parsed to determine the exact key type information. +This database upgrade updates the key_type information for either +still valid or expired certificates in the last 30 days. For RSA +keys, the algorithm is determined based on the key length. For +the rest of the keys, the certificate body is parsed to determine +the exact key_type information. -Each individual change is explicitly committed. The logs are added to file -named db_upgrade.log in current working directory. Any error encountered -while parsing a certificate will also be logged along with the certificate -ID. If faced any issue while running this upgrade, there is no harm in -re-running the upgrade. Each run processes only the keys for which key type -information is not yet determined. +Each individual DB change is explicitly committed, and the +respective log is added to a file named db_upgrade.log in the current +working directory. Any error encountered while parsing a certificate +will also be logged along with the certificate ID. If faced with +any issue while running this upgrade, there is no harm in +re-running the upgrade. Each run processes only rows for which +key_type information is not yet determined. -A successful end to end run will end up updating the Alembic Version to new -Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported -by Lemur. This could be a long running job depending upon the number of -keys it may process. +A successful complete run will end up updating the Alembic Version +to the new Revision ID c301c59688d2. Currently, only RSA and ECC +certificates are supported by Lemur. This could be a long-running +job depending upon the number of DB entries it may process. Revision ID: c301c59688d2 Revises: 434c29e40511 From 19b693f636a77b7a460e2a24f9d557aa410f1a7a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 10:21:23 -0700 Subject: [PATCH 27/34] Update c301c59688d2_.py language --- lemur/migrations/versions/c301c59688d2_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 8f1941b2..c96272ff 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -87,7 +87,7 @@ def update_key_type(): conn = op.get_bind() start_time = time.time() - # Loop through all certificates are valid today or expired in last 30 days + # Loop through all certificates that are valid today or expired in the last 30 days. for cert_id, body in conn.execute( text( "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null") From 710290f590fca0b523b73a9f35d213fc6cad6879 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 23 Sep 2020 11:45:36 -0700 Subject: [PATCH 28/34] Formatting changes --- lemur/migrations/versions/c301c59688d2_.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index c96272ff..3b0a86f7 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -6,18 +6,18 @@ keys, the algorithm is determined based on the key length. For the rest of the keys, the certificate body is parsed to determine the exact key_type information. -Each individual DB change is explicitly committed, and the -respective log is added to a file named db_upgrade.log in the current -working directory. Any error encountered while parsing a certificate -will also be logged along with the certificate ID. If faced with -any issue while running this upgrade, there is no harm in -re-running the upgrade. Each run processes only rows for which -key_type information is not yet determined. +Each individual DB change is explicitly committed, and the respective +log is added to a file named db_upgrade.log in the current working +directory. Any error encountered while parsing a certificate will +also be logged along with the certificate ID. If faced with any issue +while running this upgrade, there is no harm in re-running the upgrade. +Each run processes only rows for which key_type information is not yet +determined. -A successful complete run will end up updating the Alembic Version -to the new Revision ID c301c59688d2. Currently, only RSA and ECC -certificates are supported by Lemur. This could be a long-running -job depending upon the number of DB entries it may process. +A successful complete run will end up updating the Alembic Version to +the new Revision ID c301c59688d2. Currently, Lemur supports only RSA +and ECC certificates. This could be a long-running job depending upon +the number of DB entries it may process. Revision ID: c301c59688d2 Revises: 434c29e40511 @@ -72,7 +72,7 @@ def update_key_type_rsa(bits): log_file.write("Processing certificate with key type RSA %s\n" % bits) stmt = text( - "update certificates set key_type='RSA{0}' where bits={0} and not_after > CURRENT_DATE - 31 and key_type is null".format(bits) + f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null" ) log_file.write("Query: %s\n" % stmt) From 12af0ecb457263f9dad344c4af458de473ff6a59 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 23 Sep 2020 11:46:38 -0700 Subject: [PATCH 29/34] UT get_key_type_from_certificate --- lemur/tests/test_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 1dac39bb..162e53b0 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -2,11 +2,13 @@ import pytest from lemur.tests.vectors import ( SAN_CERT, + SAN_CERT_STR, INTERMEDIATE_CERT, ROOTCA_CERT, EC_CERT_EXAMPLE, ECDSA_PRIME256V1_CERT, ECDSA_SECP384r1_CERT, + ECDSA_SECP384r1_CERT_STR, DSA_CERT, ) @@ -106,3 +108,9 @@ def test_is_selfsigned(selfsigned_cert): # unsupported algorithm (DSA) with pytest.raises(Exception): is_selfsigned(DSA_CERT) + + +def test_get_key_type_from_certificate(): + from lemur.common.utils import get_key_type_from_certificate + assert (get_key_type_from_certificate(SAN_CERT_STR) == "RSA2048") + assert (get_key_type_from_certificate(ECDSA_SECP384r1_CERT_STR) == "ECCSECP384R1") From 1983eb79de4b46ec2bbb0e194fe2bce0c333ddfd Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 13:00:14 +0200 Subject: [PATCH 30/34] Add paragraph about reusing ACME accounts --- docs/production/index.rst | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/production/index.rst b/docs/production/index.rst index 9f90c0cc..21fca650 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -511,3 +511,45 @@ The following must be added to the config file to activate the pinning (the pinn KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg== -----END CERTIFICATE----- """ + + +LetsEncrypt: Using a pre-existing ACME account +----------------------------------------------- + +Let's Encrypt allows reusing an existing ACME account, to create and especially revoke certificates. The current +implementation in the acme plugin, only allows for a single account for all ACME authorities, which might be an issue, +when you try to use Let's Encrypt together with another certificate authority that uses the ACME protocol. + +To use an existing account, you need to configure the `ACME_PRIVATE_KEY` and `ACME_REGR` variables in the lemur +configuration. + +`ACME_PRIVATE_KEY` needs to be in the JWK format:: + + { + "kty": "RSA", + "n": "yr1qBwHizA7ME_iV32bY10ILp.....", + "e": "AQAB", + "d": "llBlYhil3I.....", + "p": "-5LW2Lewogo.........", + "q": "zk6dHqHfHksd.........", + "dp": "qfe9fFIu3mu.......", + "dq": "cXFO-loeOyU.......", + "qi": "AfK1sh0_8sLTb..........." + } + + +Using `python-jwt` converting a existing private key in PEM format is quite easy:: + + import python_jwt as jwt, jwcrypto.jwk as jwk + + priv_key = jwk.JWK.from_pem(b"""-----BEGIN RSA PRIVATE KEY----- + ... + -----END RSA PRIVATE KEY-----""") + + print(priv_key.export()) + +`ACME_REGR` needs to be a valid JSON with a `body` and a `uri` attribute, similar to this:: + + {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/"} + +The uri can be retrieved from the ACME create account endpoint, when trying to create a new account, using the existing key. \ No newline at end of file From ae1ead6d7551dab0435e2f553bf2af469158aa4f Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 13:04:51 +0200 Subject: [PATCH 31/34] Document ACME plugin specific configurations --- docs/administration.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 0cec16a0..4c0477aa 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1171,6 +1171,23 @@ The following configuration properties are required to use the PowerDNS ACME Plu File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. +ACME Plugin +~~~~~~~~~~~~ + +The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME +account. + + +.. data:: ACME_PRIVATE_KEY + :noindex: + + This is the private key, the account was registered with (in JWK format) + +.. data:: ACME_REGR + :noindex: + + This is the registration for the ACME account, the most important part is the uri attribute (in JSON) + .. _CommandLineInterface: Command Line Interface From 4f1e09e3afe88c5d04918e0acf1da8effc0c1b16 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 13:20:35 +0200 Subject: [PATCH 32/34] Add reference from configuration options, to more detailed explanation --- docs/administration.rst | 2 +- docs/production/index.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 4c0477aa..f44ad1a3 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1175,7 +1175,7 @@ ACME Plugin ~~~~~~~~~~~~ The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME -account. +account. See :ref:`Using a pre-existing ACME account ` for more details. .. data:: ACME_PRIVATE_KEY diff --git a/docs/production/index.rst b/docs/production/index.rst index 21fca650..bace15d3 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -513,6 +513,8 @@ The following must be added to the config file to activate the pinning (the pinn """ +.. _AcmeAccountReuse: + LetsEncrypt: Using a pre-existing ACME account ----------------------------------------------- From 21c2255c754b48fd2a1887cad64098fa73b99f03 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 19:14:09 +0200 Subject: [PATCH 33/34] Minor spelling improvements --- docs/production/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/production/index.rst b/docs/production/index.rst index bace15d3..c6f561ca 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -540,7 +540,7 @@ configuration. } -Using `python-jwt` converting a existing private key in PEM format is quite easy:: +Using `python-jwt` converting an existing private key in PEM format is quite easy:: import python_jwt as jwt, jwcrypto.jwk as jwk @@ -554,4 +554,4 @@ Using `python-jwt` converting a existing private key in PEM format is quite easy {"body": {}, "uri": "https://acme-staging-v02.api.letsencrypt.org/acme/acct/"} -The uri can be retrieved from the ACME create account endpoint, when trying to create a new account, using the existing key. \ No newline at end of file +The URI can be retrieved from the ACME create account endpoint when creating a new account, using the existing key. \ No newline at end of file From e5961146b9c8bcb2c83fabcface46c578c0da2e5 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 14:22:58 -0600 Subject: [PATCH 34/34] session hook complains about metadata + consistent language. --- lemur/plugins/lemur_entrust/plugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 50b9d929..9b7848ed 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -20,7 +20,7 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ - metrics.send("entrust_status_code", "counter", 1, metadata={"status_code": r.status_code}) + metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1) def determine_end_date(end_date): @@ -206,12 +206,12 @@ class EntrustIssuerPlugin(IssuerPlugin): metrics.send("entrust_revoke_certificate", "counter", 1) return handle_response(response) - def deactivate_certificate(self, certificate, comments): + def deactivate_certificate(self, certificate): """Deactivates an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") - revoke_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" - response = self.session.post(revoke_url) - metrics.send("entrust_revoke_certificate", "counter", 1) + deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" + response = self.session.post(deactivate_url) + metrics.send("entrust_deactivate_certificate", "counter", 1) return handle_response(response) @staticmethod