From c3e0597ef1f537462be1e6fd2c71bca0b7790039 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 28 Jan 2021 16:53:15 -0800 Subject: [PATCH 01/11] introducing ACME_ADDITIONAL_ATTEMPTS --- lemur/common/celery.py | 3 ++- lemur/constants.py | 3 +++ lemur/pending_certificates/cli.py | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 578592dc..f852e459 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -20,6 +20,7 @@ from flask import current_app from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate from lemur.common.redis import RedisHandler +from lemur.constants import ACME_ADDITIONAL_ATTEMPTS from lemur.destinations import service as destinations_service from lemur.dns_providers import cli as cli_dns_providers from lemur.endpoints import cli as cli_endpoints @@ -301,7 +302,7 @@ def fetch_acme_cert(id): error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn - if pending_cert.number_attempts > 4: + if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS: error_log["message"] = "Deleting pending certificate" send_pending_failure_notification( pending_cert, notify_owner=pending_cert.notify diff --git a/lemur/constants.py b/lemur/constants.py index 64bee4c3..e89160f5 100644 --- a/lemur/constants.py +++ b/lemur/constants.py @@ -12,6 +12,9 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}" SUCCESS_METRIC_STATUS = "success" FAILURE_METRIC_STATUS = "failure" +# when ACME attempts to resolve a certificate try in total 3 times +ACME_ADDITIONAL_ATTEMPTS = 2 + CERTIFICATE_KEY_TYPES = [ "RSA2048", "RSA4096", diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index 2ff29f10..73b0ce2b 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -12,10 +12,12 @@ from flask import current_app from flask_script import Manager from lemur.authorities.service import get as get_authority +from lemur.constants import ACME_ADDITIONAL_ATTEMPTS from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins + manager = Manager(usage="Handles pending certificate related tasks.") @@ -107,7 +109,7 @@ def fetch_all_acme(): error_log["last_error"] = cert.get("last_error") error_log["cn"] = pending_cert.cn - if pending_cert.number_attempts > 4: + if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS: error_log["message"] = "Marking pending certificate as resolved" send_pending_failure_notification( pending_cert, notify_owner=pending_cert.notify From da8d3f42d279e6682139eac4177f1a951d542617 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 28 Jan 2021 16:53:42 -0800 Subject: [PATCH 02/11] documenting number of attempts --- docs/administration.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 9af08407..3d4bad7c 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -362,6 +362,9 @@ Whenever a pending ACME certificate fails to be issued, Lemur will send a notifi and security team (as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter). This email is not sent if the pending certificate had notifications disabled. +Lemur will attempt 3x times to resolve a pending certificate. +This can at times result into 3 duplicate certificates, if all certificate attempts get resolved. + **Certificate rotation** Whenever a cert is rotated, Lemur will send a notification via email to the certificate owner. This notification is From 9b0cc3bbef4b15a3be17c80cb5eb75391655cc9f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 28 Jan 2021 16:54:06 -0800 Subject: [PATCH 03/11] consolidating the documentation for acme --- docs/administration.rst | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 3d4bad7c..f150296b 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -823,6 +823,20 @@ ACME Plugin Enables delegated DNS domain validation using CNAMES. When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges. +The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME +account. See :ref:`Using a pre-existing ACME account ` for more details. + + +.. data:: ACME_PRIVATE_KEY + :noindex: + + This is the private key, the account was registered with (in JWK format) + +.. data:: ACME_REGR + :noindex: + + This is the registration for the ACME account, the most important part is the uri attribute (in JSON) + Active Directory Certificate Services Plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1339,23 +1353,6 @@ The following configuration properties are required to use the PowerDNS ACME Plu File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. -ACME Plugin -~~~~~~~~~~~~ - -The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME -account. See :ref:`Using a pre-existing ACME account ` for more details. - - -.. data:: ACME_PRIVATE_KEY - :noindex: - - This is the private key, the account was registered with (in JWK format) - -.. data:: ACME_REGR - :noindex: - - This is the registration for the ACME account, the most important part is the uri attribute (in JSON) - .. _CommandLineInterface: Command Line Interface From 5fb98f747cfe45446a2b2139e528c40c296d5e4f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 29 Jan 2021 15:45:51 -0800 Subject: [PATCH 04/11] comment --- lemur/certificates/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index e6414933..b9bc16f0 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -388,6 +388,7 @@ def create(**kwargs): cert = Certificate(**kwargs) kwargs["creator"].certificates.append(cert) else: + # ACME path cert = PendingCertificate(**kwargs) kwargs["creator"].pending_certificates.append(cert) From 31b20e0a30de160a8d17d49e5a551dd8a015b5b0 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 29 Jan 2021 15:46:22 -0800 Subject: [PATCH 05/11] ensuring a resolved job doesn't get resolved twice --- lemur/common/celery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f852e459..d5faf578 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -274,7 +274,8 @@ def fetch_acme_cert(id): real_cert = cert.get("cert") # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3 pending_cert = pending_certificate_service.get(cert.get("pending_cert").id) - if not pending_cert: + if not pending_cert or pending_cert.resolved: + # pending_cert is cleared or it was resolved by another process log_data[ "message" ] = "Pending certificate doesn't exist anymore. Was it resolved by another process?" From 5fd167ad20141e767d3c1d41cf40e84d4e843b46 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 18:23:44 +0000 Subject: [PATCH 06/11] Bump pre-commit from 2.9.3 to 2.10.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.9.3 to 2.10.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.9.3...v2.10.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e0df0ccd..65c44416 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -48,7 +48,7 @@ nodeenv==1.5.0 # pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.9.3 +pre-commit==2.10.0 # via -r requirements-dev.in pycodestyle==2.6.0 # via flake8 From 144d576d03b15016c70973e54bf0021b4b8f23f5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 18:34:59 +0000 Subject: [PATCH 07/11] Bump coverage from 5.3.1 to 5.4 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.3.1 to 5.4. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.3.1...coverage-5.4) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 729ddb7c..ae8c4163 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -42,7 +42,7 @@ click==7.1.2 # via # black # flask -coverage==5.3.1 +coverage==5.4 # via -r requirements-tests.in cryptography==3.3.1 # via From 88dd4d61e299dc03e86bc069c1e90bcbefa52ce9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 18:51:13 +0000 Subject: [PATCH 08/11] Bump pytest from 6.2.1 to 6.2.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.1 to 6.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.1...6.2.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index ae8c4163..79d4e269 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -148,7 +148,7 @@ pytest-flask==1.1.0 # via -r requirements-tests.in pytest-mock==3.5.1 # via -r requirements-tests.in -pytest==6.2.1 +pytest==6.2.2 # via # -r requirements-tests.in # pytest-flask From c9462ea8b7702f40484044800c890d646ce7702c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 19:06:39 +0000 Subject: [PATCH 09/11] Bump jinja2 from 2.11.2 to 2.11.3 Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.2 to 2.11.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/master/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/2.11.2...2.11.3) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 83 +++++++++++++++++++++++++++++------------- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 59 insertions(+), 28 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index f5c94cea..e6674f70 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,32 +4,63 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -alabaster==0.7.12 # via sphinx -babel==2.8.0 # via sphinx -certifi==2020.12.5 # via requests -chardet==3.0.4 # via requests -docutils==0.15.2 # via sphinx -idna==2.9 # via requests -imagesize==1.2.0 # via sphinx -jinja2==2.11.2 # via sphinx -markupsafe==1.1.1 # via jinja2 -packaging==20.3 # via sphinx -pygments==2.6.1 # via sphinx -pyparsing==2.4.7 # via packaging -pytz==2019.3 # via babel -requests==2.25.1 # via sphinx -six==1.15.0 # via packaging, sphinxcontrib-httpdomain -snowballstemmer==2.0.0 # via sphinx -sphinx-rtd-theme==0.5.1 # via -r requirements-docs.in -sphinx==3.4.3 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain -sphinxcontrib-applehelp==1.0.2 # via sphinx -sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==1.0.3 # via sphinx -sphinxcontrib-httpdomain==1.7.0 # via -r requirements-docs.in -sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 # via sphinx -sphinxcontrib-serializinghtml==1.1.4 # via sphinx -urllib3==1.25.8 # via requests +alabaster==0.7.12 + # via sphinx +babel==2.8.0 + # via sphinx +certifi==2020.12.5 + # via requests +chardet==3.0.4 + # via requests +docutils==0.15.2 + # via sphinx +idna==2.9 + # via requests +imagesize==1.2.0 + # via sphinx +jinja2==2.11.3 + # via sphinx +markupsafe==1.1.1 + # via jinja2 +packaging==20.3 + # via sphinx +pygments==2.6.1 + # via sphinx +pyparsing==2.4.7 + # via packaging +pytz==2019.3 + # via babel +requests==2.25.1 + # via sphinx +six==1.15.0 + # via + # packaging + # sphinxcontrib-httpdomain +snowballstemmer==2.0.0 + # via sphinx +sphinx-rtd-theme==0.5.1 + # via -r requirements-docs.in +sphinx==3.4.3 + # via + # -r requirements-docs.in + # sphinx-rtd-theme + # sphinxcontrib-httpdomain +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==1.0.3 + # via sphinx +sphinxcontrib-httpdomain==1.7.0 + # via -r requirements-docs.in +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.4 + # via sphinx +urllib3==1.25.8 + # via requests # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index 79d4e269..8d13db92 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -86,7 +86,7 @@ iniconfig==1.0.1 # via pytest itsdangerous==1.1.0 # via flask -jinja2==2.11.2 +jinja2==2.11.3 # via # flask # moto diff --git a/requirements.txt b/requirements.txt index 7390c54a..dd9e869b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -117,7 +117,7 @@ itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks -jinja2==2.11.2 +jinja2==2.11.3 # via # -r requirements.in # flask From 9883173450c8626a4a9356cfa74c53a331dbbd5f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 19:18:40 +0000 Subject: [PATCH 10/11] Bump boto3 from 1.16.60 to 1.16.63 Bumps [boto3](https://github.com/boto/boto3) from 1.16.60 to 1.16.63. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.60...1.16.63) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 8d13db92..87d46e5a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -18,13 +18,13 @@ bandit==1.7.0 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.60 +boto3==1.16.63 # via # aws-sam-translator # moto boto==2.49.0 # via moto -botocore==1.19.60 +botocore==1.19.63 # via # aws-xray-sdk # boto3 diff --git a/requirements.txt b/requirements.txt index dd9e869b..c9c9f104 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,9 +31,9 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.16.60 +boto3==1.16.63 # via -r requirements.in -botocore==1.19.60 +botocore==1.19.63 # via # -r requirements.in # boto3 From 189adcb9279db7ba698357f208fba1527d04c0db Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 1 Feb 2021 12:55:57 -0800 Subject: [PATCH 11/11] resolving the of index error --- lemur/certificates/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index e00f6ea7..0a51cedc 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -261,10 +261,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c log_data["endpoint"] = endpoint.dnsname log_data["certificate"] = endpoint.certificate.replaced[0].name - request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) print( f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}" ) + request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit) current_app.logger.info(log_data) status = SUCCESS_METRIC_STATUS @@ -427,7 +427,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes 1, metric_tags={ "status": FAILURE_METRIC_STATUS, - "new_certificate_name": str(endpoint.certificate.replaced[0].name), + "new_certificate_name": str(log_data["certificate"]), "endpoint_name": str(endpoint.dnsname), "message": str(message), "region": str(region),