From 0129e97dfe779c2d46208979c232c812d795793c Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 13:56:24 -0600 Subject: [PATCH 01/88] address karma CVEs --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9b899176..20e1e356 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "gulp-util": "^3.0.1", "http-proxy": "~1.16.2", "jshint-stylish": "^2.2.1", - "karma": "~1.3.0", + "karma": "^4.4.1", "karma-jasmine": "^1.1.0", "main-bower-files": "^2.13.1", "merge-stream": "^1.0.1", From a0a8c155f6f2bdae75720b1165710c180c540c92 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 13:59:41 -0600 Subject: [PATCH 02/88] patch gulp-protractor --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 20e1e356..2cee77ab 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "gulp-notify": "^2.2.0", "gulp-plumber": "^1.1.0", "gulp-print": "^2.0.1", - "gulp-protractor": "3.0.0", + "gulp-protractor": "^4.1.1", "gulp-replace": "~0.5.3", "gulp-replace-task": "~0.11.0", "gulp-rev": "^7.1.2", From 7a2ca8969eec68fc932dd4c9b8660dbd9fe26d62 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 14:08:22 -0600 Subject: [PATCH 03/88] patch gulp-imagemin --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2cee77ab..df0bbe9a 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "gulp-flatten": "^0.3.1", "gulp-foreach": "0.1.0", "gulp-if": "^2.0.2", - "gulp-imagemin": "^3.1.1", + "gulp-imagemin": "^7.1.0", "gulp-inject": "~4.1.0", "gulp-jshint": "^2.0.4", "gulp-less": "^3.0.3", From 980360b94a1530e049aee88360e374a10eda73d7 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 14:09:57 -0600 Subject: [PATCH 04/88] patch gulp-less --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index df0bbe9a..a34fc745 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "gulp-imagemin": "^7.1.0", "gulp-inject": "~4.1.0", "gulp-jshint": "^2.0.4", - "gulp-less": "^3.0.3", + "gulp-less": "^4.0.1", "gulp-load-plugins": "^1.4.0", "gulp-minify-css": "^1.2.4", "gulp-minify-html": "~1.0.6", From ac0282529ef9ec60050be4681c1ce278cb0eab04 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 3 Feb 2020 11:05:20 -0800 Subject: [PATCH 05/88] adding basic logging on success --- lemur/plugins/lemur_acme/plugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e38870d8..54493f97 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -183,6 +183,10 @@ class AcmeHandler(object): else: raise + current_app.logger.debug( + f"Successfully resolved Acme order: {order.uri}", exc_info=True + ) + pem_certificate = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate( From 6c7bb5f9b73fe9bca34bfadb2f8a1eae01a5bcd4 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Thu, 13 Feb 2020 07:35:35 +0100 Subject: [PATCH 06/88] Fixed TLS secret format ( #2913 ) The Plugin handled the TLS secret format wrong: it sent chain certificate instead of requested public certificate #2913 --- lemur/plugins/lemur_kubernetes/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index 62ffffda..f7ff00f7 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -96,7 +96,7 @@ def build_secret(secret_format, secret_name, body, private_key, cert_chain): if secret_format == "TLS": secret["type"] = "kubernetes.io/tls" secret["data"] = { - "tls.crt": base64encode(cert_chain), + "tls.crt": base64encode(body), "tls.key": base64encode(private_key), } if secret_format == "Certificate": From 571c8bf42d83f32ee808840edd2287a665a2f6fc Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Thu, 13 Feb 2020 07:38:04 +0100 Subject: [PATCH 07/88] Error when validity_end date is empty #2905 this lines of code (114ff) in threw an error, when the validity_end date was empty: if options.get("validity_end") > arrow.utcnow().shift(years=2): raise Exception( "Verisign issued certificates cannot exceed two years in validity" ) Actually, they are not needed, because immidiately following is a check for an empty validity_end and for the length of the entered period. When I commented it out for testing, the error was gone and everything worked as expected. --- lemur/plugins/lemur_verisign/plugin.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 7bf517b7..6a49364f 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -111,11 +111,7 @@ def process_options(options): data["subject_alt_names"] = ",".join(get_additional_names(options)) - if options.get("validity_end") > arrow.utcnow().shift(years=2): - raise Exception( - "Verisign issued certificates cannot exceed two years in validity" - ) - + if options.get("validity_end"): # VeriSign (Symantec) only accepts strictly smaller than 2 year end date if options.get("validity_end") < arrow.utcnow().shift(years=2, days=-1): From 2b849a65205a00ea129247263c4ac300f8ebd25d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 13 Feb 2020 15:58:07 -0800 Subject: [PATCH 08/88] Update plugin.py making lint happy --- lemur/plugins/lemur_verisign/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 6a49364f..3e7c383f 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -110,7 +110,6 @@ def process_options(options): } data["subject_alt_names"] = ",".join(get_additional_names(options)) - if options.get("validity_end"): # VeriSign (Symantec) only accepts strictly smaller than 2 year end date From af21225918929e936ed5781246c3cca05fe44d4d Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 13 Feb 2020 16:38:33 -0800 Subject: [PATCH 09/88] adding logging on sucess and metric submission of URL for certificate issuance --- lemur/plugins/lemur_acme/plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 54493f97..179285b3 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -172,7 +172,7 @@ class AcmeHandler(object): except (AcmeError, TimeoutError): sentry.captureException(extra={"order_url": str(order.uri)}) - metrics.send("request_certificate_error", "counter", 1) + metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri}) current_app.logger.error( f"Unable to resolve Acme order: {order.uri}", exc_info=True ) @@ -183,7 +183,8 @@ class AcmeHandler(object): else: raise - current_app.logger.debug( + metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri}) + current_app.logger.info( f"Successfully resolved Acme order: {order.uri}", exc_info=True ) From 8e3cc93d6ae64bd6b75b1d54a7903d87052bb96f Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Fri, 14 Feb 2020 07:50:18 +0100 Subject: [PATCH 10/88] Whitespaces in empty line 113 removed --- lemur/plugins/lemur_verisign/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 3e7c383f..a0e2d1cb 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -110,7 +110,7 @@ def process_options(options): } data["subject_alt_names"] = ",".join(get_additional_names(options)) - + if options.get("validity_end"): # VeriSign (Symantec) only accepts strictly smaller than 2 year end date if options.get("validity_end") < arrow.utcnow().shift(years=2, days=-1): From fabcad1e46c44d86c7dfe69204fdf45cbf1c189c Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 15:52:24 +0100 Subject: [PATCH 11/88] New variable VERISIGN_PRODUCT_(authority.name) If there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type else default to "Server", to be compatoible with former versions. This enables the use of different Verisign authorities for differnt cert-products eg. EV or Standard Certs --- lemur/plugins/lemur_verisign/plugin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index a0e2d1cb..c74a71f1 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -98,10 +98,18 @@ def process_options(options): :param options: :return: dict or valid verisign options """ + + # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type + # else default to "Server", to be compatoible with former versions + authority = options.get("authority").name.upper() + product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) + if product_type is None: + product_type ="Server" + data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", - "certProductType": "Server", + "certProductType": product_type, "firstName": current_app.config.get("VERISIGN_FIRST_NAME"), "lastName": current_app.config.get("VERISIGN_LAST_NAME"), "signatureAlgorithm": "sha256WithRSAEncryption", From bfa953270d3840b9d96807b69c30466138f89b39 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 16:04:44 +0100 Subject: [PATCH 12/88] Fixed whitespace error --- lemur/plugins/lemur_verisign/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index c74a71f1..cd716e84 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -104,7 +104,7 @@ def process_options(options): authority = options.get("authority").name.upper() product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) if product_type is None: - product_type ="Server" + product_type = "Server" data = { "challenge": get_psuedo_random_string(), From 3693bc2d8be12cafce118c196fadf72e8606d672 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 16:09:25 +0100 Subject: [PATCH 13/88] removed whitespaces inserted by online editor --- lemur/plugins/lemur_verisign/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index cd716e84..ca606baf 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -98,14 +98,14 @@ def process_options(options): :param options: :return: dict or valid verisign options """ - + # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type # else default to "Server", to be compatoible with former versions authority = options.get("authority").name.upper() product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) if product_type is None: product_type = "Server" - + data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", From a70a49e4e9cffa757a608022e51e14646d766513 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sat, 15 Feb 2020 16:11:58 +0100 Subject: [PATCH 14/88] Update plugin.py --- lemur/plugins/lemur_verisign/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index ca606baf..6d9182df 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -98,14 +98,12 @@ def process_options(options): :param options: :return: dict or valid verisign options """ - # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type # else default to "Server", to be compatoible with former versions authority = options.get("authority").name.upper() product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) if product_type is None: product_type = "Server" - data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", From 1815c8997064130e903e172a7afc4a13df23f714 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Sun, 16 Feb 2020 09:28:52 +0100 Subject: [PATCH 15/88] Made the change more elegant As suggested by @hosseinsh. This is of course more elegant. --- lemur/plugins/lemur_verisign/plugin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 6d9182df..0864657a 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -101,9 +101,7 @@ def process_options(options): # if there is a config variable with VERISIGN_PRODUCT_ take the value as Cert product-type # else default to "Server", to be compatoible with former versions authority = options.get("authority").name.upper() - product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority)) - if product_type is None: - product_type = "Server" + product_type = current_app.config.get("VERISIGN_PRODUCT_{0}".format(authority), "Server") data = { "challenge": get_psuedo_random_string(), "serverType": "Apache", From 3fd0d3e1416fcb79aedb122c83c8c23213967e11 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 17 Feb 2020 12:40:36 +0100 Subject: [PATCH 16/88] Added VERISIGN_INTERMEDIATE_ parameter When using the VERISIGN_PRODUCT_ Parameter one also has to add this parameter: VERISIGN_INTERMEDIATE_ = """ """ While doing this, I also added code, so the external_id field is filled with data from CA-Answer --- lemur/plugins/lemur_verisign/plugin.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 0864657a..2d108071 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -209,7 +209,7 @@ class VerisignIssuerPlugin(IssuerPlugin): response = self.session.post(url, data=data) try: - cert = handle_response(response.content)["Response"]["Certificate"] + response_dict = handle_response(response.content) except KeyError: metrics.send( "verisign_create_certificate_error", @@ -221,8 +221,12 @@ class VerisignIssuerPlugin(IssuerPlugin): extra={"common_name": issuer_options.get("common_name", "")} ) raise Exception(f"Error with Verisign: {response.content}") - # TODO add external id - return cert, current_app.config.get("VERISIGN_INTERMEDIATE"), None + # DONE: TODO add external id + authority = issuer_options.get("authority").name.upper() + cert = response_dict['Response']['Certificate'] + external_id = response_dict['Response']['Transaction_ID'] + chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) + return cert, chain, external_id @staticmethod def create_authority(options): From ed3472d029c88b678e1e9e5104151bd888b207b4 Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 17 Feb 2020 15:21:29 +0100 Subject: [PATCH 17/88] Update plugin.py --- lemur/plugins/lemur_verisign/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 2d108071..479f3215 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -209,7 +209,7 @@ class VerisignIssuerPlugin(IssuerPlugin): response = self.session.post(url, data=data) try: - response_dict = handle_response(response.content) + response_dict = handle_response(response.content) except KeyError: metrics.send( "verisign_create_certificate_error", @@ -225,7 +225,7 @@ class VerisignIssuerPlugin(IssuerPlugin): authority = issuer_options.get("authority").name.upper() cert = response_dict['Response']['Certificate'] external_id = response_dict['Response']['Transaction_ID'] - chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) + chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) return cert, chain, external_id @staticmethod From e75df1ddc92a79dc197c698d38078f8c3d80157a Mon Sep 17 00:00:00 2001 From: sirferl <41906265+sirferl@users.noreply.github.com> Date: Mon, 17 Feb 2020 19:04:20 +0100 Subject: [PATCH 18/88] Update plugin.py --- lemur/plugins/lemur_verisign/plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_verisign/plugin.py b/lemur/plugins/lemur_verisign/plugin.py index 479f3215..f913861c 100644 --- a/lemur/plugins/lemur_verisign/plugin.py +++ b/lemur/plugins/lemur_verisign/plugin.py @@ -221,10 +221,11 @@ class VerisignIssuerPlugin(IssuerPlugin): extra={"common_name": issuer_options.get("common_name", "")} ) raise Exception(f"Error with Verisign: {response.content}") - # DONE: TODO add external id authority = issuer_options.get("authority").name.upper() cert = response_dict['Response']['Certificate'] - external_id = response_dict['Response']['Transaction_ID'] + external_id = None + if 'Transaction_ID' in response_dict['Response'].keys(): + external_id = response_dict['Response']['Transaction_ID'] chain = current_app.config.get("VERISIGN_INTERMEDIATE_{0}".format(authority), current_app.config.get("VERISIGN_INTERMEDIATE")) return cert, chain, external_id From 64a7faffd982213acf566f8b9b79229ba559c2a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 18:30:10 +0000 Subject: [PATCH 19/88] Bump bleach from 3.1.0 to 3.1.1 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.0 to 3.1.1. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.0...v3.1.1) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d1423888..224789f6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,37 +5,39 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.0 # via readme-renderer +bleach==3.1.1 # via readme-renderer certifi==2019.11.28 # via requests +cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit chardet==3.0.4 # via requests +cryptography==2.8 # via secretstorage docutils==0.15.2 # via readme-renderer flake8==3.5.0 identify==1.4.9 # via pre-commit idna==2.8 # via requests -importlib-metadata==1.3.0 # via keyring, pre-commit, twine invoke==1.3.0 +jeepney==0.4.2 # via secretstorage keyring==21.0.0 # via twine mccabe==0.6.1 # via flake8 -more-itertools==8.0.2 # via zipp nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine pre-commit==1.21.0 pycodestyle==2.3.1 # via flake8 +pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 pygments==2.5.2 # via readme-renderer pyyaml==5.2 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine -six==1.13.0 # via bleach, cfgv, pre-commit, readme-renderer +secretstorage==3.1.2 # via keyring +six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.41.1 # via twine twine==3.1.1 urllib3==1.25.7 # via requests virtualenv==16.7.9 # via pre-commit webencodings==0.5.1 # via bleach -zipp==0.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 318292704d3c2167c9291bbb26e77ca2df8c543e Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 14:29:17 -0800 Subject: [PATCH 20/88] fixing default/max DigiCert validity values --- lemur/plugins/lemur_digicert/plugin.py | 88 +++--- .../lemur_digicert/tests/test_digicert.py | 266 ++++++++++-------- 2 files changed, 202 insertions(+), 152 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 88ea5b6b..b9508357 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -14,21 +14,17 @@ .. moduleauthor:: Kevin Glisson """ import json + import arrow -import requests - import pem -from retrying import retry - -from flask import current_app - +import requests from cryptography import x509 - -from lemur.extensions import metrics +from flask import current_app from lemur.common.utils import validate_conf -from lemur.plugins.bases import IssuerPlugin, SourcePlugin - +from lemur.extensions import metrics from lemur.plugins import lemur_digicert as digicert +from lemur.plugins.bases import IssuerPlugin, SourcePlugin +from retrying import retry def log_status_code(r, *args, **kwargs): @@ -64,24 +60,38 @@ def signature_hash(signing_algorithm): raise Exception("Unsupported signing algorithm.") -def determine_validity_years(end_date): +def determine_validity_years(years): """Given an end date determine how many years into the future that date is. + :param years: + :return: validity in years + """ + default_years = current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1) + max_years = current_app.config.get("DIGICERT_MAX_VALIDITY", default_years) + + if years > max_years: + return max_years + if years not in [1, 2, 3]: + return default_years + else: + return years + + +def determine_end_date(end_date): + """ + Determine appropriate end date :param end_date: - :return: str validity in years + :return: validity_end """ - now = arrow.utcnow() + default_years = current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1) + max_validity_end = arrow.utcnow().shift(years=current_app.config.get("DIGICERT_MAX_VALIDITY", default_years)) - if end_date < now.shift(years=+1): - return 1 - elif end_date < now.shift(years=+2): - return 2 - elif end_date < now.shift(years=+3): - return 3 + if not end_date: + end_date = arrow.utcnow().shift(years=default_years) - raise Exception( - "DigiCert issued certificates cannot exceed three" " years in validity" - ) + if end_date > max_validity_end: + end_date = max_validity_end + return end_date def get_additional_names(options): @@ -107,12 +117,6 @@ def map_fields(options, csr): :param csr: :return: dict or valid DigiCert options """ - if not options.get("validity_years"): - if not options.get("validity_end"): - options["validity_years"] = current_app.config.get( - "DIGICERT_DEFAULT_VALIDITY", 1 - ) - data = dict( certificate={ "common_name": options["common_name"], @@ -125,9 +129,11 @@ def map_fields(options, csr): data["certificate"]["dns_names"] = get_additional_names(options) if options.get("validity_years"): - data["validity_years"] = options["validity_years"] + data["validity_years"] = determine_validity_years(options.get("validity_years")) + elif options.get("validity_end"): + data["custom_expiration_date"] = determine_end_date(options.get("validity_end")).format("YYYY-MM-DD") else: - data["custom_expiration_date"] = options["validity_end"].format("YYYY-MM-DD") + data["validity_years"] = determine_validity_years(0) if current_app.config.get("DIGICERT_PRIVATE", False): if "product" in data: @@ -144,18 +150,15 @@ def map_cis_fields(options, csr): :param options: :param csr: - :return: + :return: data """ - if not options.get("validity_years"): - if not options.get("validity_end"): - options["validity_end"] = arrow.utcnow().shift( - years=current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1) - ) - options["validity_years"] = determine_validity_years(options["validity_end"]) + + if options.get("validity_years"): + validity_end = determine_end_date(arrow.utcnow().shift(years=options["validity_years"])) + elif options.get("validity_end"): + validity_end = determine_end_date(options.get("validity_end")) else: - options["validity_end"] = arrow.utcnow().shift( - years=options["validity_years"] - ) + validity_end = determine_end_date(False) data = { "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES", {}).get(options['authority'].name), @@ -164,7 +167,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": options["validity_end"].format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z" }, "organization": { "name": options["organization"], @@ -173,7 +176,8 @@ def map_cis_fields(options, csr): } # possibility to default to a SIGNING_ALGORITHM for a given profile if current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name): - data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name) + data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get( + options['authority'].name) return data diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 77b0a1fa..b9576f51 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -1,117 +1,125 @@ -import pytest -import arrow import json -from unittest.mock import patch - -from freezegun import freeze_time - -from lemur.tests.vectors import CSR_STR +import arrow +import pytest from cryptography import x509 +from freezegun import freeze_time +from lemur.plugins.lemur_digicert import plugin +from lemur.tests.vectors import CSR_STR +from mock import Mock, patch -def test_map_fields_with_validity_end_and_start(app): - from lemur.plugins.lemur_digicert.plugin import map_fields - - names = [u"one.example.com", u"two.example.com", u"three.example.com"] - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "validity_end": arrow.get(2017, 5, 7), - "validity_start": arrow.get(2016, 10, 30), - } - - data = map_fields(options, CSR_STR) - - assert data == { - "certificate": { - "csr": CSR_STR, - "common_name": "example.com", - "dns_names": names, - "signature_hash": "sha256", - }, - "organization": {"id": 111111}, - "custom_expiration_date": arrow.get(2017, 5, 7).format("YYYY-MM-DD"), +def config_mock(*args): + values = { + "DIGICERT_ORG_ID": 111111, + "DIGICERT_PRIVATE": False, + "DIGICERT_DEFAULT_SIGNING_ALGORITHM": "sha256", + "DIGICERT_DEFAULT_VALIDITY": 1, + "DIGICERT_MAX_VALIDITY": 2, + "DIGICERT_CIS_PROFILE_NAMES": {"digicert": 'digicert'}, + "DIGICERT_CIS_SIGNING_ALGORITHMS": {"verisign-sha1-intermediary": 'sha1'}, } + return values[args[0]] -def test_map_fields_with_validity_years(app): - from lemur.plugins.lemur_digicert.plugin import map_fields - - names = [u"one.example.com", u"two.example.com", u"three.example.com"] - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "validity_years": 2, - "validity_end": arrow.get(2017, 10, 30), - } - - data = map_fields(options, CSR_STR) - - assert data == { - "certificate": { - "csr": CSR_STR, - "common_name": "example.com", - "dns_names": names, - "signature_hash": "sha256", - }, - "organization": {"id": 111111}, - "validity_years": 2, - } +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_determine_validity_years(mock_current_app): + mock_current_app.config.get = Mock(return_value=2) + assert plugin.determine_validity_years(1) == 1 + assert plugin.determine_validity_years(0) == 2 + assert plugin.determine_validity_years(3) == 2 -def test_map_cis_fields(app, authority): - from lemur.plugins.lemur_digicert.plugin import map_cis_fields - - names = [u"one.example.com", u"two.example.com", u"three.example.com"] - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "organization": "Example, Inc.", - "organizational_unit": "Example Org", - "validity_end": arrow.get(2017, 5, 7), - "validity_start": arrow.get(2016, 10, 30), - "authority": authority, - } - - data = map_cis_fields(options, CSR_STR) - - assert data == { - "common_name": "example.com", - "csr": CSR_STR, - "additional_dns_names": names, - "signature_hash": "sha256", - "organization": {"name": "Example, Inc.", "units": ["Example Org"]}, - "validity": { - "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z" - }, - "profile_name": None, - } - - options = { - "common_name": "example.com", - "owner": "bob@example.com", - "description": "test certificate", - "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, - "organization": "Example, Inc.", - "organizational_unit": "Example Org", - "validity_years": 2, - "authority": authority, - } - +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_determine_end_date(mock_current_app): + mock_current_app.config.get = Mock(return_value=2) with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime): - data = map_cis_fields(options, CSR_STR) + assert arrow.get(2018, 11, 3) == plugin.determine_end_date(0) + assert arrow.get(2018, 5, 7) == plugin.determine_end_date(arrow.get(2018, 5, 7)) + assert arrow.get(2018, 11, 3) == plugin.determine_end_date(arrow.get(2020, 5, 7)) - assert data == { + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_fields_with_validity_years(mock_current_app): + mock_current_app.config.get = Mock(side_effect=config_mock) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "validity_years": 2 + } + expected = { + "certificate": { + "csr": CSR_STR, + "common_name": "example.com", + "dns_names": names, + "signature_hash": "sha256", + }, + "organization": {"id": 111111}, + "validity_years": 2, + } + assert expected == plugin.map_fields(options, CSR_STR) + + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_fields_with_validity_end_and_start(mock_current_app): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7)) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "validity_end": arrow.get(2017, 5, 7), + "validity_start": arrow.get(2016, 10, 30), + } + + expected = { + "certificate": { + "csr": CSR_STR, + "common_name": "example.com", + "dns_names": names, + "signature_hash": "sha256", + }, + "organization": {"id": 111111}, + "custom_expiration_date": arrow.get(2017, 5, 7).format("YYYY-MM-DD"), + } + + assert expected == plugin.map_fields(options, CSR_STR) + + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_cis_fields_with_validity_years(mock_current_app, authority): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2018, 11, 3)) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "organization": "Example, Inc.", + "organizational_unit": "Example Org", + "validity_years": 2, + "authority": authority, + } + + expected = { "common_name": "example.com", "csr": CSR_STR, "additional_dns_names": names, @@ -123,21 +131,59 @@ def test_map_cis_fields(app, authority): "profile_name": None, } + assert expected == plugin.map_cis_fields(options, CSR_STR) -def test_signature_hash(app): - from lemur.plugins.lemur_digicert.plugin import signature_hash - assert signature_hash(None) == "sha256" - assert signature_hash("sha256WithRSA") == "sha256" - assert signature_hash("sha384WithRSA") == "sha384" - assert signature_hash("sha512WithRSA") == "sha512" +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, authority): + mock_current_app.config.get = Mock(side_effect=config_mock) + plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7)) + + with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash: + mock_signature_hash.return_value = "sha256" + + names = [u"one.example.com", u"two.example.com", u"three.example.com"] + options = { + "common_name": "example.com", + "owner": "bob@example.com", + "description": "test certificate", + "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, + "organization": "Example, Inc.", + "organizational_unit": "Example Org", + "validity_end": arrow.get(2017, 5, 7), + "validity_start": arrow.get(2016, 10, 30), + "authority": authority + } + + expected = { + "common_name": "example.com", + "csr": CSR_STR, + "additional_dns_names": names, + "signature_hash": "sha256", + "organization": {"name": "Example, Inc.", "units": ["Example Org"]}, + "validity": { + "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z" + }, + "profile_name": None, + } + + assert expected == plugin.map_cis_fields(options, CSR_STR) + + +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_signature_hash(mock_current_app, app): + mock_current_app.config.get = Mock(side_effect=config_mock) + assert plugin.signature_hash(None) == "sha256" + assert plugin.signature_hash("sha256WithRSA") == "sha256" + assert plugin.signature_hash("sha384WithRSA") == "sha384" + assert plugin.signature_hash("sha512WithRSA") == "sha512" with pytest.raises(Exception): - signature_hash("sdfdsf") + plugin.signature_hash("sdfdsf") def test_issuer_plugin_create_certificate( - certificate_="""\ + certificate_="""\ -----BEGIN CERTIFICATE----- abc -----END CERTIFICATE----- From 6c46481ffd9dd3dc0479a6de351b51fceb653091 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 14:40:50 -0800 Subject: [PATCH 21/88] simplifying return statement for validity years --- lemur/plugins/lemur_digicert/plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index b9508357..e5c4b2ce 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -72,8 +72,7 @@ def determine_validity_years(years): return max_years if years not in [1, 2, 3]: return default_years - else: - return years + return years def determine_end_date(end_date): From fdc1e20c234303adc27ea7768b72978d6bf5d80b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 17:27:15 -0800 Subject: [PATCH 22/88] updating config_mock defaults --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index b9576f51..1e9ebca4 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -17,7 +17,7 @@ def config_mock(*args): "DIGICERT_DEFAULT_VALIDITY": 1, "DIGICERT_MAX_VALIDITY": 2, "DIGICERT_CIS_PROFILE_NAMES": {"digicert": 'digicert'}, - "DIGICERT_CIS_SIGNING_ALGORITHMS": {"verisign-sha1-intermediary": 'sha1'}, + "DIGICERT_CIS_SIGNING_ALGORITHMS": {"digicert": 'digicert'}, } return values[args[0]] From a873d69859d2a68d3dd36a2981a721c951ec19ef Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Tue, 3 Mar 2020 18:24:48 -0800 Subject: [PATCH 23/88] adding documentation for DIGICERT_MAX_VALIDITY --- docs/administration.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 8f055147..341eda6a 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -735,6 +735,12 @@ The following configuration properties are required to use the Digicert issuer p This is the default validity (in years), if no end date is specified. (Default: 1) +.. data:: DIGICERT_MAX_VALIDITY + :noindex: + + This is the maximum validity (in years). (Default: DIGICERT_DEFAULT_VALIDITY) + + .. data:: DIGICERT_PRIVATE :noindex: From 9ef538305d8a91b06cd128945fe8f19e063ddc8a Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 4 Mar 2020 11:45:47 -0800 Subject: [PATCH 24/88] updating default language for digicert max validity option --- docs/administration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 341eda6a..6cbf42bd 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -738,7 +738,7 @@ The following configuration properties are required to use the Digicert issuer p .. data:: DIGICERT_MAX_VALIDITY :noindex: - This is the maximum validity (in years). (Default: DIGICERT_DEFAULT_VALIDITY) + This is the maximum validity (in years). (Default: value of DIGICERT_DEFAULT_VALIDITY) .. data:: DIGICERT_PRIVATE From c0004e506e538c849e6c69c6e7b42640256dc64f Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 4 Mar 2020 14:50:44 -0800 Subject: [PATCH 25/88] removing 2 year option from Lemur certificate request form --- .../app/angular/certificates/certificate/tracking.tpl.html | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html index 7ac2107f..027add0f 100644 --- a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html @@ -140,7 +140,6 @@ From 5dfb6acb17eaf34ce4970da3154934113188577b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 14:59:21 -0800 Subject: [PATCH 26/88] adding support for ACME_POWERDNS_VERIFY option to support CA Bundles and disabling Server validation --- docs/administration.rst | 9 +++++++++ lemur/plugins/lemur_acme/powerdns.py | 7 +++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 8f055147..ea9537da 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1008,6 +1008,15 @@ The following configuration properties are required to use the PowerDNS ACME Plu This is the number of times DNS Verification should be attempted (i.e. 20) + +.. data:: ACME_POWERDNS_VERIFY + :noindex: + + This configures how PowerDNS verifies TLS certificates. The PowerDNS Plugin relies on the requests library, supported options are as follows: + * True: Verifies the certificate chains to a known publicly-trusted CA. (Default) + * False: Disable certificate validation (Not Recommended) + * File/Dir path to CA Bundle: Verify that the certificate chains to a Certificate Authority in the provided CA bundle. + .. _CommandLineInterface: Command Line Interface diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index f3ad9965..1b7cf1d4 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -246,11 +246,12 @@ def _get_zone_name(domain, account_number): def _get(path, params=None): """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") + verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.get( f"{base_uri}{path}", headers=_generate_header(), params=params, - verify=True, + verify=verify_value, ) resp.raise_for_status() return resp.json() @@ -259,9 +260,11 @@ def _get(path, params=None): def _patch(path, payload): """ Execute a Patch request on the given URL (base_uri + path) with given payload """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") + verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.patch( f"{base_uri}{path}", data=json.dumps(payload), - headers=_generate_header() + headers=_generate_header(), + verify=verify_value, ) resp.raise_for_status() From b85fe2f2b5c95ba0d2f3a31c22d5daeade8ca378 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 15:03:43 -0800 Subject: [PATCH 27/88] updated documentation language --- docs/administration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index ea9537da..df3a5cff 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1012,10 +1012,10 @@ The following configuration properties are required to use the PowerDNS ACME Plu .. data:: ACME_POWERDNS_VERIFY :noindex: - This configures how PowerDNS verifies TLS certificates. The PowerDNS Plugin relies on the requests library, supported options are as follows: - * True: Verifies the certificate chains to a known publicly-trusted CA. (Default) - * False: Disable certificate validation (Not Recommended) - * File/Dir path to CA Bundle: Verify that the certificate chains to a Certificate Authority in the provided CA bundle. + This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options: + * True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) + * False: Disables certificate validation (Not Recommended) + * File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. .. _CommandLineInterface: From 771e72187a48b33c6ccea78aaa5e8b58907f8f76 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 15:24:56 -0800 Subject: [PATCH 28/88] updates based on feedback --- docs/administration.rst | 2 +- lemur/plugins/lemur_acme/powerdns.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index df3a5cff..d2dcf4c2 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1012,7 +1012,7 @@ The following configuration properties are required to use the PowerDNS ACME Plu .. data:: ACME_POWERDNS_VERIFY :noindex: - This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options: + This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options for the verify parameter: * True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) * False: Disables certificate validation (Not Recommended) * File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 1b7cf1d4..a26faaac 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -251,7 +251,7 @@ def _get(path, params=None): f"{base_uri}{path}", headers=_generate_header(), params=params, - verify=verify_value, + verify=verify_value ) resp.raise_for_status() return resp.json() @@ -265,6 +265,6 @@ def _patch(path, payload): f"{base_uri}{path}", data=json.dumps(payload), headers=_generate_header(), - verify=verify_value, + verify=verify_value ) resp.raise_for_status() From 6227e4aa89e0064de790c1152c4f045c11dfe670 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 5 Mar 2020 16:51:21 -0800 Subject: [PATCH 29/88] fixing formatting of ACME_POWERDNS_VERIFY options --- docs/administration.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index d2dcf4c2..0db374ff 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1013,9 +1013,12 @@ The following configuration properties are required to use the PowerDNS ACME Plu :noindex: This configures how TLS certificates on the PowerDNS API target are validated. The PowerDNS Plugin depends on the PyPi requests library, which supports the following options for the verify parameter: - * True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) - * False: Disables certificate validation (Not Recommended) - * File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. + + True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default) + + False: Disables certificate validation (Not Recommended) + + File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle. .. _CommandLineInterface: From 921d52b3608db7d2c9ce3dab0c64a12bb6726e14 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 13 Mar 2020 00:03:31 -0700 Subject: [PATCH 30/88] fixing get_dns_challenge() logic so duplicate domains (such as wildcard and not wildcard) do not match the wrong authorziations --- lemur/plugins/lemur_acme/plugin.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 91781966..e566352c 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -54,18 +54,30 @@ class AcmeHandler(object): current_app.logger.error(f"Unable to fetch DNS Providers: {e}") self.all_dns_providers = [] - def find_dns_challenge(self, host, authorizations): + def get_dns_challenges(self, host, authorizations): + """Get final domain to validate and dns challenges for it""" + + domain_to_validate, is_wildcard = self.strip_wildcard(host) dns_challenges = [] for authz in authorizations: - if not authz.body.identifier.value.lower() == host.lower(): + if not authz.body.identifier.value.lower() == domain_to_validate.lower(): + continue + if is_wildcard and not authz.body.wildcard: + continue + if not is_wildcard and authz.body.wildcard: continue for combo in authz.body.challenges: if isinstance(combo.chall, challenges.DNS01): dns_challenges.append(combo) - return dns_challenges - def maybe_remove_wildcard(self, host): - return host.replace("*.", "") + return domain_to_validate, dns_challenges + + def strip_wildcard(self, host): + """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" + prefix = "*." + if host.startswith(prefix): + return host[len(prefix):], True + return host, False def maybe_add_extension(self, host, dns_provider_options): if dns_provider_options and dns_provider_options.get( @@ -86,9 +98,7 @@ class AcmeHandler(object): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) change_ids = [] - - host_to_validate = self.maybe_remove_wildcard(host) - dns_challenges = self.find_dns_challenge(host_to_validate, order.authorizations) + host_to_validate, dns_challenges = self.get_dns_challenges(host, order.authorizations) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) @@ -325,7 +335,7 @@ class AcmeHandler(object): ) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") - host_to_validate = self.maybe_remove_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.host) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) @@ -357,7 +367,7 @@ class AcmeHandler(object): dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_challenges = authz_record.dns_challenge - host_to_validate = self.maybe_remove_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.host) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) From 593c35776c868cfe937c79a119bc52a03dec446d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:17:05 -0700 Subject: [PATCH 31/88] adding new methods for getting pending clean --- lemur/certificates/service.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 0e91b563..ff558284 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -117,6 +117,41 @@ def get_all_pending_cleaning(source): ) +def get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire): + """ + Retrieves all certificates that are available for cleaning: not attached to endpoint, + and within X days from expiration. + + :param days_to_expire: + :param source: + :return: + """ + expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD") + return ( + Certificate.query.filter(Certificate.sources.any(id=source.id)) + .filter(not_(Certificate.endpoints.any())) + .filter(Certificate.not_after < expiration_window) + .all() + ) + + +def get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance): + """ + Retrieves all certificates that are available for cleaning: not attached to endpoint, and X days since issuance. + + :param days_since_issuance: + :param source: + :return: + """ + not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + return ( + Certificate.query.filter(Certificate.sources.any(id=source.id)) + .filter(not_(Certificate.endpoints.any())) + .filter(Certificate.date_created < not_in_use_window) + .all() + ) + + def get_all_pending_reissue(): """ Retrieves all certificates that need to be rotated. From c96695c966229039e78e4a5f799bd525a0d122af Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:18:07 -0700 Subject: [PATCH 32/88] refactor --- lemur/sources/cli.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index c41a1cf7..c28600c2 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -54,6 +54,17 @@ def validate_sources(source_strings): return sources +def execute_clean(plugin, certificate, source): + try: + plugin.clean(certificate, source.options) + certificate.sources.remove(source) + certificate_service.database.update(certificate) + return SUCCESS_METRIC_STATUS + except Exception as e: + current_app.logger.exception(e) + sentry.captureException() + + @manager.option( "-s", "--sources", @@ -147,14 +158,7 @@ def clean(source_strings, commit): for certificate in certificate_service.get_all_pending_cleaning(source): status = FAILURE_METRIC_STATUS if commit: - try: - s.clean(certificate, source.options) - certificate.sources.remove(source) - certificate_service.database.update(certificate) - status = SUCCESS_METRIC_STATUS - except Exception as e: - current_app.logger.exception(e) - sentry.captureException() + status = execute_clean(s, certificate, source) metrics.send( "clean", From b28b4f9a28feba72c1d705d2085b30e57c75574d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:19:26 -0700 Subject: [PATCH 33/88] adding to new cli commands for cleaning certificates from source: a) either about to expire in X days and not attached to an endpoint a) or issued since X days but still not attached to an endpoint --- lemur/sources/cli.py | 155 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 19 deletions(-) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index c28600c2..a5b670a0 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -143,11 +143,9 @@ def clean(source_strings, commit): s = plugins.get(source.plugin_name) if not hasattr(s, "clean"): - print( - "Cannot clean source: {0}, source plugin does not implement 'clean()'".format( - source.label - ) - ) + info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'" + current_app.logger.warning(info_text) + print(info_text) continue start_time = time.time() @@ -155,28 +153,147 @@ def clean(source_strings, commit): print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - for certificate in certificate_service.get_all_pending_cleaning(source): + certificates = certificate_service.get_all_pending_cleaning(source) + for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: status = execute_clean(s, certificate, source) metrics.send( - "clean", + "certificate_clean", "counter", 1, - metric_tags={"source": source.label, "status": status}, + metric_tags={"status": status, "source": source.label, "certificate": certificate.name}, ) - - current_app.logger.warning( - "Removed {0} from source {1} during cleaning".format( - certificate.name, source.label - ) - ) - + current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning") cleaned += 1 - print( - "[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format( - label=source.label, time=(time.time() - start_time), cleaned=cleaned + info_text = f"[+] Finished cleaning source: {source.label}. " \ + f"Removed {cleaned} certificates from source. " \ + f"Run Time: {(time.time() - start_time)}\n" + print(info_text) + current_app.logger.warning(info_text) + + +@manager.option( + "-s", + "--sources", + dest="source_strings", + action="append", + help="Sources to operate on.", +) +@manager.option( + "-d", + "--days", + dest="days_to_expire", + type=int, + action="store", + required=True, + help="The expiry range within days.", +) +@manager.option( + "-c", + "--commit", + dest="commit", + action="store_true", + default=False, + help="Persist changes.", +) +def clean_unused_and_expiring_within_days(source_strings, days_to_expire, commit): + sources = validate_sources(source_strings) + for source in sources: + s = plugins.get(source.plugin_name) + + if not hasattr(s, "clean"): + info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'" + current_app.logger.warning(info_text) + print(info_text) + continue + + start_time = time.time() + + print("[+] Staring to clean source: {label}!\n".format(label=source.label)) + + cleaned = 0 + certificates = certificate_service.get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire) + for certificate in certificates: + status = FAILURE_METRIC_STATUS + if commit: + status = execute_clean(s, certificate, source) + + metrics.send( + "certificate_clean", + "counter", + 1, + metric_tags={"status": status, "source": source.label, "certificate": certificate.name}, ) - ) + current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning") + cleaned += 1 + + info_text = f"[+] Finished cleaning source: {source.label}. " \ + f"Removed {cleaned} certificates from source. " \ + f"Run Time: {(time.time() - start_time)}\n" + print(info_text) + current_app.logger.warning(info_text) + + +@manager.option( + "-s", + "--sources", + dest="source_strings", + action="append", + help="Sources to operate on.", +) +@manager.option( + "-d", + "--days", + dest="days_since_issuance", + type=int, + action="store", + required=True, + help="Days since issuance.", +) +@manager.option( + "-c", + "--commit", + dest="commit", + action="store_true", + default=False, + help="Persist changes.", +) +def clean_unused_and_issued_since_days(source_strings, days_since_issuance, commit): + sources = validate_sources(source_strings) + for source in sources: + s = plugins.get(source.plugin_name) + + if not hasattr(s, "clean"): + info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'" + current_app.logger.warning(info_text) + print(info_text) + continue + + start_time = time.time() + + print("[+] Staring to clean source: {label}!\n".format(label=source.label)) + + cleaned = 0 + certificates = certificate_service.get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance) + for certificate in certificates: + status = FAILURE_METRIC_STATUS + if commit: + status = execute_clean(s, certificate, source) + + metrics.send( + "certificate_clean", + "counter", + 1, + metric_tags={"status": status, "source": source.label, "certificate": certificate.name}, + ) + current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning") + cleaned += 1 + + info_text = f"[+] Finished cleaning source: {source.label}. " \ + f"Removed {cleaned} certificates from source. " \ + f"Run Time: {(time.time() - start_time)}\n" + print(info_text) + current_app.logger.warning(info_text) From 34d23503def14520e951d2b976d9db6ad6e4b5e8 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 14 Mar 2020 20:41:03 -0700 Subject: [PATCH 34/88] fixing the data bug --- lemur/certificates/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index ff558284..0889f64c 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -147,7 +147,7 @@ def get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance): return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) .filter(not_(Certificate.endpoints.any())) - .filter(Certificate.date_created < not_in_use_window) + .filter(Certificate.date_created > not_in_use_window) .all() ) From 1a19e250bb6561e52e3ab20bbb8dd9b82b36009b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 16 Mar 2020 11:24:17 -0700 Subject: [PATCH 35/88] updating and cleaning up tests --- lemur/plugins/lemur_acme/plugin.py | 7 +- lemur/plugins/lemur_acme/tests/test_acme.py | 105 +++++++++++--------- 2 files changed, 63 insertions(+), 49 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e566352c..3fc1df61 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -55,7 +55,7 @@ class AcmeHandler(object): self.all_dns_providers = [] def get_dns_challenges(self, host, authorizations): - """Get final domain to validate and dns challenges for it""" + """Get dns challenges for provided domain""" domain_to_validate, is_wildcard = self.strip_wildcard(host) dns_challenges = [] @@ -70,7 +70,7 @@ class AcmeHandler(object): if isinstance(combo.chall, challenges.DNS01): dns_challenges.append(combo) - return domain_to_validate, dns_challenges + return dns_challenges def strip_wildcard(self, host): """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" @@ -98,7 +98,8 @@ class AcmeHandler(object): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) change_ids = [] - host_to_validate, dns_challenges = self.get_dns_challenges(host, order.authorizations) + dns_challenges = self.get_dns_challenges(host, order.authorizations) + host_to_validate, _ = self.strip_wildcard(host) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 990a556e..eefd6c5b 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -23,11 +23,12 @@ class TestAcme(unittest.TestCase): } @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - def test_find_dns_challenge(self, mock_len): + def test_get_dns_challenges(self, mock_len): assert mock_len from acme import challenges + host = "example.com" c = challenges.DNS01() mock_authz = Mock() @@ -35,9 +36,18 @@ class TestAcme(unittest.TestCase): mock_entry = Mock() mock_entry.chall = c mock_authz.body.resolved_combinations.append(mock_entry) - result = yield self.acme.find_dns_challenge(mock_authz) + result = yield self.acme.get_dns_challenges(host, mock_authz) self.assertEqual(result, mock_entry) + def test_strip_wildcard(self): + expected = ("example.com", False) + result = self.acme.strip_wildcard("example.com") + self.assertEqual(expected, result) + + expected = ("example.com", True) + result = self.acme.strip_wildcard("*.example.com") + self.assertEqual(expected, result) + def test_authz_record(self): a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") self.assertEqual(type(a), plugin.AuthorizationRecord) @@ -45,9 +55,9 @@ class TestAcme(unittest.TestCase): @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") def test_start_dns_challenge( - self, mock_find_dns_challenge, mock_len, mock_app, mock_acme + self, mock_get_dns_challenges, mock_len, mock_app, mock_acme ): assert mock_len mock_order = Mock() @@ -65,9 +75,12 @@ class TestAcme(unittest.TestCase): mock_dns_provider.create_txt_record = Mock(return_value=1) values = [mock_entry] - iterable = mock_find_dns_challenge.return_value + iterable = mock_get_dns_challenges.return_value iterator = iter(values) iterable.__iter__.return_value = iterator + + # mock_get_dns_challenges = Mock(return_value="") + result = self.acme.start_dns_challenge( mock_acme, "accountid", "host", mock_dns_provider, mock_order, {} ) @@ -102,7 +115,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -127,15 +140,15 @@ class TestAcme(unittest.TestCase): @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") @patch("josepy.util.ComparableX509") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( - self, - mock_current_app, - mock_find_dns_challenge, - mock_jose, - mock_crypto, - mock_acme, + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, ): mock_cert_response = Mock() mock_cert_response.body = "123" @@ -256,11 +269,11 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.cloudflare.current_app") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") def test_get_dns_provider( - self, - mock_dns_provider_service, - mock_current_app_cloudflare, - mock_current_app_dyn, - mock_current_app, + self, + mock_dns_provider_service, + mock_current_app_cloudflare, + mock_current_app_dyn, + mock_current_app, ): provider = plugin.ACMEIssuerPlugin() route53 = provider.get_dns_provider("route53") @@ -278,14 +291,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -309,14 +322,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -349,14 +362,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate( - self, - mock_authorization_service, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_current_app, - mock_dns_provider_service, - mock_acme, + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_acme, ): provider = plugin.ACMEIssuerPlugin() mock_authority = Mock() @@ -423,10 +436,10 @@ class TestAcme(unittest.TestCase): ultradns._post = Mock() ultradns._get = Mock() ultradns._get.return_value = {'zoneName': 'test.example.com.com', - 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', - 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], - 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, - 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} + 'rrSets': [{'ownerName': '_acme-challenge.test.example.com.', + 'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}], + 'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100}, + 'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}} ultradns._delete = Mock() mock_metrics.send = Mock() ultradns.delete_txt_record(change_id, account_number, domain, token) From 07dc31bed7c92650f7ab5e5e3247bf36bebb2c52 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 16 Mar 2020 11:41:05 -0700 Subject: [PATCH 36/88] cleaning up whitespace changes --- lemur/plugins/lemur_acme/tests/test_acme.py | 49 ++++++++++----------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index eefd6c5b..b2c32eec 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -78,9 +78,6 @@ class TestAcme(unittest.TestCase): iterable = mock_get_dns_challenges.return_value iterator = iter(values) iterable.__iter__.return_value = iterator - - # mock_get_dns_challenges = Mock(return_value="") - result = self.acme.start_dns_challenge( mock_acme, "accountid", "host", mock_dns_provider, mock_order, {} ) @@ -115,7 +112,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -143,12 +140,12 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( - self, - mock_current_app, - mock_get_dns_challenges, - mock_jose, - mock_crypto, - mock_acme, + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, ): mock_cert_response = Mock() mock_cert_response.body = "123" @@ -291,14 +288,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -322,14 +319,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") From ecca003ab4a2d8710caa55b620a92e1a39f62650 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 17 Mar 2020 16:55:36 -0700 Subject: [PATCH 37/88] improving the documentation and method naming --- lemur/certificates/service.py | 27 ++++++++++++++------------- lemur/sources/cli.py | 6 +++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 0889f64c..1fa4d64e 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -102,12 +102,13 @@ def get_all_certs(): return Certificate.query.all() -def get_all_pending_cleaning(source): +def get_all_pending_cleaning_expired(source): """ - Retrieves all certificates that are available for cleaning. + Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not + attached to any endpoints. - :param source: - :return: + :param source: the source to search for certificates + :return: the pending certificates """ return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) @@ -117,14 +118,14 @@ def get_all_pending_cleaning(source): ) -def get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire): +def get_all_pending_cleaning_expiring_in_days(source, days_to_expire): """ - Retrieves all certificates that are available for cleaning: not attached to endpoint, + Retrieves all certificates that are available for cleaning, not attached to endpoint, and within X days from expiration. - :param days_to_expire: - :param source: - :return: + :param days_to_expire: defines how many days till the certificate is expired + :param source: the source to search for certificates + :return: the pending certificates """ expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD") return ( @@ -135,13 +136,13 @@ def get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire): ) -def get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance): +def get_all_pending_cleaning_issued_since_days(source, days_since_issuance): """ Retrieves all certificates that are available for cleaning: not attached to endpoint, and X days since issuance. - :param days_since_issuance: - :param source: - :return: + :param days_since_issuance: defines how many days since the certificate is issued + :param source: the source to search for certificates + :return: the pending certificates """ not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") return ( diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index a5b670a0..0d537500 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -153,7 +153,7 @@ def clean(source_strings, commit): print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - certificates = certificate_service.get_all_pending_cleaning(source) + certificates = certificate_service.get_all_pending_cleaning_expired(source) for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: @@ -215,7 +215,7 @@ def clean_unused_and_expiring_within_days(source_strings, days_to_expire, commit print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - certificates = certificate_service.get_all_pending_cleaning_about_to_expire_certs(source, days_to_expire) + certificates = certificate_service.get_all_pending_cleaning_expiring_in_days(source, days_to_expire) for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: @@ -277,7 +277,7 @@ def clean_unused_and_issued_since_days(source_strings, days_since_issuance, comm print("[+] Staring to clean source: {label}!\n".format(label=source.label)) cleaned = 0 - certificates = certificate_service.get_all_pending_cleaning_not_in_use_certs(source, days_since_issuance) + certificates = certificate_service.get_all_pending_cleaning_issued_since_days(source, days_since_issuance) for certificate in certificates: status = FAILURE_METRIC_STATUS if commit: From 1d4da0e3d808b1b747daab2196364498c8f61f2a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 17 Mar 2020 16:59:09 -0700 Subject: [PATCH 38/88] another polish --- lemur/certificates/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1fa4d64e..a6bbba30 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -108,7 +108,7 @@ def get_all_pending_cleaning_expired(source): attached to any endpoints. :param source: the source to search for certificates - :return: the pending certificates + :return: list of pending certificates """ return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) @@ -125,7 +125,7 @@ def get_all_pending_cleaning_expiring_in_days(source, days_to_expire): :param days_to_expire: defines how many days till the certificate is expired :param source: the source to search for certificates - :return: the pending certificates + :return: list of pending certificates """ expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD") return ( @@ -142,7 +142,7 @@ def get_all_pending_cleaning_issued_since_days(source, days_since_issuance): :param days_since_issuance: defines how many days since the certificate is issued :param source: the source to search for certificates - :return: the pending certificates + :return: list of pending certificates """ not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") return ( From 697215f8bc47433c27dcd1b371aac5e19057088f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Sat, 21 Mar 2020 20:05:35 -0700 Subject: [PATCH 39/88] better handling of destination plugin errors, and also checking cert expiration before upload --- lemur/certificates/models.py | 3 +++ lemur/plugins/lemur_aws/plugin.py | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 0a76cd6b..2ca88b00 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -445,6 +445,9 @@ def update_destinations(target, value, initiator): """ destination_plugin = plugins.get(value.plugin_name) status = FAILURE_METRIC_STATUS + + if target.expired: + return try: if target.private_key or not destination_plugin.requires_key: destination_plugin.upload( diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 6669f641..7bb7a3a2 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -325,14 +325,17 @@ class AWSDestinationPlugin(DestinationPlugin): ] def upload(self, name, body, private_key, cert_chain, options, **kwargs): - iam.upload_cert( - name, - body, - private_key, - self.get_option("path", options), - cert_chain=cert_chain, - account_number=self.get_option("accountNumber", options), - ) + try: + iam.upload_cert( + name, + body, + private_key, + self.get_option("path", options), + cert_chain=cert_chain, + account_number=self.get_option("accountNumber", options), + ) + except ClientError: + sentry.captureException() def deploy(self, elb_name, account, region, certificate): pass From 8ba9ae1148bfdadc3121b8c4ffb69689648f5725 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Mar 2020 16:18:43 +0000 Subject: [PATCH 40/88] Bump bleach from 3.1.1 to 3.1.2 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.1 to 3.1.2. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.1...v3.1.2) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 224789f6..d369cef4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,28 +5,28 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.1 # via readme-renderer +bleach==3.1.2 # via readme-renderer certifi==2019.11.28 # via requests cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit chardet==3.0.4 # via requests cryptography==2.8 # via secretstorage docutils==0.15.2 # via readme-renderer -flake8==3.5.0 +flake8==3.5.0 # via -r requirements-dev.in identify==1.4.9 # via pre-commit idna==2.8 # via requests -invoke==1.3.0 +invoke==1.3.0 # via -r requirements-dev.in jeepney==0.4.2 # via secretstorage keyring==21.0.0 # via twine mccabe==0.6.1 # via flake8 -nodeenv==1.3.3 +nodeenv==1.3.3 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==1.21.0 +pre-commit==1.21.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 pygments==2.5.2 # via readme-renderer -pyyaml==5.2 +pyyaml==5.2 # via -r requirements-dev.in, aspy.yaml, pre-commit readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine @@ -34,7 +34,7 @@ secretstorage==3.1.2 # via keyring six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.41.1 # via twine -twine==3.1.1 +twine==3.1.1 # via -r requirements-dev.in urllib3==1.25.7 # via requests virtualenv==16.7.9 # via pre-commit webencodings==0.5.1 # via bleach From 5206997468a3deb66be0393342d1d20561067605 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 26 Mar 2020 19:01:07 -0700 Subject: [PATCH 41/88] expired is now called for new certs, where the not_after field might be in datetime format, and not comparable to utc --- lemur/certificates/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 2ca88b00..9d7a459c 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -8,6 +8,8 @@ from datetime import timedelta import arrow +import pytz +import datetime from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa from flask import current_app @@ -321,8 +323,13 @@ class Certificate(db.Model): @hybrid_property def expired(self): - if self.not_after <= arrow.utcnow(): - return True + if isinstance(self.not_after, datetime.datetime): + # can't compare offset-naive and offset-aware datetimes + if self.not_after.replace(tzinfo=pytz.UTC) <= arrow.utcnow(): + return True + else: + if self.not_after <= arrow.utcnow(): + return True @expired.expression def expired(cls): From 2a2499a929bd30a80d19f4cca2be35dfdfa67098 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 26 Mar 2020 20:43:52 -0700 Subject: [PATCH 42/88] simplifying code --- lemur/certificates/models.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 9d7a459c..58630ee6 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -8,8 +8,6 @@ from datetime import timedelta import arrow -import pytz -import datetime from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa from flask import current_app @@ -323,13 +321,9 @@ class Certificate(db.Model): @hybrid_property def expired(self): - if isinstance(self.not_after, datetime.datetime): - # can't compare offset-naive and offset-aware datetimes - if self.not_after.replace(tzinfo=pytz.UTC) <= arrow.utcnow(): - return True - else: - if self.not_after <= arrow.utcnow(): - return True + # can't compare offset-naive and offset-aware datetimes + if arrow.Arrow.fromdatetime(self.not_after) <= arrow.utcnow(): + return True @expired.expression def expired(cls): From 0149f8b0d3b9c86739000c8372d92d7ac2d2ace3 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 26 Mar 2020 22:15:10 -0700 Subject: [PATCH 43/88] add support for wildcard and naked domains to PowerDNS module --- lemur/plugins/lemur_acme/powerdns.py | 210 +++++++++++++----- .../plugins/lemur_acme/tests/test_powerdns.py | 63 +++++- 2 files changed, 207 insertions(+), 66 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index a26faaac..e3f7e575 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -49,16 +49,20 @@ class Record: return self._data["name"] @property - def disabled(self): - return self._data["disabled"] + def type(self): + return self._data["type"] + + @property + def ttl(self): + return self._data["ttl"] @property def content(self): return self._data["content"] @property - def ttl(self): - return self._data["ttl"] + def disabled(self): + return self._data["disabled"] def get_zones(account_number): @@ -92,42 +96,32 @@ def get_zones(account_number): def create_txt_record(domain, token, account_number): """ Create a TXT record for the given domain and token and return a change_id tuple """ _check_conf() - zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_id = zone_name + "." - domain_id = domain + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "REPLACE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } + function = sys._getframe().f_code.co_name log_data = { "function": function, "fqdn": domain, "token": token, } + + # Create new record + domain_id = domain + "." + records = [Record({'name': domain_id, 'content': f"\"{token}\"", 'disabled': False})] + + # Get current records + cur_records = _get_txt_records(domain) + for record in cur_records: + if record.content != token: + records.append(record) + try: - _patch(path, payload) - log_data["message"] = "TXT record successfully created" + _patch_txt_records(domain, account_number, records) + log_data["message"] = "TXT record(s) successfully created" current_app.logger.debug(log_data) except Exception as e: sentry.captureException() log_data["Exception"] = e - log_data["message"] = "Unable to create TXT record" + log_data["message"] = "Unable to create TXT record(s)" current_app.logger.debug(log_data) change_id = (domain, token) @@ -173,43 +167,78 @@ def wait_for_dns_change(change_id, account_number=None): def delete_txt_record(change_id, account_number, domain, token): """ Delete the TXT record for the given domain and token """ _check_conf() - zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_id = zone_name + "." - domain_id = domain + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "DELETE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } + function = sys._getframe().f_code.co_name log_data = { "function": function, "fqdn": domain, - "token": token + "token": token, } - try: - _patch(path, payload) - log_data["message"] = "TXT record successfully deleted" - current_app.logger.debug(log_data) - except Exception as e: - sentry.captureException() - log_data["Exception"] = e - log_data["message"] = "Unable to delete TXT record" + + # Determine if we can delete whole RRset or just one record + cur_records = _get_txt_records(domain) + found = False + new_records = [] + for record in cur_records: + if record.content == f"\"{token}\"": + found = True + else: + new_records.append(record) + + if not found: # Record not found in DNS + log_data["message"] = "Unable to delete TXT record: TXT record not found" current_app.logger.debug(log_data) + return + + elif new_records: # Removing Record from RRSet via Patch + try: + _patch_txt_records(domain, account_number, new_records) + log_data["message"] = "TXT record successfully deleted" + current_app.logger.debug(log_data) + except Exception as e: + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to delete TXT record: patching exception" + current_app.logger.debug(log_data) + + else: # Delete current records + zone_name = _get_zone_name(domain, account_number) + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + zone_id = zone_name + "." + domain_id = domain + "." + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + payload = { + "rrsets": [ + { + "name": domain_id, + "type": "TXT", + "ttl": 300, + "changetype": "DELETE", + "records": [ + { + "content": f"\"{token}\"", + "disabled": False + } + ], + "comments": [] + } + ] + } + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "token": token + } + try: + _patch(path, payload) + log_data["message"] = "TXT record successfully deleted" + current_app.logger.debug(log_data) + except Exception as e: + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to delete TXT record" + current_app.logger.debug(log_data) def _check_conf(): @@ -243,6 +272,33 @@ def _get_zone_name(domain, account_number): return zone_name +def _get_txt_records(domain): + """Retrieve TXT records for a given domain and return list of Record Objects""" + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + + path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record" + function = sys._getframe().f_code.co_name + log_data = { + "function": function + } + try: + records = _get(path) + log_data["message"] = "Retrieved TXT Records Successfully" + current_app.logger.debug(log_data) + + except Exception as e: + sentry.captureException() + log_data["message"] = "Failed to Retrieve TXT Records" + current_app.logger.debug(log_data) + raise + + txt_records = [] + for record in records: + cur_record = Record(record) + txt_records.append(cur_record) + return txt_records + + def _get(path, params=None): """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") @@ -257,6 +313,40 @@ def _get(path, params=None): return resp.json() +def _patch_txt_records(domain, account_number, records): + """Send Patch request to PowerDNS Server""" + + domain_id = domain + "." + + # Create records + txt_records = [] + for record in records: + txt_records.append( + {'content': record.content, 'disabled': record.disabled} + ) + + # Create RRSet + payload = { + "rrsets": [ + { + "name": domain_id, + "type": "TXT", + "ttl": 300, + "changetype": "REPLACE", + "records": txt_records, + "comments": [] + } + ] + } + + # Create Txt Records + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + zone_name = _get_zone_name(domain, account_number) + zone_id = zone_name + "." + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + _patch(path, payload) + + def _patch(path, payload): """ Execute a Patch request on the given URL (base_uri + path) with given payload """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index c8b0a11e..d24827bb 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -48,13 +48,14 @@ class TestPowerdns(unittest.TestCase): self.assertEqual(result, zone) @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_create_txt_record(self, mock_current_app): + def test_create_txt_record_write_only(self, mock_current_app): domain = "_acme_challenge.test.example.com" zone = "test.example.com" token = "ABCDEFGHIJ" account_number = "1234567890" change_id = (domain, token) powerdns._check_conf = Mock() + powerdns._get_txt_records = Mock(return_value=[]) powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() mock_current_app.config.get = Mock(return_value="localhost") @@ -63,24 +64,74 @@ class TestPowerdns(unittest.TestCase): "function": "create_txt_record", "fqdn": domain, "token": token, - "message": "TXT record successfully created" + "message": "TXT record(s) successfully created" } result = powerdns.create_txt_record(domain, token, account_number) mock_current_app.logger.debug.assert_called_with(log_data) self.assertEqual(result, change_id) + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + def test_create_txt_record_append(self, mock_current_app): + domain = "_acme_challenge.test.example.com" + zone = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + powerdns._check_conf = Mock() + cur_token = "123456" + cur_records = [powerdns.Record({'name': domain, 'content': cur_token, 'disabled': False})] + powerdns._get_txt_records = Mock(return_value=cur_records) + powerdns._get_zone_name = Mock(return_value=zone) + mock_current_app.logger.debug = Mock() + mock_current_app.config.get = Mock(return_value="localhost") + powerdns._patch = Mock() + log_data = { + "function": "create_txt_record", + "fqdn": domain, + "token": token, + "message": "TXT record(s) successfully created" + } + expected_path = f"/api/v1/servers/localhost/zones/test.example.com." + expected_payload = { + "rrsets": [ + { + "name": domain + ".", + "type": "TXT", + "ttl": 300, + "changetype": "REPLACE", + "records": [ + { + "content": f"\"{token}\"", + "disabled": False + }, + { + "content": f"\"{cur_token}\"", + "disabled": False + } + ], + "comments": [] + } + ] + } + + result = powerdns.create_txt_record(domain, token, account_number) + mock_current_app.logger.debug.assert_called_with(log_data) + powerdns._patch.assert_called_with(expected_path, expected_payload) + self.assertEqual(result, change_id) + @patch("lemur.plugins.lemur_acme.powerdns.dnsutil") @patch("lemur.plugins.lemur_acme.powerdns.current_app") @patch("lemur.extensions.metrics") @patch("time.sleep") def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil): domain = "_acme-challenge.test.example.com" - token = "ABCDEFG" + token1 = "ABCDEFG" + token2 = "HIJKLMN" zone_name = "test.example.com" nameserver = "1.1.1.1" - change_id = (domain, token) + change_id = (domain, token1) powerdns._check_conf = Mock() - mock_records = (token,) + mock_records = (token2, token1) mock_current_app.config.get = Mock(return_value=1) powerdns._get_zone_name = Mock(return_value=zone_name) mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver) @@ -114,7 +165,7 @@ class TestPowerdns(unittest.TestCase): "function": "delete_txt_record", "fqdn": domain, "token": token, - "message": "TXT record successfully deleted" + "message": "Unable to delete TXT record: TXT record not found" } powerdns.delete_txt_record(change_id, account_number, domain, token) mock_current_app.logger.debug.assert_called_with(log_data) From 0e314d00281fbdcd31769cda2335d27df22d0d0b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 27 Mar 2020 10:18:38 -0700 Subject: [PATCH 44/88] adding documentation and final cleanup --- lemur/plugins/lemur_acme/powerdns.py | 97 ++++++++++++++++--- .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- 2 files changed, 82 insertions(+), 17 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index e3f7e575..c988fac9 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -1,11 +1,10 @@ -import time -import requests import json import sys +import time import lemur.common.utils as utils import lemur.dns_providers.util as dnsutil - +import requests from flask import current_app from lemur.extensions import metrics, sentry @@ -17,7 +16,9 @@ REQUIRED_VARIABLES = [ class Zone: - """ This class implements a PowerDNS zone in JSON. """ + """ + This class implements a PowerDNS zone in JSON. + """ def __init__(self, _data): self._data = _data @@ -39,7 +40,9 @@ class Zone: class Record: - """ This class implements a PowerDNS record. """ + """ + This class implements a PowerDNS record. + """ def __init__(self, _data): self._data = _data @@ -66,7 +69,12 @@ class Record: def get_zones(account_number): - """Retrieve authoritative zones from the PowerDNS API and return a list""" + """ + Retrieve authoritative zones from the PowerDNS API and return a list + :param account_number: + :raise: Exception + :return: list of Zone Objects + """ _check_conf() server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") path = f"/api/v1/servers/{server_id}/zones" @@ -94,7 +102,14 @@ def get_zones(account_number): def create_txt_record(domain, token, account_number): - """ Create a TXT record for the given domain and token and return a change_id tuple """ + """ + Create a TXT record for the given domain and token and return a change_id tuple + + :param domain: FQDN + :param token: challenge value + :param account_number: + :return: tuple of domain/token + """ _check_conf() function = sys._getframe().f_code.co_name @@ -130,8 +145,11 @@ def create_txt_record(domain, token, account_number): def wait_for_dns_change(change_id, account_number=None): """ - Checks the authoritative DNS Server to see if changes have propagated to DNS - Retries and waits until successful. + Checks the authoritative DNS Server to see if changes have propagated. + + :param change_id: tuple of domain/token + :param account_number: + :return: """ _check_conf() domain, token = change_id @@ -165,7 +183,15 @@ def wait_for_dns_change(change_id, account_number=None): def delete_txt_record(change_id, account_number, domain, token): - """ Delete the TXT record for the given domain and token """ + """ + Delete the TXT record for the given domain and token + + :param change_id: tuple of domain/token + :param account_number: + :param domain: FQDN + :param token: challenge to delete + :return: + """ _check_conf() function = sys._getframe().f_code.co_name @@ -242,11 +268,20 @@ def delete_txt_record(change_id, account_number, domain, token): def _check_conf(): + """ + Verifies required configuration variables are set + + :return: + """ utils.validate_conf(current_app, REQUIRED_VARIABLES) def _generate_header(): - """Generate a PowerDNS API header and return it as a dictionary""" + """ + Generate a PowerDNS API header and return it as a dictionary + + :return: Dict of header parameters + """ api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME") api_key = current_app.config.get("ACME_POWERDNS_APIKEY") headers = {api_key_name: api_key} @@ -254,7 +289,13 @@ def _generate_header(): def _get_zone_name(domain, account_number): - """Get most specific matching zone for the given domain and return as a String""" + """ + Get most specific matching zone for the given domain and return as a String + + :param domain: FQDN + :param account_number: + :return: FQDN of domain + """ zones = get_zones(account_number) zone_name = "" for z in zones: @@ -273,7 +314,13 @@ def _get_zone_name(domain, account_number): def _get_txt_records(domain): - """Retrieve TXT records for a given domain and return list of Record Objects""" + """ + Retrieve TXT records for a given domain and return list of Record Objects + + :param domain: FQDN + :raise: Exception + :return: list of Record objects + """ server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record" @@ -300,7 +347,13 @@ def _get_txt_records(domain): def _get(path, params=None): - """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ + """ + Execute a GET request on the given URL (base_uri + path) and return response as JSON object + + :param path: Relative URL path + :param params: additional parameters + :return: json response + """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.get( @@ -314,8 +367,14 @@ def _get(path, params=None): def _patch_txt_records(domain, account_number, records): - """Send Patch request to PowerDNS Server""" + """ + Send Patch request to PowerDNS Server + :param domain: FQDN + :param account_number: + :param records: List of Record objects + :return: + """ domain_id = domain + "." # Create records @@ -348,7 +407,13 @@ def _patch_txt_records(domain, account_number, records): def _patch(path, payload): - """ Execute a Patch request on the given URL (base_uri + path) with given payload """ + """ + Execute a Patch request on the given URL (base_uri + path) with given payload + + :param path: + :param payload: + :return: + """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.patch( diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index d24827bb..707ce9e4 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -79,7 +79,7 @@ class TestPowerdns(unittest.TestCase): change_id = (domain, token) powerdns._check_conf = Mock() cur_token = "123456" - cur_records = [powerdns.Record({'name': domain, 'content': cur_token, 'disabled': False})] + cur_records = [powerdns.Record({'name': domain, 'content': f"\"{cur_token}\"", 'disabled': False})] powerdns._get_txt_records = Mock(return_value=cur_records) powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() From d6cc8a8a9a86f6c1353d9ab15c9c0391856a139f Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 30 Mar 2020 09:01:28 -0700 Subject: [PATCH 45/88] fixing whitespace --- lemur/plugins/lemur_acme/powerdns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index c988fac9..1542ad7b 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -71,6 +71,7 @@ class Record: def get_zones(account_number): """ Retrieve authoritative zones from the PowerDNS API and return a list + :param account_number: :raise: Exception :return: list of Zone Objects From 6f3ba23fa0bb3f046fd76ac04cc2629dcf48eac7 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 30 Mar 2020 13:34:24 -0700 Subject: [PATCH 46/88] updating sinlge line of comments --- lemur/plugins/lemur_acme/powerdns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 1542ad7b..c1e5e19b 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -70,8 +70,8 @@ class Record: def get_zones(account_number): """ - Retrieve authoritative zones from the PowerDNS API and return a list - + Retrieve authoritative zones from the PowerDNS API and return a list of zones + :param account_number: :raise: Exception :return: list of Zone Objects From 67d24caef586920ee0d91f790596dead5f119bb5 Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:31:12 -0700 Subject: [PATCH 47/88] Remove equivalent destinations when cleaning certificates Remove equivalent destinations when cleaning certificates. This will prevent Lemur from attempting to re-upload a certificate after it has been cleaned. --- lemur/sources/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index 0d537500..c415b567 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -58,6 +58,13 @@ def execute_clean(plugin, certificate, source): try: plugin.clean(certificate, source.options) certificate.sources.remove(source) + + # If we want to remove the source from the certificate, we also need to clear any equivalent destinations to + # prevent Lemur from re-uploading the certificate. + for destination in certificate.destinations: + if destination.label == source.label: + certificate.destinations.remove(destination) + certificate_service.database.update(certificate) return SUCCESS_METRIC_STATUS except Exception as e: From e25f97fce7da04b316ba2f6fc7359630f236f99f Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:50:24 -0700 Subject: [PATCH 48/88] Bump time limit for clean_source Celery job For larger accounts, I've hit SoftTimeLimit exceptions before completion of this celery job. Bumping up the time limit on this job. --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 4af33d86..ebf85ed7 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -332,7 +332,7 @@ def clean_all_sources(): metrics.send(f"{function}.success", 'counter', 1) -@celery.task(soft_time_limit=600) +@celery.task(soft_time_limit=3600) def clean_source(source): """ This celery task will clean the specified source. This is a destructive operation that will delete unused From d825616ea6cef6fd02ee80eedd7a75f5fc4cea61 Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:53:17 -0700 Subject: [PATCH 49/88] No need to retry 25 times on DeleteConflict errors --- lemur/plugins/lemur_aws/iam.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 13590ddd..8d80e020 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -24,6 +24,12 @@ def retry_throttled(exception): if exception.response["Error"]["Code"] == "NoSuchEntity": return False + # No need to retry deletion requests if there is a DeleteConflict error. + # This error indicates that the certificate is still attached to an entity + # and cannot be deleted. + if exception.response["Error"]["Code"] == "DeleteConflict": + return False + metrics.send("iam_retry", "counter", 1, metric_tags={"exception": str(exception)}) return True From 2e1b58c70a233ad656fa88f69004b786d00dd4b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2020 18:27:38 +0000 Subject: [PATCH 50/88] Bump bleach from 3.1.2 to 3.1.4 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.2...v3.1.4) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d369cef4..b5521d38 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.2 # via readme-renderer +bleach==3.1.4 # via readme-renderer certifi==2019.11.28 # via requests cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit From 5add64714883d5c07b688a50f5fc5bbc9722839f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 3 Apr 2020 16:51:24 -0700 Subject: [PATCH 51/88] # emitting the count of certificates on the source --- lemur/sources/service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f4783313..408d411a 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -193,6 +193,11 @@ def sync_certificates(source, user): s = plugins.get(source.plugin_name) certificates = s.get_certificates(source.options) + # emitting the count of certificates on the source + metrics.send("sync_certificates_count", + "gauge", len(certificates), + metric_tags={"source": source.label}) + for certificate in certificates: exists, updated_by_hash = find_cert(certificate) From 5c2a2f8ff24939b297788abff8322001e9b82513 Mon Sep 17 00:00:00 2001 From: David Stipp Date: Sat, 4 Apr 2020 11:24:04 -0400 Subject: [PATCH 52/88] OAUTH2 fixes * Use OAUTH2 variable instead of PING while using OAUTH * Some IDPs require a POST instead of a GET to user data --- lemur/auth/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lemur/auth/views.py b/lemur/auth/views.py index e7f87356..eaed419d 100644 --- a/lemur/auth/views.py +++ b/lemur/auth/views.py @@ -127,6 +127,10 @@ def retrieve_user(user_api_url, access_token): # retrieve information about the current user. r = requests.get(user_api_url, params=user_params, headers=headers) + # Some IDPs, like "Keycloak", require a POST instead of a GET + if r.status_code == 400: + r = requests.post(user_api_url, data=user_params, headers=headers) + profile = r.json() user = user_service.get_by_email(profile["email"]) @@ -434,7 +438,7 @@ class OAuth2(Resource): verify_cert=verify_cert, ) - jwks_url = current_app.config.get("PING_JWKS_URL") + jwks_url = current_app.config.get("OAUTH2_JWKS_URL") error_code = validate_id_token(id_token, args["clientId"], jwks_url) if error_code: return error_code From f82ec24dfaf6a1e8fb56fac3a394f2990085b711 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Sun, 5 Apr 2020 21:46:33 -0700 Subject: [PATCH 53/88] updating _get_txt_records return values and docstrings --- lemur/plugins/lemur_acme/powerdns.py | 24 +++++++++++++------ .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index c1e5e19b..a5d02353 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -202,7 +202,11 @@ def delete_txt_record(change_id, account_number, domain, token): "token": token, } - # Determine if we can delete whole RRset or just one record + """ + Get existing TXT records matching the domain from DNS + The token to be deleted should already exist + There may be other records with different tokens as well + """ cur_records = _get_txt_records(domain) found = False new_records = [] @@ -212,12 +216,16 @@ def delete_txt_record(change_id, account_number, domain, token): else: new_records.append(record) - if not found: # Record not found in DNS - log_data["message"] = "Unable to delete TXT record: TXT record not found" + # Since the matching token is not in DNS, there is nothing to delete + if not found: + log_data["message"] = "Unable to delete TXT record: Token not found in existing TXT records" current_app.logger.debug(log_data) return - elif new_records: # Removing Record from RRSet via Patch + # The record to delete has been found AND there are other tokens set on the same domain + # Since we only want to delete one token value from the RRSet, we need to use the Patch command to + # overwrite the current RRSet with the existing records. + elif new_records: try: _patch_txt_records(domain, account_number, new_records) log_data["message"] = "TXT record successfully deleted" @@ -228,7 +236,9 @@ def delete_txt_record(change_id, account_number, domain, token): log_data["message"] = "Unable to delete TXT record: patching exception" current_app.logger.debug(log_data) - else: # Delete current records + # The record to delete has been found AND there are no other token values set on the same domain + # Use the Delete command to delete the whole RRSet. + else: zone_name = _get_zone_name(domain, account_number) server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") zone_id = zone_name + "." @@ -319,7 +329,6 @@ def _get_txt_records(domain): Retrieve TXT records for a given domain and return list of Record Objects :param domain: FQDN - :raise: Exception :return: list of Record objects """ server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") @@ -336,9 +345,10 @@ def _get_txt_records(domain): except Exception as e: sentry.captureException() + log_data["Exception"] = e log_data["message"] = "Failed to Retrieve TXT Records" current_app.logger.debug(log_data) - raise + return [] txt_records = [] for record in records: diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 707ce9e4..167381f2 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -165,7 +165,7 @@ class TestPowerdns(unittest.TestCase): "function": "delete_txt_record", "fqdn": domain, "token": token, - "message": "Unable to delete TXT record: TXT record not found" + "message": "Unable to delete TXT record: Token not found in existing TXT records" } powerdns.delete_txt_record(change_id, account_number, domain, token) mock_current_app.logger.debug.assert_called_with(log_data) From eb138fc96011167138159590178f9584d6972014 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 08:38:40 -0700 Subject: [PATCH 54/88] Add default celery metrics and logging using celery signals --- lemur/common/celery.py | 223 +++++++++++++++++++++++++++++++++-------- 1 file changed, 180 insertions(+), 43 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index ebf85ed7..b0193515 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -10,27 +10,27 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B import copy import sys import time -from datetime import datetime, timezone, timedelta - from celery import Celery +from celery.app.task import Context from celery.exceptions import SoftTimeLimitExceeded +from celery.signals import task_failure, task_received, task_revoked, task_success +from datetime import datetime, timezone, timedelta from flask import current_app from lemur.authorities.service import get as get_authority +from lemur.certificates import cli as cli_certificate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service +from lemur.dns_providers import cli as cli_dns_providers +from lemur.endpoints import cli as cli_endpoints from lemur.extensions import metrics, sentry from lemur.factory import create_app +from lemur.notifications import cli as cli_notification from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources from lemur.sources.service import add_aws_destination_to_sources -from lemur.certificates import cli as cli_certificate -from lemur.dns_providers import cli as cli_dns_providers -from lemur.notifications import cli as cli_notification -from lemur.endpoints import cli as cli_endpoints - if current_app: flask_app = current_app @@ -67,7 +67,7 @@ def is_task_active(fun, task_id, args): from celery.task.control import inspect if not args: - args = '()' # empty args + args = "()" # empty args i = inspect() active_tasks = i.active() @@ -80,6 +80,37 @@ def is_task_active(fun, task_id, args): return False +def get_celery_request_tags(**kwargs): + request = kwargs.get("request") + sender_hostname = "unknown" + sender = kwargs.get("sender") + if sender: + try: + sender_hostname = sender.hostname + except AttributeError: + sender_hostname = vars(sender.request).get("origin", "unknown") + if request and not isinstance( + request, Context + ): # unlike others, task_revoked sends a Context for `request` + task_name = request.name + task_id = request.id + receiver_hostname = request.hostname + else: + task_name = sender.name + task_id = sender.request.id + receiver_hostname = sender.request.hostname + + tags = { + "task_name": task_name, + "task_id": task_id, + "sender_hostname": sender_hostname, + "receiver_hostname": receiver_hostname, + } + if kwargs.get("exception"): + tags["error"] = repr(kwargs["exception"]) + return tags + + @celery.task() def report_celery_last_success_metrics(): """ @@ -108,15 +139,115 @@ def report_celery_last_success_metrics(): return current_time = int(time.time()) - schedule = current_app.config.get('CELERYBEAT_SCHEDULE') + schedule = current_app.config.get("CELERYBEAT_SCHEDULE") for _, t in schedule.items(): task = t.get("task") last_success = int(red.get(f"{task}.last_success") or 0) - metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success) + metrics.send( + f"{task}.time_since_last_success", "gauge", current_time - last_success + ) red.set( f"{function}.last_success", int(time.time()) ) # Alert if this metric is not seen - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + + +@task_received.connect +def report_number_pending_tasks(**kwargs): + """ + Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used + for autoscaling workers. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + metrics.send( + "celery.new_pending_task", + "TIMER", + 1, + metric_tags=get_celery_request_tags(**kwargs), + ) + + +@task_success.connect +def report_successful_task(**kwargs): + """ + Report a generic success metric as tasks to our metrics broker every time a task finished correctly. + This metric can be used for autoscaling workers. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + tags = get_celery_request_tags(**kwargs) + red.set(f"{tags['task_name']}.last_success", int(time.time())) + metrics.send("celery.successful_task", "TIMER", 1, metric_tags=tags) + + +@task_failure.connect +def report_failed_task(**kwargs): + """ + Report a generic failure metric as tasks to our metrics broker every time a task fails. + This metric can be used for alerting. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "Message": "Celery Task Failure", + } + + # Add traceback if exception info is in the kwargs + einfo = kwargs.get("einfo") + if einfo: + log_data["traceback"] = einfo.traceback + + error_tags = get_celery_request_tags(**kwargs) + + log_data.update(error_tags) + current_app.logger.error(log_data) + metrics.send("celery.failed_task", "TIMER", 1, metric_tags=error_tags) + + +@task_revoked.connect +def report_revoked_task(**kwargs): + """ + Report a generic failure metric as tasks to our metrics broker every time a task is revoked. + This metric can be used for alerting. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "Message": "Celery Task Revoked", + } + + error_tags = get_celery_request_tags(**kwargs) + + log_data.update(error_tags) + current_app.logger.error(log_data) + metrics.send("celery.revoked_task", "TIMER", 1, metric_tags=error_tags) @celery.task(soft_time_limit=600) @@ -217,15 +348,15 @@ def fetch_acme_cert(id): log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) - metrics.send(f"{function}.resolved", 'gauge', new) - metrics.send(f"{function}.failed", 'gauge', failed) - metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer) + metrics.send(f"{function}.resolved", "gauge", new) + metrics.send(f"{function}.failed", "gauge", failed) + metrics.send(f"{function}.wrong_issuer", "gauge", wrong_issuer) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( new=new, failed=failed, wrong_issuer=wrong_issuer ) ) - red.set(f'{function}.last_success', int(time.time())) + return log_data @celery.task() @@ -262,8 +393,8 @@ def fetch_all_pending_acme_certs(): current_app.logger.debug(log_data) fetch_acme_cert.delay(cert.id) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task() @@ -296,8 +427,8 @@ def remove_old_acme_certs(): current_app.logger.debug(log_data) pending_certificate_service.delete(cert) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task() @@ -328,8 +459,8 @@ def clean_all_sources(): current_app.logger.debug(log_data) clean_source.delay(source.label) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -366,6 +497,7 @@ def clean_source(source): current_app.logger.error(log_data) sentry.captureException() metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) + return log_data @celery.task() @@ -395,8 +527,8 @@ def sync_all_sources(): current_app.logger.debug(log_data) sync_source.delay(source.label) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=7200) @@ -428,19 +560,23 @@ def sync_source(source): current_app.logger.debug(log_data) try: sync([source]) - metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) + metrics.send( + f"{function}.success", "counter", 1, metric_tags={"source": source} + ) except SoftTimeLimitExceeded: log_data["message"] = "Error syncing source: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("sync_source_timeout", "counter", 1, metric_tags={"source": source}) + metrics.send( + "sync_source_timeout", "counter", 1, metric_tags={"source": source} + ) metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) - metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) - red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", "counter", 1, metric_tags={"source": source}) + return log_data @celery.task() @@ -477,8 +613,8 @@ def sync_source_destination(): log_data["message"] = "completed Syncing AWS destinations and sources" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -515,8 +651,8 @@ def certificate_reissue(): log_data["message"] = "reissuance completed" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -534,7 +670,6 @@ def certificate_rotate(): "function": function, "message": "rotating certificates", "task_id": task_id, - } if task_id and is_task_active(function, task_id, None): @@ -554,8 +689,8 @@ def certificate_rotate(): log_data["message"] = "rotation completed" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -590,8 +725,8 @@ def endpoints_expire(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=600) @@ -626,8 +761,8 @@ def get_all_zones(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -662,8 +797,8 @@ def check_revoked(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -690,7 +825,9 @@ def notify_expirations(): current_app.logger.debug(log_data) try: - cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) + cli_notification.expirations( + current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []) + ) except SoftTimeLimitExceeded: log_data["message"] = "Notify expiring Time limit exceeded." current_app.logger.error(log_data) @@ -698,5 +835,5 @@ def notify_expirations(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data From 11b15e7e234e0d5b7b494e35697d2ced5fbb5972 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 08:41:48 -0700 Subject: [PATCH 55/88] Clean up docstrings --- lemur/common/celery.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b0193515..7c183dc9 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -120,7 +120,6 @@ def report_celery_last_success_metrics(): report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful. Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted from this function. - """ function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None @@ -158,12 +157,6 @@ def report_number_pending_tasks(**kwargs): Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used for autoscaling workers. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): metrics.send( @@ -180,12 +173,6 @@ def report_successful_task(**kwargs): Report a generic success metric as tasks to our metrics broker every time a task finished correctly. This metric can be used for autoscaling workers. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): tags = get_celery_request_tags(**kwargs) @@ -199,12 +186,6 @@ def report_failed_task(**kwargs): Report a generic failure metric as tasks to our metrics broker every time a task fails. This metric can be used for alerting. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): log_data = { @@ -230,12 +211,6 @@ def report_revoked_task(**kwargs): Report a generic failure metric as tasks to our metrics broker every time a task is revoked. This metric can be used for alerting. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): log_data = { From 1360d846fd16d0b375de8ce2b810a94aee312c49 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 11:50:42 -0700 Subject: [PATCH 56/88] Improve error logging for a couple of use cases --- lemur/common/defaults.py | 17 +++++++++++++---- lemur/plugins/lemur_aws/plugin.py | 22 ++++++++++++---------- lemur/sources/service.py | 16 ++++++++++------ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index d563dbd0..b9c88e49 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -2,6 +2,7 @@ import re import unicodedata from cryptography import x509 +from cryptography.hazmat.primitives.serialization import Encoding from flask import current_app from lemur.common.utils import is_selfsigned @@ -71,12 +72,20 @@ def common_name(cert): :return: Common name or None """ try: - return cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[ - 0 - ].value.strip() + subject_oid = cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME) + if len(subject_oid) > 0: + return subject_oid[0].value.strip() + return None except Exception as e: sentry.captureException() - current_app.logger.error("Unable to get common name! {0}".format(e)) + current_app.logger.error( + { + "message": "Unable to get common name", + "error": e, + "public_key": cert.public_bytes(Encoding.PEM).decode("utf-8") + }, + exc_info=True + ) def organization(cert): diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 7bb7a3a2..8692348a 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -216,22 +216,24 @@ class AWSSourcePlugin(SourcePlugin): for region in regions: elbs = elb.get_all_elbs(account_number=account_number, region=region) - current_app.logger.info( - "Describing classic load balancers in {0}-{1}".format( - account_number, region - ) - ) + current_app.logger.info({ + "message": "Describing classic load balancers", + "account_number": account_number, + "region": region, + "number_of_load_balancers": len(elbs) + }) for e in elbs: endpoints.extend(get_elb_endpoints(account_number, region, e)) # fetch advanced ELBs elbs_v2 = elb.get_all_elbs_v2(account_number=account_number, region=region) - current_app.logger.info( - "Describing advanced load balancers in {0}-{1}".format( - account_number, region - ) - ) + current_app.logger.info({ + "message": "Describing advanced load balancers", + "account_number": account_number, + "region": region, + "number_of_load_balancers": len(elbs_v2) + }) for e in elbs_v2: endpoints.extend(get_elb_endpoints_v2(account_number, region, e)) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f4783313..e0f0aacf 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -123,15 +123,19 @@ def sync_endpoints(source): "acct": s.get_option("accountNumber", source.options)}) if not endpoint["certificate"]: - current_app.logger.error( - "Certificate Not Found. Name: {0} Endpoint: {1}".format( - certificate_name, endpoint["name"] - ) - ) + current_app.logger.error({ + "message": "Certificate Not Found", + "certificate_name": certificate_name, + "endpoint_name": endpoint["name"], + "dns_name": endpoint.get("dnsname"), + "account": s.get_option("accountNumber", source.options), + }) + metrics.send("endpoint.certificate.not.found", "counter", 1, metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], - "acct": s.get_option("accountNumber", source.options)}) + "acct": s.get_option("accountNumber", source.options), + "dnsname": endpoint.get("dnsname")}) continue policy = endpoint.pop("policy") From cee81bd693dcc2df831f9652b727542f49de1d22 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 9 Apr 2020 18:17:05 -0700 Subject: [PATCH 57/88] updated requirements, fixed unittests, pytest, and distinguidedName ordering --- lemur/plugins/lemur_acme/tests/test_acme.py | 59 +++--- .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- .../lemur_digicert/tests/test_digicert.py | 2 +- lemur/tests/test_certificates.py | 13 +- requirements-dev.txt | 46 ++-- requirements-docs.txt | 197 +++++++++--------- requirements-tests.txt | 110 +++++----- requirements.txt | 137 ++++++------ 8 files changed, 284 insertions(+), 282 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index b2c32eec..bec7be2b 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,11 +1,10 @@ import unittest +from unittest.mock import patch, Mock from cryptography.x509 import DNSName -from requests.models import Response - -from mock import MagicMock, Mock, patch - from lemur.plugins.lemur_acme import plugin, ultradns +from mock import MagicMock +from requests.models import Response class TestAcme(unittest.TestCase): @@ -57,7 +56,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") def test_start_dns_challenge( - self, mock_get_dns_challenges, mock_len, mock_app, mock_acme + self, mock_get_dns_challenges, mock_len, mock_app, mock_acme ): assert mock_len mock_order = Mock() @@ -88,7 +87,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") @patch("time.sleep") def test_complete_dns_challenge_success( - self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -112,7 +111,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -140,12 +139,12 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( - self, - mock_current_app, - mock_get_dns_challenges, - mock_jose, - mock_crypto, - mock_acme, + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, ): mock_cert_response = Mock() mock_cert_response.body = "123" @@ -182,7 +181,7 @@ class TestAcme(unittest.TestCase): assert result_client assert result_registration - @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch('lemur.plugins.lemur_acme.plugin.current_app') def test_get_domains_single(self, mock_current_app): options = {"common_name": "test.netflix.net"} result = self.acme.get_domains(options) @@ -288,14 +287,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -319,14 +318,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 167381f2..714cc938 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -1,5 +1,5 @@ import unittest -from mock import Mock, patch +from unittest.mock import patch, Mock from lemur.plugins.lemur_acme import plugin, powerdns diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 1e9ebca4..8bfd1dcf 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -1,4 +1,5 @@ import json +from unittest.mock import patch, Mock import arrow import pytest @@ -6,7 +7,6 @@ from cryptography import x509 from freezegun import freeze_time from lemur.plugins.lemur_digicert import plugin from lemur.tests.vectors import CSR_STR -from mock import Mock, patch def config_mock(*args): diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index adafa605..41584cb3 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -9,7 +9,8 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from marshmallow import ValidationError from freezegun import freeze_time -from mock import patch +# from mock import patch +from unittest.mock import patch from lemur.certificates.service import create_csr from lemur.certificates.views import * # noqa @@ -906,12 +907,12 @@ def test_certificate_get_body(client): assert response_body["serial"] == "211983098819107449768450703123665283596" assert response_body["serialHex"] == "9F7A75B39DAE4C3F9524C68B06DA6A0C" assert response_body["distinguishedName"] == ( - "CN=LemurTrust Unittests Class 1 CA 2018," - "O=LemurTrust Enterprises Ltd," - "OU=Unittesting Operations Center," - "C=EE," + "L=Earth," "ST=N/A," - "L=Earth" + "C=EE," + "OU=Unittesting Operations Center," + "O=LemurTrust Enterprises Ltd," + "CN=LemurTrust Unittests Class 1 CA 2018" ) diff --git a/requirements-dev.txt b/requirements-dev.txt index b5521d38..cb2edc22 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,40 +4,44 @@ # # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # -aspy.yaml==1.3.0 # via pre-commit +appdirs==1.4.3 # via virtualenv bleach==3.1.4 # via readme-renderer -certifi==2019.11.28 # via requests +certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography -cfgv==2.0.1 # via pre-commit +cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests -cryptography==2.8 # via secretstorage -docutils==0.15.2 # via readme-renderer +cryptography==2.9 # via secretstorage +distlib==0.3.0 # via virtualenv +docutils==0.16 # via readme-renderer +filelock==3.0.12 # via virtualenv flake8==3.5.0 # via -r requirements-dev.in -identify==1.4.9 # via pre-commit -idna==2.8 # via requests -invoke==1.3.0 # via -r requirements-dev.in -jeepney==0.4.2 # via secretstorage -keyring==21.0.0 # via twine +identify==1.4.14 # via pre-commit +idna==2.9 # via requests +importlib-metadata==1.6.0 # via keyring, pre-commit, twine, virtualenv +invoke==1.4.1 # via -r requirements-dev.in +jeepney==0.4.3 # via keyring, secretstorage +keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 -nodeenv==1.3.3 # via -r requirements-dev.in, pre-commit +nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==1.21.0 # via -r requirements-dev.in +pre-commit==2.2.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 -pycparser==2.19 # via cffi +pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 -pygments==2.5.2 # via readme-renderer -pyyaml==5.2 # via -r requirements-dev.in, aspy.yaml, pre-commit -readme-renderer==24.0 # via twine +pygments==2.6.1 # via readme-renderer +pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit +readme-renderer==25.0 # via twine requests-toolbelt==0.9.1 # via twine -requests==2.22.0 # via requests-toolbelt, twine +requests==2.23.0 # via requests-toolbelt, twine secretstorage==3.1.2 # via keyring -six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer +six==1.14.0 # via bleach, cryptography, readme-renderer, virtualenv toml==0.10.0 # via pre-commit -tqdm==4.41.1 # via twine +tqdm==4.45.0 # via twine twine==3.1.1 # via -r requirements-dev.in -urllib3==1.25.7 # via requests -virtualenv==16.7.9 # via pre-commit +urllib3==1.25.8 # via requests +virtualenv==20.0.17 # via pre-commit webencodings==0.5.1 # via bleach +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-docs.txt b/requirements-docs.txt index 893965ca..d3eaa4d1 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,111 +4,108 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.0.0 +acme==1.3.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx -alembic-autogenerate-enums==0.0.2 -alembic==1.3.2 -amqp==2.5.2 -aniso8601==8.0.0 -arrow==0.15.5 -asyncpool==1.0 +alembic-autogenerate-enums==0.0.2 # via -r requirements.txt +alembic==1.4.2 # via -r requirements.txt, flask-migrate +amqp==2.5.2 # via -r requirements.txt, kombu +aniso8601==8.0.0 # via -r requirements.txt, flask-restful +arrow==0.15.5 # via -r requirements.txt +asyncpool==1.0 # via -r requirements.txt babel==2.8.0 # via sphinx -bcrypt==3.1.7 -billiard==3.6.1.0 -blinker==1.4 -boto3==1.10.46 -botocore==1.13.46 -celery[redis]==4.4.0 -certifi==2019.11.28 -certsrv==2.1.1 -cffi==1.13.2 -chardet==3.0.4 -click==7.0 -cloudflare==2.3.1 -cryptography==2.8 -dnspython3==1.15.0 -dnspython==1.15.0 -docutils==0.15.2 -dyn==1.8.1 -flask-bcrypt==0.7.1 -flask-cors==3.0.8 -flask-mail==0.9.1 -flask-migrate==2.5.2 -flask-principal==0.4.0 -flask-replicated==1.3 -flask-restful==0.3.7 -flask-script==2.0.6 -flask-sqlalchemy==2.4.1 -flask==1.1.1 -future==0.18.2 -gunicorn==20.0.4 -hvac==0.9.6 -idna==2.8 +bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko +billiard==3.6.3.0 # via -r requirements.txt, celery +blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven +boto3==1.12.39 # via -r requirements.txt +botocore==1.15.39 # via -r requirements.txt, boto3, s3transfer +celery[redis]==4.4.2 # via -r requirements.txt +certifi==2020.4.5.1 # via -r requirements.txt, requests +certsrv==2.1.1 # via -r requirements.txt +cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl +chardet==3.0.4 # via -r requirements.txt, requests +click==7.1.1 # via -r requirements.txt, flask +cloudflare==2.6.5 # via -r requirements.txt +cryptography==2.9 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +dnspython3==1.15.0 # via -r requirements.txt +dnspython==1.15.0 # via -r requirements.txt, dnspython3 +docutils==0.15.2 # via -r requirements.txt, botocore, sphinx +dyn==1.8.1 # via -r requirements.txt +flask-bcrypt==0.7.1 # via -r requirements.txt +flask-cors==3.0.8 # via -r requirements.txt +flask-mail==0.9.1 # via -r requirements.txt +flask-migrate==2.5.3 # via -r requirements.txt +flask-principal==0.4.0 # via -r requirements.txt +flask-replicated==1.3 # via -r requirements.txt +flask-restful==0.3.8 # via -r requirements.txt +flask-script==2.0.6 # via -r requirements.txt +flask-sqlalchemy==2.4.1 # via -r requirements.txt, flask-migrate +flask==1.1.2 # via -r requirements.txt, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven +future==0.18.2 # via -r requirements.txt, cloudflare +gunicorn==20.0.4 # via -r requirements.txt +hvac==0.10.1 # via -r requirements.txt +idna==2.9 # via -r requirements.txt, requests imagesize==1.2.0 # via sphinx -importlib-metadata==1.3.0 -inflection==0.3.1 -itsdangerous==1.1.0 -javaobj-py3==0.4.0.1 -jinja2==2.10.3 -jmespath==0.9.4 -josepy==1.2.0 -jsonlines==1.2.0 -kombu==4.6.7 -lockfile==0.12.2 -logmatic-python==0.1.7 -mako==1.1.0 -markupsafe==1.1.1 -marshmallow-sqlalchemy==0.21.0 -marshmallow==2.20.4 -mock==3.0.5 -more-itertools==8.0.2 -ndg-httpsclient==0.5.1 -packaging==19.2 # via sphinx -paramiko==2.7.1 -pem==19.3.0 -psycopg2==2.8.4 -pyasn1-modules==0.2.7 -pyasn1==0.4.8 -pycparser==2.19 -pycryptodomex==3.9.4 -pygments==2.5.2 # via sphinx -pyjks==19.0.0 -pyjwt==1.7.1 -pynacl==1.3.0 -pyopenssl==19.1.0 -pyparsing==2.4.6 # via packaging -pyrfc3339==1.1 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-json-logger==0.1.11 -pytz==2019.3 -pyyaml==5.2 -raven[flask]==6.10.0 -redis==3.3.11 -requests-toolbelt==0.9.1 -requests[security]==2.22.0 -retrying==1.3.3 -s3transfer==0.2.1 -six==1.13.0 +importlib-metadata==1.6.0 # via -r requirements.txt, kombu +inflection==0.4.0 # via -r requirements.txt +itsdangerous==1.1.0 # via -r requirements.txt, flask +javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks +jinja2==2.11.1 # via -r requirements.txt, flask, sphinx +jmespath==0.9.5 # via -r requirements.txt, boto3, botocore +josepy==1.3.0 # via -r requirements.txt, acme +jsonlines==1.2.0 # via -r requirements.txt, cloudflare +kombu==4.6.8 # via -r requirements.txt, celery +lockfile==0.12.2 # via -r requirements.txt +logmatic-python==0.1.7 # via -r requirements.txt +mako==1.1.2 # via -r requirements.txt, alembic +markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako +marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt +marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy +mock==4.0.2 # via -r requirements.txt, acme +ndg-httpsclient==0.5.1 # via -r requirements.txt +packaging==20.3 # via sphinx +paramiko==2.7.1 # via -r requirements.txt +pem==20.1.0 # via -r requirements.txt +psycopg2==2.8.5 # via -r requirements.txt +pycparser==2.20 # via -r requirements.txt, cffi +pycryptodomex==3.9.7 # via -r requirements.txt, pyjks +pygments==2.6.1 # via sphinx +pyjks==19.0.0 # via -r requirements.txt +pyjwt==1.7.1 # via -r requirements.txt +pynacl==1.3.0 # via -r requirements.txt, paramiko +pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests +pyparsing==2.4.7 # via packaging +pyrfc3339==1.1 # via -r requirements.txt, acme +python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore +python-editor==1.0.4 # via -r requirements.txt, alembic +python-json-logger==0.1.11 # via -r requirements.txt, logmatic-python +pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 +pyyaml==5.3.1 # via -r requirements.txt, cloudflare +raven[flask]==6.10.0 # via -r requirements.txt +redis==3.4.1 # via -r requirements.txt, celery +requests-toolbelt==0.9.1 # via -r requirements.txt, acme +requests[security]==2.23.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx +retrying==1.3.3 # via -r requirements.txt +s3transfer==0.3.3 # via -r requirements.txt, boto3 +six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx -sphinx-rtd-theme==0.4.3 -sphinx==2.3.1 -sphinxcontrib-applehelp==1.0.1 # via sphinx -sphinxcontrib-devhelp==1.0.1 # via sphinx -sphinxcontrib-htmlhelp==1.0.2 # via sphinx -sphinxcontrib-httpdomain==1.7.0 +sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in +sphinx==3.0.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx +sphinxcontrib-httpdomain==1.7.0 # via -r requirements-docs.in sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.2 # via sphinx -sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.12 -tabulate==0.8.6 -twofish==0.3.0 -urllib3==1.25.7 -vine==1.3.0 -werkzeug==0.16.0 -xmltodict==0.12.0 -zipp==0.6.0 +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy-utils==0.36.3 # via -r requirements.txt +sqlalchemy==1.3.16 # via -r requirements.txt, alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.7 # via -r requirements.txt +twofish==0.3.0 # via -r requirements.txt, pyjks +urllib3==1.25.8 # via -r requirements.txt, botocore, requests +vine==1.3.0 # via -r requirements.txt, amqp, celery +werkzeug==1.0.1 # via -r requirements.txt, flask +xmltodict==0.12.0 # via -r requirements.txt +zipp==3.1.0 # via -r requirements.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index 293bd350..874fbb5c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,85 +6,87 @@ # appdirs==1.4.3 # via black attrs==19.3.0 # via black, jsonschema, pytest -aws-sam-translator==1.19.1 # via cfn-lint +aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.4.3 # via moto -bandit==1.6.2 -black==19.10b0 -boto3==1.10.46 # via aws-sam-translator, moto +bandit==1.6.2 # via -r requirements-tests.in +black==19.10b0 # via -r requirements-tests.in +boto3==1.12.39 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.13.46 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2019.11.28 # via requests -cffi==1.13.2 # via cryptography -cfn-lint==0.26.2 # via moto +botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2020.4.5.1 # via requests +cffi==1.14.0 # via cryptography +cfn-lint==0.29.4 # via moto chardet==3.0.4 # via requests -click==7.0 # via black, flask -coverage==5.0.1 -cryptography==2.8 # via moto, sshpubkeys -docker==4.1.0 # via moto +click==7.1.1 # via black, flask +coverage==5.0.4 # via -r requirements-tests.in +cryptography==2.9 # via moto, sshpubkeys +decorator==4.4.2 # via networkx +docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys -factory-boy==2.12.0 -faker==3.0.0 -fakeredis==1.1.0 -flask==1.1.1 # via pytest-flask -freezegun==0.3.12 +factory-boy==2.12.0 # via -r requirements-tests.in +faker==4.0.2 # via -r requirements-tests.in, factory-boy +fakeredis==1.4.0 # via -r requirements-tests.in +flask==1.1.2 # via pytest-flask +freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk -gitdb2==2.0.6 # via gitpython -gitpython==3.0.5 # via bandit +gitdb==4.0.2 # via gitpython +gitpython==3.1.0 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.3.0 # via jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.10.3 # via flask, moto -jmespath==0.9.4 # via boto3, botocore +jinja2==2.11.1 # via flask, moto +jmespath==0.9.5 # via boto3, botocore jsondiff==1.1.2 # via moto -jsonpatch==1.24 # via cfn-lint -jsonpickle==1.2 # via aws-xray-sdk +jsonpatch==1.25 # via cfn-lint +jsonpickle==1.3 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==3.2.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 -mock==3.0.5 # via moto -more-itertools==8.0.2 # via pytest, zipp -moto==1.3.14 -nose==1.3.7 -packaging==19.2 # via pytest -pathspec==0.7.0 # via black -pbr==5.4.4 # via stevedore +mock==4.0.2 # via moto +more-itertools==8.2.0 # via pytest +moto==1.3.14 # via -r requirements-tests.in +networkx==2.4 # via cfn-lint +nose==1.3.7 # via -r requirements-tests.in +packaging==20.3 # via pytest +pathspec==0.8.0 # via black +pbr==5.4.5 # via stevedore pluggy==0.13.1 # via pytest py==1.8.1 # via pytest pyasn1==0.4.8 # via python-jose, rsa -pycparser==2.19 # via cffi -pyflakes==2.1.1 -pyparsing==2.4.6 # via packaging -pyrsistent==0.15.6 # via jsonschema -pytest-flask==0.15.0 -pytest-mock==1.13.0 -pytest==5.3.2 +pycparser==2.20 # via cffi +pyflakes==2.1.1 # via -r requirements-tests.in +pyparsing==2.4.7 # via packaging +pyrsistent==0.16.0 # via jsonschema +pytest-flask==1.0.0 # via -r requirements-tests.in +pytest-mock==3.0.0 # via -r requirements-tests.in +pytest==5.4.1 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto -pyyaml==5.2 -redis==3.3.11 # via fakeredis -regex==2019.12.20 # via black -requests-mock==1.7.0 -requests==2.22.0 # via docker, moto, requests-mock, responses -responses==0.10.9 # via moto +pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto +redis==3.4.1 # via fakeredis +regex==2020.4.4 # via black +requests-mock==1.7.0 # via -r requirements-tests.in +requests==2.23.0 # via docker, moto, requests-mock, responses +responses==0.10.12 # via moto rsa==4.0 # via python-jose -s3transfer==0.2.1 # via boto3 -six==1.13.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, faker, fakeredis, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client -smmap2==2.0.5 # via gitdb2 +s3transfer==0.3.3 # via boto3 +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +smmap==3.0.1 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto -stevedore==1.31.0 # via bandit +stevedore==1.32.0 # via bandit text-unidecode==1.3 # via faker toml==0.10.0 # via black -typed-ast==1.4.0 # via black -urllib3==1.25.7 # via botocore, requests -wcwidth==0.1.8 # via pytest +typed-ast==1.4.1 # via black +urllib3==1.25.8 # via botocore, requests +wcwidth==0.1.9 # via pytest websocket-client==0.57.0 # via docker -werkzeug==0.16.0 # via flask, moto, pytest-flask -wrapt==1.11.2 # via aws-xray-sdk +werkzeug==1.0.1 # via flask, moto, pytest-flask +wrapt==1.12.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto -zipp==0.6.0 # via importlib-metadata +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.txt b/requirements.txt index 639c9377..964576d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,96 +4,95 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.0.0 -alembic-autogenerate-enums==0.0.2 -alembic==1.3.2 # via flask-migrate +acme==1.3.0 # via -r requirements.in +alembic-autogenerate-enums==0.0.2 # via -r requirements.in +alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.15.5 -asyncpool==1.0 +arrow==0.15.5 # via -r requirements.in +asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko -billiard==3.6.1.0 # via celery +billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.10.46 -botocore==1.13.46 -celery[redis]==4.4.0 -certifi==2019.11.28 -certsrv==2.1.1 -cffi==1.13.2 # via bcrypt, cryptography, pynacl +boto3==1.12.39 # via -r requirements.in +botocore==1.15.39 # via -r requirements.in, boto3, s3transfer +celery[redis]==4.4.2 # via -r requirements.in +certifi==2020.4.5.1 # via -r requirements.in, requests +certsrv==2.1.1 # via -r requirements.in +cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.0 # via flask -cloudflare==2.3.1 -cryptography==2.8 -dnspython3==1.15.0 +click==7.1.1 # via flask +cloudflare==2.6.5 # via -r requirements.in +cryptography==2.9 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 docutils==0.15.2 # via botocore -dyn==1.8.1 -flask-bcrypt==0.7.1 -flask-cors==3.0.8 -flask-mail==0.9.1 -flask-migrate==2.5.2 -flask-principal==0.4.0 -flask-replicated==1.3 -flask-restful==0.3.7 -flask-script==2.0.6 -flask-sqlalchemy==2.4.1 -flask==1.1.1 -future==0.18.2 -gunicorn==20.0.4 -hvac==0.9.6 -idna==2.8 # via requests -importlib-metadata==1.3.0 # via kombu -inflection==0.3.1 +dyn==1.8.1 # via -r requirements.in +flask-bcrypt==0.7.1 # via -r requirements.in +flask-cors==3.0.8 # via -r requirements.in +flask-mail==0.9.1 # via -r requirements.in +flask-migrate==2.5.3 # via -r requirements.in +flask-principal==0.4.0 # via -r requirements.in +flask-replicated==1.3 # via -r requirements.in +flask-restful==0.3.8 # via -r requirements.in +flask-script==2.0.6 # via -r requirements.in +flask-sqlalchemy==2.4.1 # via -r requirements.in, flask-migrate +flask==1.1.2 # via -r requirements.in, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven +future==0.18.2 # via -r requirements.in, cloudflare +gunicorn==20.0.4 # via -r requirements.in +hvac==0.10.1 # via -r requirements.in +idna==2.9 # via requests +importlib-metadata==1.6.0 # via kombu +inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks -jinja2==2.10.3 -jmespath==0.9.4 # via boto3, botocore -josepy==1.2.0 # via acme +jinja2==2.11.1 # via -r requirements.in, flask +jmespath==0.9.5 # via boto3, botocore +josepy==1.3.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.6.7 # via celery -lockfile==0.12.2 -logmatic-python==0.1.7 -mako==1.1.0 # via alembic +kombu==4.6.8 # via celery +lockfile==0.12.2 # via -r requirements.in +logmatic-python==0.1.7 # via -r requirements.in +mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.21.0 -marshmallow==2.20.4 -mock==3.0.5 # via acme -more-itertools==8.0.2 # via zipp -ndg-httpsclient==0.5.1 -paramiko==2.7.1 -pem==19.3.0 -psycopg2==2.8.4 -pyasn1-modules==0.2.7 # via pyjks, python-ldap +marshmallow-sqlalchemy==0.22.3 # via -r requirements.in +marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy +mock==4.0.2 # via acme +ndg-httpsclient==0.5.1 # via -r requirements.in +paramiko==2.7.1 # via -r requirements.in +pem==20.1.0 # via -r requirements.in +psycopg2==2.8.5 # via -r requirements.in +pyasn1-modules==0.2.8 # via pyjks, python-ldap pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap -pycparser==2.19 # via cffi -pycryptodomex==3.9.4 # via pyjks -pyjks==19.0.0 -pyjwt==1.7.1 +pycparser==2.20 # via cffi +pycryptodomex==3.9.7 # via pyjks +pyjks==19.0.0 # via -r requirements.in +pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko -pyopenssl==19.1.0 +pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests pyrfc3339==1.1 # via acme python-dateutil==2.8.1 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python -python-ldap==3.2.0 +python-ldap==3.2.0 # via -r requirements.in pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.2 -raven[flask]==6.10.0 -redis==3.3.11 +pyyaml==5.3.1 # via -r requirements.in, cloudflare +raven[flask]==6.10.0 # via -r requirements.in +redis==3.4.1 # via -r requirements.in, celery requests-toolbelt==0.9.1 # via acme -requests[security]==2.22.0 -retrying==1.3.3 -s3transfer==0.2.1 # via boto3 -six==1.13.0 -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.6 +requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt +retrying==1.3.3 # via -r requirements.in +s3transfer==0.3.3 # via boto3 +six==1.14.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils +sqlalchemy-utils==0.36.3 # via -r requirements.in +sqlalchemy==1.3.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.7 # via -r requirements.in twofish==0.3.0 # via pyjks -urllib3==1.25.7 # via botocore, requests +urllib3==1.25.8 # via botocore, requests vine==1.3.0 # via amqp, celery -werkzeug==0.16.0 # via flask -xmltodict==0.12.0 -zipp==0.6.0 # via importlib-metadata +werkzeug==1.0.1 # via flask +xmltodict==0.12.0 # via -r requirements.in +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From f8657998e64cf3c4cb32406dbaa15296803962bb Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 15 Apr 2020 14:17:34 -0700 Subject: [PATCH 58/88] updated python requirements and still works --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 22 +++++++++++----------- requirements.txt | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d3eaa4d1..54bde14e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -49,7 +49,7 @@ importlib-metadata==1.6.0 # via -r requirements.txt, kombu inflection==0.4.0 # via -r requirements.txt itsdangerous==1.1.0 # via -r requirements.txt, flask javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks -jinja2==2.11.1 # via -r requirements.txt, flask, sphinx +jinja2==2.11.2 # via -r requirements.txt, flask, sphinx jmespath==0.9.5 # via -r requirements.txt, boto3, botocore josepy==1.3.0 # via -r requirements.txt, acme jsonlines==1.2.0 # via -r requirements.txt, cloudflare @@ -89,7 +89,7 @@ s3transfer==0.3.3 # via -r requirements.txt, boto3 six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in -sphinx==3.0.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.0.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 874fbb5c..5f9cafcc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,7 +7,7 @@ appdirs==1.4.3 # via black attrs==19.3.0 # via black, jsonschema, pytest aws-sam-translator==1.22.0 # via cfn-lint -aws-xray-sdk==2.4.3 # via moto +aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in boto3==1.12.39 # via aws-sam-translator, moto @@ -15,31 +15,31 @@ boto==2.49.0 # via moto botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography -cfn-lint==0.29.4 # via moto +cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.1 # via black, flask -coverage==5.0.4 # via -r requirements-tests.in +coverage==5.1 # via -r requirements-tests.in cryptography==2.9 # via moto, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in -faker==4.0.2 # via -r requirements-tests.in, factory-boy +faker==4.0.3 # via -r requirements-tests.in, factory-boy fakeredis==1.4.0 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk -gitdb==4.0.2 # via gitpython -gitpython==3.1.0 # via bandit +gitdb==4.0.4 # via gitpython +gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.11.1 # via flask, moto +jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore jsondiff==1.1.2 # via moto jsonpatch==1.25 # via cfn-lint -jsonpickle==1.3 # via aws-xray-sdk +jsonpickle==1.4 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==3.2.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 @@ -55,7 +55,7 @@ pluggy==0.13.1 # via pytest py==1.8.1 # via pytest pyasn1==0.4.8 # via python-jose, rsa pycparser==2.20 # via cffi -pyflakes==2.1.1 # via -r requirements-tests.in +pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in @@ -73,7 +73,7 @@ responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client -smmap==3.0.1 # via gitdb +smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto stevedore==1.32.0 # via bandit diff --git a/requirements.txt b/requirements.txt index 964576d2..bdad75b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ importlib-metadata==1.6.0 # via kombu inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks -jinja2==2.11.1 # via -r requirements.in, flask +jinja2==2.11.2 # via -r requirements.in, flask jmespath==0.9.5 # via boto3, botocore josepy==1.3.0 # via acme jsonlines==1.2.0 # via cloudflare From e33b767d12f9850d8fbfbf698e32ecdc3a50fbb7 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 15 Apr 2020 14:31:41 -0700 Subject: [PATCH 59/88] updating package.jso based on `npm update` --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9b899176..84e5289d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ }, "devDependencies": { "gulp": "^3.9.1", - "jshint": "^2.8.0", + "jshint": "^2.11.0", "karma-chrome-launcher": "^2.0.0" } } From 4fcb050fa81b4080ff92e3665679d870fc112f57 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 16 Apr 2020 01:08:39 -0700 Subject: [PATCH 60/88] fixing bootstrap, bootswatch, and updating bower --- bower.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bower.json b/bower.json index f7d5500d..44cce994 100644 --- a/bower.json +++ b/bower.json @@ -11,12 +11,12 @@ "angular": "1.4.9", "json3": "~3.3", "es5-shim": "~4.5.0", - "bootstrap": "~3.3.6", "angular-bootstrap": "~1.1.1", "angular-animate": "~1.4.9", "restangular": "~1.5.1", "ng-table": "~0.8.3", "moment": "~2.11.1", + "bootstrap": "~3.4.1", "angular-loading-bar": "~0.8.0", "angular-moment": "~0.10.3", "moment-range": "~2.1.0", @@ -24,7 +24,7 @@ "angularjs-toaster": "~1.0.0", "angular-chart.js": "~0.8.8", "ngletteravatar": "~4.0.0", - "bootswatch": "~3.3.6", + "bootswatch": "~3.4.1", "fontawesome": "~4.5.0", "satellizer": "~0.13.4", "angular-ui-router": "~0.2.15", diff --git a/package.json b/package.json index 771a4386..1a54eccc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "url": "git://github.com/netflix/lemur.git" }, "dependencies": { - "bower": "^1.8.2", + "bower": "^1.8.8", "browser-sync": "^2.26.7", "del": "^2.2.2", "gulp-autoprefixer": "^3.1.1", From 8d0007b9c06f3e40305eab1083dcf7bb3fdda14f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 24 Apr 2020 15:48:06 -0700 Subject: [PATCH 61/88] fixing the private DNS zone issue. Private hosted zones will never be visible to third-parties like LetsEncrypt, and Lemur should not consider them as authoritative zones. This fix, make sure they are not added to the dns_provider table. --- lemur/plugins/lemur_acme/route53.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index 55da5161..aaccb57e 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -35,9 +35,10 @@ def get_zones(client=None): zones = [] for page in paginator.paginate(): for zone in page["HostedZones"]: - zones.append( - zone["Name"][:-1] - ) # We need [:-1] to strip out the trailing dot. + if not zone["Config"]["PrivateZone"]: + zones.append( + zone["Name"][:-1] + ) # We need [:-1] to strip out the trailing dot. return zones From 273c3e2793647389e00e2509b0e0fd047aa540fe Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 11:52:43 -0700 Subject: [PATCH 62/88] Celery task to enable autorotate for all certificates attached to endpoints without it enabled --- lemur/certificates/service.py | 25 ++++++++++++++++++++++--- lemur/common/celery.py | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index a6bbba30..b031d86b 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -118,6 +118,21 @@ def get_all_pending_cleaning_expired(source): ) +def get_all_certs_attached_to_endpoint_without_rotate(): + """ + Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled. + + :return: list of certificates attached to an endpoint without autorotate + """ + return ( + Certificate.query.filter(Certificate.endpoints.any()) + .filter(Certificate.rotation == False) + .filter(Certificate.not_after >= arrow.now()) + .filter(not_(Certificate.replaced.any())) + .all() # noqa + ) + + def get_all_pending_cleaning_expiring_in_days(source, days_to_expire): """ Retrieves all certificates that are available for cleaning, not attached to endpoint, @@ -144,7 +159,9 @@ def get_all_pending_cleaning_issued_since_days(source, days_since_issuance): :param source: the source to search for certificates :return: list of pending certificates """ - not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + not_in_use_window = ( + arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + ) return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) .filter(not_(Certificate.endpoints.any())) @@ -367,9 +384,11 @@ def render(args): show_expired = args.pop("showExpired") if show_expired != 1: - one_month_old = arrow.now()\ - .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\ + one_month_old = ( + arrow.now() + .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1)) .format("YYYY-MM-DD") + ) query = query.filter(Certificate.not_after > one_month_old) time_range = args.pop("time_range") diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 7c183dc9..a5f608b2 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -17,8 +17,10 @@ from celery.signals import task_failure, task_received, task_revoked, task_succe from datetime import datetime, timezone, timedelta from flask import current_app +from lemur import database from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate +from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_rotate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.dns_providers import cli as cli_dns_providers @@ -812,3 +814,25 @@ def notify_expirations(): metrics.send(f"{function}.success", "counter", 1) return log_data + + +@celery.task(soft_time_limit=3600) +def enable_autorotate_for_certs_attached_to_endpoint(): + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "task_id": task_id, + } + + eligible_certs = get_all_certs_attached_to_endpoint_without_rotate() + for cert in eligible_certs: + log_data["certificate"] = cert.name + log_data["certificate_id"] = cert.id + log_data["message"] = "Enabling auto-rotate for certificate" + current_app.logger.info(log_data) + cert.rotation = True + database.update(cert) From 863af7a3e5f509b984ff2adba31515ebb7187624 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 12:16:46 -0700 Subject: [PATCH 63/88] Making CLI command ; Running black --- lemur/certificates/cli.py | 50 +++++++++++++++++++++++------------ lemur/certificates/service.py | 2 +- lemur/common/celery.py | 21 +++++++-------- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index b57ff175..ca6b0248 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -5,29 +5,18 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import sys import multiprocessing -from tabulate import tabulate -from sqlalchemy import or_ - +import sys from flask import current_app - -from flask_script import Manager from flask_principal import Identity, identity_changed - +from flask_script import Manager +from sqlalchemy import or_ +from tabulate import tabulate from lemur import database -from lemur.extensions import sentry -from lemur.extensions import metrics -from lemur.plugins.base import plugins -from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS -from lemur.deployment import service as deployment_service -from lemur.endpoints import service as endpoint_service -from lemur.notifications.messaging import send_rotation_notification -from lemur.domains.models import Domain from lemur.authorities.models import Authority -from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.models import Certificate +from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.service import ( reissue_certificate, get_certificate_primitives, @@ -35,9 +24,16 @@ from lemur.certificates.service import ( get_by_name, get_all_certs, get, + get_all_certs_attached_to_endpoint_without_autorotate, ) - from lemur.certificates.verify import verify_string +from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS +from lemur.deployment import service as deployment_service +from lemur.domains.models import Domain +from lemur.endpoints import service as endpoint_service +from lemur.extensions import sentry, metrics +from lemur.notifications.messaging import send_rotation_notification +from lemur.plugins.base import plugins manager = Manager(usage="Handles all certificate related tasks.") @@ -482,3 +478,23 @@ def check_revoked(): cert.status = "unknown" database.update(cert) + + +@manager.command +def automatically_enable_autorotate(): + """ + This function automatically enables autorotation for unexpired certificates that are + attached to an endpoint but do not have autorotate enabled. + """ + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + } + + eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate() + for cert in eligible_certs: + log_data["certificate"] = cert.name + log_data["certificate_id"] = cert.id + log_data["message"] = "Enabling auto-rotate for certificate" + current_app.logger.info(log_data) + cert.rotation = True + database.update(cert) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index b031d86b..5d1e6e63 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -118,7 +118,7 @@ def get_all_pending_cleaning_expired(source): ) -def get_all_certs_attached_to_endpoint_without_rotate(): +def get_all_certs_attached_to_endpoint_without_autorotate(): """ Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled. diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a5f608b2..7701b82d 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -17,10 +17,8 @@ from celery.signals import task_failure, task_received, task_revoked, task_succe from datetime import datetime, timezone, timedelta from flask import current_app -from lemur import database from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate -from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_rotate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.dns_providers import cli as cli_dns_providers @@ -818,21 +816,20 @@ def notify_expirations(): @celery.task(soft_time_limit=3600) def enable_autorotate_for_certs_attached_to_endpoint(): - function = f"{__name__}.{sys._getframe().f_code.co_name}" + """ + This celery task automatically enables autorotation for unexpired certificates that are + attached to an endpoint but do not have autorotate enabled. + :return: + """ task_id = None if celery.current_task: task_id = celery.current_task.request.id log_data = { - "function": function, + "function": f"{__name__}.{sys._getframe().f_code.co_name}", "task_id": task_id, + "message": "Enabling autorotate to eligible certificates", } + current_app.logger.debug(log_data) - eligible_certs = get_all_certs_attached_to_endpoint_without_rotate() - for cert in eligible_certs: - log_data["certificate"] = cert.name - log_data["certificate_id"] = cert.id - log_data["message"] = "Enabling auto-rotate for certificate" - current_app.logger.info(log_data) - cert.rotation = True - database.update(cert) + cli_certificate.automatically_enable_autorotate() From 7e97d885dfea1b86d7274c472f106a318ddc4738 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 13:16:27 -0700 Subject: [PATCH 64/88] Address comments --- lemur/common/celery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 7701b82d..5df470ab 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -821,15 +821,18 @@ def enable_autorotate_for_certs_attached_to_endpoint(): attached to an endpoint but do not have autorotate enabled. :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None if celery.current_task: task_id = celery.current_task.request.id log_data = { - "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "function": function, "task_id": task_id, "message": "Enabling autorotate to eligible certificates", } current_app.logger.debug(log_data) cli_certificate.automatically_enable_autorotate() + metrics.send(f"{function}.success", "counter", 1) + return log_data From 4c40e806bcb0be7006f20654c39803dd0b4c8ab0 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 29 Apr 2020 08:41:42 -0700 Subject: [PATCH 65/88] Requirements fix for Pip 20.1 --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index fa5a23bc..f73ebb6f 100644 --- a/setup.py +++ b/setup.py @@ -45,16 +45,16 @@ with open(os.path.join(ROOT, 'lemur', '__about__.py')) as f: exec(f.read(), about) # nosec: about file is benign install_requires_g = parse_requirements("requirements.txt", session=PipSession()) -install_requires = [str(ir.req) for ir in install_requires_g] +install_requires = [str(ir.requirement) for ir in install_requires_g] tests_require_g = parse_requirements("requirements-tests.txt", session=PipSession()) -tests_require = [str(ir.req) for ir in tests_require_g] +tests_require = [str(ir.requirement) for ir in tests_require_g] -docs_require_g = parse_requirements("requirements-docs.txt", session=PipSession()) -docs_require = [str(ir.req) for ir in docs_require_g] +docs_require_g = parse_requirements("require20ments-docs.txt", session=PipSession()) +docs_require = [str(ir.requirement) for ir in docs_require_g] dev_requires_g = parse_requirements("requirements-dev.txt", session=PipSession()) -dev_requires = [str(ir.req) for ir in dev_requires_g] +dev_requires = [str(ir.requirement) for ir in dev_requires_g] class SmartInstall(install): From ba8184c87492413dfed09a3405e0ae560ed9b9aa Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 29 Apr 2020 08:51:09 -0700 Subject: [PATCH 66/88] Fix requirement parsing for pip 20.1 --- setup.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index f73ebb6f..4ce03d70 100644 --- a/setup.py +++ b/setup.py @@ -45,16 +45,20 @@ with open(os.path.join(ROOT, 'lemur', '__about__.py')) as f: exec(f.read(), about) # nosec: about file is benign install_requires_g = parse_requirements("requirements.txt", session=PipSession()) -install_requires = [str(ir.requirement) for ir in install_requires_g] - tests_require_g = parse_requirements("requirements-tests.txt", session=PipSession()) -tests_require = [str(ir.requirement) for ir in tests_require_g] - -docs_require_g = parse_requirements("require20ments-docs.txt", session=PipSession()) -docs_require = [str(ir.requirement) for ir in docs_require_g] - +docs_require_g = parse_requirements("requirements-docs.txt", session=PipSession()) dev_requires_g = parse_requirements("requirements-dev.txt", session=PipSession()) -dev_requires = [str(ir.requirement) for ir in dev_requires_g] + +if tuple(map(int, pip.__version__.split('.'))) >= (20, 1): + install_requires = [str(ir.requirement) for ir in install_requires_g] + tests_require = [str(ir.requirement) for ir in tests_require_g] + docs_require = [str(ir.requirement) for ir in docs_require_g] + dev_requires = [str(ir.requirement) for ir in dev_requires_g] +else: + install_requires = [str(ir.req) for ir in install_requires_g] + tests_require = [str(ir.req) for ir in tests_require_g] + docs_require = [str(ir.req) for ir in docs_require_g] + dev_requires = [str(ir.req) for ir in dev_requires_g] class SmartInstall(install): From d469700cbf46c820f9f73366fb524c6232787c6f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 00:23:52 +0000 Subject: [PATCH 67/88] Bump fakeredis from 1.4.0 to 1.4.1 Bumps [fakeredis](https://github.com/jamesls/fakeredis) from 1.4.0 to 1.4.1. - [Release notes](https://github.com/jamesls/fakeredis/releases) - [Commits](https://github.com/jamesls/fakeredis/compare/1.4.0...1.4.1) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 5f9cafcc..7ffb2d5f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -26,14 +26,14 @@ docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in faker==4.0.3 # via -r requirements-tests.in, factory-boy -fakeredis==1.4.0 # via -r requirements-tests.in +fakeredis==1.4.1 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk gitdb==4.0.4 # via gitpython gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle itsdangerous==1.1.0 # via flask jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore @@ -72,7 +72,7 @@ requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 -six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto From 7f6eae7213db19d62c0963fea6eb68396fa22f54 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 00:25:47 +0000 Subject: [PATCH 68/88] Bump cryptography from 2.9 to 2.9.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.9 to 2.9.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.9...2.9.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 4 +--- requirements-docs.txt | 7 ++++--- requirements-tests.txt | 6 +++--- requirements.txt | 4 +--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cb2edc22..36a0bd7f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,14 +10,13 @@ certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests -cryptography==2.9 # via secretstorage +cryptography==2.9.2 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv flake8==3.5.0 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests -importlib-metadata==1.6.0 # via keyring, pre-commit, twine, virtualenv invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine @@ -41,7 +40,6 @@ twine==3.1.1 # via -r requirements-dev.in urllib3==1.25.8 # via requests virtualenv==20.0.17 # via pre-commit webencodings==0.5.1 # via bleach -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-docs.txt b/requirements-docs.txt index 54bde14e..c1abe7e0 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -25,7 +25,7 @@ cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynac chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask cloudflare==2.6.5 # via -r requirements.txt -cryptography==2.9 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +cryptography==2.9.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 docutils==0.15.2 # via -r requirements.txt, botocore, sphinx @@ -45,7 +45,6 @@ gunicorn==20.0.4 # via -r requirements.txt hvac==0.10.1 # via -r requirements.txt idna==2.9 # via -r requirements.txt, requests imagesize==1.2.0 # via sphinx -importlib-metadata==1.6.0 # via -r requirements.txt, kombu inflection==0.4.0 # via -r requirements.txt itsdangerous==1.1.0 # via -r requirements.txt, flask javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks @@ -66,6 +65,8 @@ packaging==20.3 # via sphinx paramiko==2.7.1 # via -r requirements.txt pem==20.1.0 # via -r requirements.txt psycopg2==2.8.5 # via -r requirements.txt +pyasn1-modules==0.2.8 # via -r requirements.txt, pyjks, python-ldap +pyasn1==0.4.8 # via -r requirements.txt, ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.20 # via -r requirements.txt, cffi pycryptodomex==3.9.7 # via -r requirements.txt, pyjks pygments==2.6.1 # via sphinx @@ -78,6 +79,7 @@ pyrfc3339==1.1 # via -r requirements.txt, acme python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore python-editor==1.0.4 # via -r requirements.txt, alembic python-json-logger==0.1.11 # via -r requirements.txt, logmatic-python +python-ldap==3.2.0 # via -r requirements.txt pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt @@ -105,7 +107,6 @@ urllib3==1.25.8 # via -r requirements.txt, botocore, requests vine==1.3.0 # via -r requirements.txt, amqp, celery werkzeug==1.0.1 # via -r requirements.txt, flask xmltodict==0.12.0 # via -r requirements.txt -zipp==3.1.0 # via -r requirements.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index 5f9cafcc..494ac1d4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -19,7 +19,7 @@ cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.1 # via black, flask coverage==5.1 # via -r requirements-tests.in -cryptography==2.9 # via moto, sshpubkeys +cryptography==2.9.2 # via moto, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto docutils==0.15.2 # via botocore @@ -33,7 +33,7 @@ future==0.18.2 # via aws-xray-sdk gitdb==4.0.4 # via gitpython gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle itsdangerous==1.1.0 # via flask jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore @@ -72,7 +72,7 @@ requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 -six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto diff --git a/requirements.txt b/requirements.txt index bdad75b0..70a04ee1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask cloudflare==2.6.5 # via -r requirements.in -cryptography==2.9 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +cryptography==2.9.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 docutils==0.15.2 # via botocore @@ -42,7 +42,6 @@ future==0.18.2 # via -r requirements.in, cloudflare gunicorn==20.0.4 # via -r requirements.in hvac==0.10.1 # via -r requirements.in idna==2.9 # via requests -importlib-metadata==1.6.0 # via kombu inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks @@ -92,7 +91,6 @@ urllib3==1.25.8 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==1.0.1 # via flask xmltodict==0.12.0 # via -r requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From f68900d2b355660f495f9243cdab19cdc5338a30 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 May 2020 18:28:01 -0700 Subject: [PATCH 69/88] improving logging and the possibility of defining which Authorities qualify for auto-rotation --- lemur/certificates/cli.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index ca6b0248..3be794e7 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -483,18 +483,35 @@ def check_revoked(): @manager.command def automatically_enable_autorotate(): """ - This function automatically enables autorotation for unexpired certificates that are + This function automatically enables auto-rotation for unexpired certificates that are attached to an endpoint but do not have autorotate enabled. + + WARNING: This will overwrite the Auto-rotate toggle! """ log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", } + permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", []) + eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate() for cert in eligible_certs: + + if cert.authority_id not in permitted_authorities: + continue + log_data["certificate"] = cert.name log_data["certificate_id"] = cert.id log_data["message"] = "Enabling auto-rotate for certificate" current_app.logger.info(log_data) cert.rotation = True database.update(cert) + # TODO: add the cert destination to the logging + metrics.send("automatically_enable_autorotate", + "counter", 1, + metric_tags={"certificate": cert.name, + "certificate_id": cert.id, + "authority_id": cert.authority_id, + "authority_name": Authority.get(cert.authority_id).name}) + cert.rotation = True + database.update(cert) From 529ee04ae748f1b138ca90929bdf1b72a1e9b5f2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 8 May 2020 09:15:18 -0700 Subject: [PATCH 70/88] removing duplicate line --- lemur/certificates/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 3be794e7..54455eec 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -504,8 +504,6 @@ def automatically_enable_autorotate(): log_data["certificate_id"] = cert.id log_data["message"] = "Enabling auto-rotate for certificate" current_app.logger.info(log_data) - cert.rotation = True - database.update(cert) # TODO: add the cert destination to the logging metrics.send("automatically_enable_autorotate", "counter", 1, From fa13bda99e2ff56355ee5aa6ab4d313282ec019c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 16:45:06 +0000 Subject: [PATCH 71/88] Bump sphinx from 3.0.1 to 3.0.3 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.0.1 to 3.0.3. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.0.1...v3.0.3) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index c1abe7e0..7dc714a9 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -91,7 +91,7 @@ s3transfer==0.3.3 # via -r requirements.txt, boto3 six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in -sphinx==3.0.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.0.3 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 8e2226180a38af17e2589cd60fe235374f6c3946 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 18:35:37 +0000 Subject: [PATCH 72/88] Bump pre-commit from 2.2.0 to 2.3.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.2.0...v2.3.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 36a0bd7f..68e28b73 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.2.0 # via -r requirements-dev.in +pre-commit==2.3.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 From 5ad9c11716565bf67c81fdb98d849b44b6dada44 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 18:42:41 +0000 Subject: [PATCH 73/88] Bump arrow from 0.15.5 to 0.15.6 Bumps [arrow](https://github.com/crsmithdev/arrow) from 0.15.5 to 0.15.6. - [Release notes](https://github.com/crsmithdev/arrow/releases) - [Changelog](https://github.com/crsmithdev/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/crsmithdev/arrow/compare/0.15.5...0.15.6) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 7dc714a9..db255de3 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -10,7 +10,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate amqp==2.5.2 # via -r requirements.txt, kombu aniso8601==8.0.0 # via -r requirements.txt, flask-restful -arrow==0.15.5 # via -r requirements.txt +arrow==0.15.6 # via -r requirements.txt asyncpool==1.0 # via -r requirements.txt babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko diff --git a/requirements.txt b/requirements.txt index 70a04ee1..f8d553d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.15.5 # via -r requirements.in +arrow==0.15.6 # via -r requirements.in asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery From dac95313b84c4db66d179531e7e8e2f7f56dc35f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:29:51 +0000 Subject: [PATCH 74/88] Bump pyjks from 19.0.0 to 20.0.0 Bumps [pyjks](https://github.com/kurtbrose/pyjks) from 19.0.0 to 20.0.0. - [Release notes](https://github.com/kurtbrose/pyjks/releases) - [Changelog](https://github.com/kurtbrose/pyjks/blob/master/CHANGELOG.md) - [Commits](https://github.com/kurtbrose/pyjks/compare/v19.0.0...v20.0.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index db255de3..14e54b59 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -70,7 +70,7 @@ pyasn1==0.4.8 # via -r requirements.txt, ndg-httpsclient, pyasn1-mod pycparser==2.20 # via -r requirements.txt, cffi pycryptodomex==3.9.7 # via -r requirements.txt, pyjks pygments==2.6.1 # via sphinx -pyjks==19.0.0 # via -r requirements.txt +pyjks==20.0.0 # via -r requirements.txt pyjwt==1.7.1 # via -r requirements.txt pynacl==1.3.0 # via -r requirements.txt, paramiko pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests diff --git a/requirements.txt b/requirements.txt index f8d553d4..83585828 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,7 +65,7 @@ pyasn1-modules==0.2.8 # via pyjks, python-ldap pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.20 # via cffi pycryptodomex==3.9.7 # via pyjks -pyjks==19.0.0 # via -r requirements.in +pyjks==20.0.0 # via -r requirements.in pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests From 6bef8fb9d7f86e7e76b9ef4489367f03a6d72489 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:36:54 +0000 Subject: [PATCH 75/88] Bump acme from 1.3.0 to 1.4.0 Bumps [acme](https://github.com/letsencrypt/letsencrypt) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/letsencrypt/letsencrypt/releases) - [Commits](https://github.com/letsencrypt/letsencrypt/compare/v1.3.0...v1.4.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 3 +-- requirements.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 14e54b59..6c2a37e4 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.3.0 # via -r requirements.txt +acme==1.4.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate @@ -59,7 +59,6 @@ mako==1.1.2 # via -r requirements.txt, alembic markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy -mock==4.0.2 # via -r requirements.txt, acme ndg-httpsclient==0.5.1 # via -r requirements.txt packaging==20.3 # via sphinx paramiko==2.7.1 # via -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 83585828..ad92d95f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.3.0 # via -r requirements.in +acme==1.4.0 # via -r requirements.in alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu @@ -56,7 +56,6 @@ mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.22.3 # via -r requirements.in marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy -mock==4.0.2 # via acme ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.1 # via -r requirements.in pem==20.1.0 # via -r requirements.in From 5ac65062c93e52bcac58209337018aaae3db3e19 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:49:15 +0000 Subject: [PATCH 76/88] Bump sqlalchemy-utils from 0.36.3 to 0.36.5 Bumps [sqlalchemy-utils](https://github.com/kvesteri/sqlalchemy-utils) from 0.36.3 to 0.36.5. - [Release notes](https://github.com/kvesteri/sqlalchemy-utils/releases) - [Changelog](https://github.com/kvesteri/sqlalchemy-utils/blob/master/CHANGES.rst) - [Commits](https://github.com/kvesteri/sqlalchemy-utils/compare/0.36.3...0.36.5) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 6c2a37e4..d8bc26d7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -98,7 +98,7 @@ sphinxcontrib-httpdomain==1.7.0 # via -r requirements-docs.in sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sqlalchemy-utils==0.36.3 # via -r requirements.txt +sqlalchemy-utils==0.36.5 # via -r requirements.txt sqlalchemy==1.3.16 # via -r requirements.txt, alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.7 # via -r requirements.txt twofish==0.3.0 # via -r requirements.txt, pyjks diff --git a/requirements.txt b/requirements.txt index ad92d95f..60b38553 100644 --- a/requirements.txt +++ b/requirements.txt @@ -82,7 +82,7 @@ requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, retrying==1.3.3 # via -r requirements.in s3transfer==0.3.3 # via boto3 six==1.14.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils -sqlalchemy-utils==0.36.3 # via -r requirements.in +sqlalchemy-utils==0.36.5 # via -r requirements.in sqlalchemy==1.3.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.7 # via -r requirements.in twofish==0.3.0 # via pyjks From 5afa32fa226ff972eb4ca2d078058e5258ea6874 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:57:55 +0000 Subject: [PATCH 77/88] Bump boto3 from 1.12.39 to 1.13.6 Bumps [boto3](https://github.com/boto/boto3) from 1.12.39 to 1.13.6. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.12.39...1.13.6) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d8bc26d7..a468419c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,8 +16,8 @@ babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.12.39 # via -r requirements.txt -botocore==1.15.39 # via -r requirements.txt, boto3, s3transfer +boto3==1.13.6 # via -r requirements.txt +botocore==1.16.6 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 9c9bbff0..5bb54cc6 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in -boto3==1.12.39 # via aws-sam-translator, moto +boto3==1.13.6 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.16.6 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 60b38553..c2e269e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,8 @@ asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.12.39 # via -r requirements.in -botocore==1.15.39 # via -r requirements.in, boto3, s3transfer +boto3==1.13.6 # via -r requirements.in +botocore==1.16.6 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 201a96ff72d7c518a37b5497fbb920c61d5a6144 Mon Sep 17 00:00:00 2001 From: Luka Matijevic Date: Mon, 11 May 2020 13:09:23 +0200 Subject: [PATCH 78/88] Improve periodic tasks docs --- docs/production/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/production/index.rst b/docs/production/index.rst index cd044ca4..b91ed6bd 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -390,6 +390,10 @@ Here are the Celery configuration variables that should be set:: CELERY_IMPORTS = ('lemur.common.celery') CELERY_TIMEZONE = 'UTC' +Do not forget to import crontab module in your configuration file:: + + from celery.task.schedules import crontab + You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks. The scheduler can be started with:: From 5d75204a4f81ec30bdebfba277b8def133dc2a1f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 13:45:08 +0000 Subject: [PATCH 79/88] Bump faker from 4.0.3 to 4.1.0 Bumps [faker](https://github.com/joke2k/faker) from 4.0.3 to 4.1.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.0.3...v4.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 5bb54cc6..32b4d72b 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in -faker==4.0.3 # via -r requirements-tests.in, factory-boy +faker==4.1.0 # via -r requirements-tests.in, factory-boy fakeredis==1.4.1 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in From c0829478fdf62d46b7ba006129ec21b8489c5ba0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:14:27 +0000 Subject: [PATCH 80/88] Bump requests-mock from 1.7.0 to 1.8.0 Bumps [requests-mock](https://github.com/jamielennox/requests-mock) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/jamielennox/requests-mock/releases) - [Commits](https://github.com/jamielennox/requests-mock/compare/1.7.0...1.8.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 32b4d72b..cac0f42a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -67,7 +67,7 @@ pytz==2019.3 # via moto pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto redis==3.4.1 # via fakeredis regex==2020.4.4 # via black -requests-mock==1.7.0 # via -r requirements-tests.in +requests-mock==1.8.0 # via -r requirements-tests.in requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose From 9e1737edc49b307a8526b278ddec8010620ac6d5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:25:47 +0000 Subject: [PATCH 81/88] Bump pytest from 5.4.1 to 5.4.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.1 to 5.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.1...5.4.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index cac0f42a..369408dd 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -60,7 +60,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in pytest-mock==3.0.0 # via -r requirements-tests.in -pytest==5.4.1 # via -r requirements-tests.in, pytest-flask, pytest-mock +pytest==5.4.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto From dfd9acb0d7f0b604162c57ebf791c7a060a569a3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:37:21 +0000 Subject: [PATCH 82/88] Bump pytest-mock from 3.0.0 to 3.1.0 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v3.0.0...v3.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 369408dd..3b5f83db 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -59,7 +59,7 @@ pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in -pytest-mock==3.0.0 # via -r requirements-tests.in +pytest-mock==3.1.0 # via -r requirements-tests.in pytest==5.4.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto From 23cd7d3d625a8919ebeb9d8497ad76e0e30154ba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:47:18 +0000 Subject: [PATCH 83/88] Bump marshmallow-sqlalchemy from 0.22.3 to 0.23.0 Bumps [marshmallow-sqlalchemy](https://github.com/marshmallow-code/marshmallow-sqlalchemy) from 0.22.3 to 0.23.0. - [Release notes](https://github.com/marshmallow-code/marshmallow-sqlalchemy/releases) - [Changelog](https://github.com/marshmallow-code/marshmallow-sqlalchemy/blob/dev/CHANGELOG.rst) - [Commits](https://github.com/marshmallow-code/marshmallow-sqlalchemy/compare/0.22.3...0.23.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index a468419c..8b22bb16 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -57,7 +57,7 @@ lockfile==0.12.2 # via -r requirements.txt logmatic-python==0.1.7 # via -r requirements.txt mako==1.1.2 # via -r requirements.txt, alembic markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako -marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt +marshmallow-sqlalchemy==0.23.0 # via -r requirements.txt marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy ndg-httpsclient==0.5.1 # via -r requirements.txt packaging==20.3 # via sphinx diff --git a/requirements.txt b/requirements.txt index c2e269e5..40fff126 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ lockfile==0.12.2 # via -r requirements.in logmatic-python==0.1.7 # via -r requirements.in mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.22.3 # via -r requirements.in +marshmallow-sqlalchemy==0.23.0 # via -r requirements.in marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.1 # via -r requirements.in From d3a0fe74911bbe891bc7cd3b7d4d91b448229454 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:58:47 +0000 Subject: [PATCH 84/88] Bump cloudflare from 2.6.5 to 2.7.1 Bumps [cloudflare](https://github.com/cloudflare/python-cloudflare) from 2.6.5 to 2.7.1. - [Release notes](https://github.com/cloudflare/python-cloudflare/releases) - [Changelog](https://github.com/cloudflare/python-cloudflare/blob/master/CHANGELOG.md) - [Commits](https://github.com/cloudflare/python-cloudflare/compare/2.6.5...2.7.1) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 8b22bb16..33b53872 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -24,7 +24,7 @@ certsrv==2.1.1 # via -r requirements.txt cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask -cloudflare==2.6.5 # via -r requirements.txt +cloudflare==2.7.1 # via -r requirements.txt cryptography==2.9.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 diff --git a/requirements.txt b/requirements.txt index 40fff126..d413c566 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask -cloudflare==2.6.5 # via -r requirements.in +cloudflare==2.7.1 # via -r requirements.in cryptography==2.9.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 From 9bc017c9bf4e2d81411750f06ce00a537f703d01 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:11:39 +0000 Subject: [PATCH 85/88] Bump botocore from 1.16.6 to 1.16.11 Bumps [botocore](https://github.com/boto/botocore) from 1.16.6 to 1.16.11. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.16.6...1.16.11) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 33b53872..7d49fe56 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,7 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.13.6 # via -r requirements.txt -botocore==1.16.6 # via -r requirements.txt, boto3, s3transfer +botocore==1.16.11 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 3b5f83db..5ca1f238 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in boto3==1.13.6 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.16.6 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.16.11 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index d413c566..72f3938b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.13.6 # via -r requirements.in -botocore==1.16.6 # via -r requirements.in, boto3, s3transfer +botocore==1.16.11 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From d010553d23b1166122dab68394973f0ffc5c8847 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:23:23 +0000 Subject: [PATCH 86/88] Bump redis from 3.4.1 to 3.5.2 Bumps [redis](https://github.com/andymccurdy/redis-py) from 3.4.1 to 3.5.2. - [Release notes](https://github.com/andymccurdy/redis-py/releases) - [Changelog](https://github.com/andymccurdy/redis-py/blob/master/CHANGES) - [Commits](https://github.com/andymccurdy/redis-py/compare/3.4.1...3.5.2) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 7d49fe56..d3e61970 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -82,7 +82,7 @@ python-ldap==3.2.0 # via -r requirements.txt pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt -redis==3.4.1 # via -r requirements.txt, celery +redis==3.5.2 # via -r requirements.txt, celery requests-toolbelt==0.9.1 # via -r requirements.txt, acme requests[security]==2.23.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx retrying==1.3.3 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 5ca1f238..93d992e5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -65,7 +65,7 @@ python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto -redis==3.4.1 # via fakeredis +redis==3.5.2 # via fakeredis regex==2020.4.4 # via black requests-mock==1.8.0 # via -r requirements-tests.in requests==2.23.0 # via docker, moto, requests-mock, responses diff --git a/requirements.txt b/requirements.txt index 72f3938b..0ad0708c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -76,7 +76,7 @@ python-ldap==3.2.0 # via -r requirements.in pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.in, cloudflare raven[flask]==6.10.0 # via -r requirements.in -redis==3.4.1 # via -r requirements.in, celery +redis==3.5.2 # via -r requirements.in, celery requests-toolbelt==0.9.1 # via acme requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt retrying==1.3.3 # via -r requirements.in From 2bf03a0bc245c20524034b72477104bbd3f0a4b2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:35:43 +0000 Subject: [PATCH 87/88] Bump boto3 from 1.13.6 to 1.13.11 Bumps [boto3](https://github.com/boto/boto3) from 1.13.6 to 1.13.11. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.13.6...1.13.11) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d3e61970..16d97413 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,7 +16,7 @@ babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.13.6 # via -r requirements.txt +boto3==1.13.11 # via -r requirements.txt botocore==1.16.11 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests diff --git a/requirements-tests.txt b/requirements-tests.txt index 93d992e5..79340e51 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in -boto3==1.13.6 # via aws-sam-translator, moto +boto3==1.13.11 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.16.11 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests diff --git a/requirements.txt b/requirements.txt index 0ad0708c..315f39b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.13.6 # via -r requirements.in +boto3==1.13.11 # via -r requirements.in botocore==1.16.11 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests From 4d4ff8509062dc82369c00b107120a4c5bffb88d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:44:22 +0000 Subject: [PATCH 88/88] Bump pre-commit from 2.3.0 to 2.4.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.3.0...v2.4.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 68e28b73..785d3f29 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.3.0 # via -r requirements-dev.in +pre-commit==2.4.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8