From bfe89e131e71115a779359dd9a389ba8ec306bdb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 15 Oct 2020 18:13:50 -0700 Subject: [PATCH 01/49] adding delete and put interfaces for the S3 plugin --- lemur/plugins/lemur_aws/s3.py | 40 ++++++++++++++++++++++-- lemur/plugins/lemur_aws/tests/test_s3.py | 38 ++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 lemur/plugins/lemur_aws/tests/test_s3.py diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 43faa28f..c868c7a3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -9,9 +9,12 @@ from flask import current_app from .sts import sts_client +from botocore.exceptions import ClientError +from lemur.extensions import sentry + @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ @@ -32,4 +35,37 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs): ServerSideEncryption="AES256", ) else: - bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + try: + bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + except ClientError: + sentry.captureException() + + +@sts_client("s3", service_type="client") +def delete(bucket_name, prefix, **kwargs): + """ + Use STS to delete an object + """ + try: + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix) + current_app.logger.debug(f"Delete data from S3." + f"Bucket: {bucket_name}," + f"Prefix: {prefix}," + f"Status_code: {response}") + return response['ResponseMetadata']['HTTPStatusCode'] < 300 + except ClientError: + sentry.captureException() + + +@sts_client("s3", service_type="client") +def get(bucket_name, prefix, **kwargs): + """ + Use STS to get an object + """ + try: + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix) + current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," + f"Prefix: {prefix}") + return response['Body'].read().decode("utf-8") + except ClientError: + sentry.captureException() diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py new file mode 100644 index 00000000..f7a36496 --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -0,0 +1,38 @@ +import boto3 +from moto import mock_sts, mock_s3 + + +@mock_sts() +@mock_s3() +def test_put_delete_s3_object(app): + from lemur.plugins.lemur_aws.s3 import put, delete, get + + bucket = "public-bucket" + account = "123456789012" + path = "some_path/foo" + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + + data = "dummy data" + put(bucket_name=bucket, + prefix=path, + data=data, + encrypt=None, + account_number=account) + + response = get(bucket_name=bucket, prefix=path, account_number=account) + + # put data, and getting the same data + assert (response == data) + + response = get(bucket_name="wrong-bucket", prefix=path, account_number=account) + + # attempting to get thccle wrong data + assert (response is None) + + delete(bucket_name=bucket, prefix=path, account_number=account) + response = get(bucket_name=bucket, prefix=path, account_number=account) + + # delete data, and getting the same data + assert (response is None) From d73db59d2352ad54d230948cb4e323cbc46cd30b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:48:47 -0700 Subject: [PATCH 02/49] revsering removing region --- lemur/plugins/lemur_aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index c868c7a3..0e9db182 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -14,7 +14,7 @@ from lemur.extensions import sentry @sts_client("s3", service_type="resource") -def put(bucket_name, prefix, data, encrypt, **kwargs): +def put(bucket_name, region, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ From 6aad37e1f9d115da065c321d88e21b38045ae80c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:00 -0700 Subject: [PATCH 03/49] cleaning up code --- lemur/plugins/lemur_aws/s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 0e9db182..186b715d 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -6,12 +6,12 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from flask import current_app -from .sts import sts_client - from botocore.exceptions import ClientError +from flask import current_app from lemur.extensions import sentry +from .sts import sts_client + @sts_client("s3", service_type="resource") def put(bucket_name, region, prefix, data, encrypt, **kwargs): From 7d8eb1c61edba2656257fbe2c42f864ce9b87107 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:26 -0700 Subject: [PATCH 04/49] improving test --- lemur/plugins/lemur_aws/tests/test_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index f7a36496..7b8b4ac3 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -9,16 +9,16 @@ def test_put_delete_s3_object(app): bucket = "public-bucket" account = "123456789012" - path = "some_path/foo" + path = "some-path/foo" + data = "dummy data" s3_client = boto3.client('s3') s3_client.create_bucket(Bucket=bucket) - data = "dummy data" put(bucket_name=bucket, prefix=path, data=data, - encrypt=None, + encrypt=False, account_number=account) response = get(bucket_name=bucket, prefix=path, account_number=account) From d705e3ae3b5cce2fd8cfd34ebc2283217d180557 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:56 -0700 Subject: [PATCH 05/49] expanding the S3 destination plugin to support the acme token upload inteface --- lemur/plugins/lemur_aws/plugin.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8692348a..6d161ac3 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -406,3 +406,25 @@ class S3DestinationPlugin(ExportDestinationPlugin): self.get_option("encrypt", options), account_number=self.get_option("accountNumber", options), ) + + def upload_acme_token(self, token_path, token, options, **kwargs): + """ + This is called from the acme http challenge + :param self: + :param token_path: + :param token: + :param options: + :param kwargs: + :return: + """ + current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge") + + account_number = self.get_option("accountNumber", options) + bucket_name = self.get_option("bucket", options) + prefix = self.get_option("prefix", options) + region = self.get_option("region", options) + filename = token_path.split("/")[-1] + if not prefix.endswith("/"): + prefix + "/" + + s3.put(bucket_name, region, prefix + filename, token, encrypt=False, account_number=account_number) From 17e528b5dd56284b097f85bb2f6d1e895ac7bfc6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:50:35 -0700 Subject: [PATCH 06/49] adding testing for acme_upload method --- lemur/plugins/lemur_aws/tests/test_plugin.py | 77 ++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index dbad7b02..a471f7c8 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -1,5 +1,82 @@ +import boto3 +from moto import mock_sts, mock_s3 + + def test_get_certificates(app): from lemur.plugins.base import plugins p = plugins.get("aws-s3") assert p + + +@mock_sts() +@mock_s3() +def test_upload_acme_token(app): + from lemur.plugins.base import plugins + from lemur.plugins.lemur_aws.s3 import get + + bucket = "public-bucket" + account = "123456789012" + prefix = "some-path/more-path/" + token_content = "Challenge" + token_name = "TOKEN" + token_path = ".well-known/acme-challenge/" + token_name + + additional_options = [ + { + "name": "bucket", + "value": bucket, + "type": "str", + "required": True, + "validation": "[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account, + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + p = plugins.get("aws-s3") + + p.upload_acme_token(token_path=token_path, + token_content=token_content, + token=token_content, + options=additional_options) + + response = get(bucket_name=bucket, + prefix=prefix + token_name, + encrypt=False, + account_number=account) + + # put data, and getting the same data + assert (response == token_content) From 9c04a888d8122f100aefe646137cb8c0908ae489 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:52:04 -0700 Subject: [PATCH 07/49] adjusting the S3 test --- lemur/plugins/lemur_aws/tests/test_s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index 7b8b4ac3..bfb5a9f9 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -16,6 +16,7 @@ def test_put_delete_s3_object(app): s3_client.create_bucket(Bucket=bucket) put(bucket_name=bucket, + region=None, prefix=path, data=data, encrypt=False, From 11ce540246676b9441fde40b198d8ca0ea7ac2a7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 10:31:19 -0700 Subject: [PATCH 08/49] formatting --- lemur/plugins/lemur_aws/plugin.py | 7 ++++++- lemur/plugins/lemur_aws/s3.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 6d161ac3..ad80d87f 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -427,4 +427,9 @@ class S3DestinationPlugin(ExportDestinationPlugin): if not prefix.endswith("/"): prefix + "/" - s3.put(bucket_name, region, prefix + filename, token, encrypt=False, account_number=account_number) + s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 186b715d..7c4177ff 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -14,7 +14,7 @@ from .sts import sts_client @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ From 503530e93512291985d5d865b9b7af55d2e05488 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 10:32:10 -0700 Subject: [PATCH 09/49] the test requires region param for sts --- lemur/plugins/lemur_aws/tests/test_s3.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index bfb5a9f9..88bd30d2 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -8,6 +8,7 @@ def test_put_delete_s3_object(app): from lemur.plugins.lemur_aws.s3 import put, delete, get bucket = "public-bucket" + region = "us-east-1" account = "123456789012" path = "some-path/foo" data = "dummy data" @@ -16,11 +17,12 @@ def test_put_delete_s3_object(app): s3_client.create_bucket(Bucket=bucket) put(bucket_name=bucket, - region=None, + region_name=region, prefix=path, data=data, encrypt=False, - account_number=account) + account_number=account, + region=region) response = get(bucket_name=bucket, prefix=path, account_number=account) From c5769378cf74356f094af5587b6d0861e3df6798 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 15:21:22 -0700 Subject: [PATCH 10/49] making lint happy --- lemur/plugins/lemur_aws/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8c94dc45..489fa823 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -408,7 +408,6 @@ class S3DestinationPlugin(ExportDestinationPlugin): account_number=self.get_option("accountNumber", options), ) - def upload_acme_token(self, token_path, token, options, **kwargs): """ This is called from the acme http challenge @@ -483,4 +482,3 @@ class SNSNotificationPlugin(ExpirationNotificationPlugin): current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}") sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) - From ba8eb7a3f59e8201810f63e6a174fc2e3b0c700d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:17:02 -0700 Subject: [PATCH 11/49] better logging and metrics --- lemur/plugins/lemur_aws/plugin.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 489fa823..b54787ac 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -33,6 +33,7 @@ .. moduleauthor:: Harm Weites """ +import sys from acme.errors import ClientError from flask import current_app @@ -420,6 +421,8 @@ class S3DestinationPlugin(ExportDestinationPlugin): """ current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge") + function = f"{__name__}.{sys._getframe().f_code.co_name}" + account_number = self.get_option("accountNumber", options) bucket_name = self.get_option("bucket", options) prefix = self.get_option("prefix", options) @@ -428,12 +431,24 @@ class S3DestinationPlugin(ExportDestinationPlugin): if not prefix.endswith("/"): prefix + "/" - s3.put(bucket_name=bucket_name, - region_name=region, - prefix=prefix + filename, - data=token, - encrypt=False, - account_number=account_number) + res = s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) + res = "Success" if res else "Failure" + log_data = { + "function": function, + "message": "check if any valid certificate is revoked", + "result": res, + "bucket_name": bucket_name, + "filename": filename + } + current_app.logger.info(log_data) + metrics.send(f"{function}", "counter", 1, metric_tags={"result": res, + "bucket_name": bucket_name, + "filename": filename}) class SNSNotificationPlugin(ExpirationNotificationPlugin): From cc2aa5c1de131ec389df3ffd57059d0fd9ebc134 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:17:34 -0700 Subject: [PATCH 12/49] cli for live testing --- lemur/acme_providers/cli.py | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 310efad1..56301aae 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -1,12 +1,16 @@ import time import json +import arrow from flask_script import Manager from flask import current_app from lemur.extensions import sentry from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler +from lemur.plugins.lemur_aws import s3 +from lemur.utils import get_random_secret manager = Manager( usage="Handles all ACME related tasks" @@ -84,3 +88,96 @@ def dnstest(domain, token): status = SUCCESS_METRIC_STATUS print("[+] Done with ACME Tests.") + + +@manager.option( + "-t", + "--token", + dest="token", + default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="Value of the Token", +) +@manager.option( + "-n", + "--token_name", + dest="token_name", + default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="path", +) +@manager.option( + "-p", + "--prefix", + dest="prefix", + default="test/", + required=False, + help="S3 bucket prefix", +) +@manager.option( + "-a", + "--account_number", + dest="account_number", + required=True, + help="AWS Account", +) +@manager.option( + "-b", + "--bucket_name", + dest="bucket_name", + required=True, + help="Bucket Name", +) +def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): + + additional_options = [ + { + "name": "bucket", + "value": bucket_name, + "type": "str", + "required": True, + "validation": "[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account_number, + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + p = plugins.get("aws-s3") + p.upload_acme_token(token_name, token, additional_options) + + if not prefix.endswith("/"): + prefix + "/" + + token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number) + assert(token_res == token) + s3.delete(bucket_name, prefix + token_name, account_number=account_number) From e1ff89eb2d6c3e6fbc8149020c689c0a530a0675 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:18:14 -0700 Subject: [PATCH 13/49] better return arguments --- lemur/plugins/lemur_aws/s3.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 7c4177ff..e15f6b6e 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -37,8 +37,10 @@ def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): else: try: bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + return True except ClientError: sentry.captureException() + return False @sts_client("s3", service_type="client") @@ -55,6 +57,7 @@ def delete(bucket_name, prefix, **kwargs): return response['ResponseMetadata']['HTTPStatusCode'] < 300 except ClientError: sentry.captureException() + return False @sts_client("s3", service_type="client") @@ -69,3 +72,4 @@ def get(bucket_name, prefix, **kwargs): return response['Body'].read().decode("utf-8") except ClientError: sentry.captureException() + return None From add0960579e7e4bdfbf87a18cc0cf5b1afac4b30 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:18:37 -0700 Subject: [PATCH 14/49] more meaningful variable naming --- lemur/plugins/lemur_aws/s3.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index e15f6b6e..1b0831b3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -44,15 +44,15 @@ def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): @sts_client("s3", service_type="client") -def delete(bucket_name, prefix, **kwargs): +def delete(bucket_name, prefixed_object_name, **kwargs): """ Use STS to delete an object """ try: - response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix) + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name) current_app.logger.debug(f"Delete data from S3." f"Bucket: {bucket_name}," - f"Prefix: {prefix}," + f"Prefix: {prefixed_object_name}," f"Status_code: {response}") return response['ResponseMetadata']['HTTPStatusCode'] < 300 except ClientError: @@ -61,14 +61,14 @@ def delete(bucket_name, prefix, **kwargs): @sts_client("s3", service_type="client") -def get(bucket_name, prefix, **kwargs): +def get(bucket_name, prefixed_object_name, **kwargs): """ Use STS to get an object """ try: - response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix) + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name) current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," - f"Prefix: {prefix}") + f"object_name: {prefixed_object_name}") return response['Body'].read().decode("utf-8") except ClientError: sentry.captureException() From 3dfafa00218b384ef420cf48cbd4bbeece8743f7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:28:10 -0700 Subject: [PATCH 15/49] making lint happy --- lemur/acme_providers/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 56301aae..ec6326bd 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -10,7 +10,6 @@ from lemur.constants import SUCCESS_METRIC_STATUS from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler from lemur.plugins.lemur_aws import s3 -from lemur.utils import get_random_secret manager = Manager( usage="Handles all ACME related tasks" From 7bca42776b6e0d50825dd7d07222076b18ef32f6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:28:34 -0700 Subject: [PATCH 16/49] better comments --- lemur/acme_providers/cli.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index ec6326bd..313876e6 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -128,7 +128,16 @@ def dnstest(domain, token): help="Bucket Name", ) def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): - + """ + This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it. + It mainly serves for testing purposes. + :param token: + :param token_name: + :param prefix: + :param account_number: + :param bucket_name: + :return: + """ additional_options = [ { "name": "bucket", From 9c6856bcdd7bf93705e02a887bfef3df9cdab542 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:36:32 -0700 Subject: [PATCH 17/49] adjusting the tests to the better naming --- lemur/plugins/lemur_aws/tests/test_plugin.py | 2 +- lemur/plugins/lemur_aws/tests/test_s3.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index a471f7c8..a3227296 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -74,7 +74,7 @@ def test_upload_acme_token(app): options=additional_options) response = get(bucket_name=bucket, - prefix=prefix + token_name, + prefixed_object_name=prefix + token_name, encrypt=False, account_number=account) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index 88bd30d2..7d0fa843 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -24,18 +24,18 @@ def test_put_delete_s3_object(app): account_number=account, region=region) - response = get(bucket_name=bucket, prefix=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) # put data, and getting the same data assert (response == data) - response = get(bucket_name="wrong-bucket", prefix=path, account_number=account) + response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account) # attempting to get thccle wrong data assert (response is None) - delete(bucket_name=bucket, prefix=path, account_number=account) - response = get(bucket_name=bucket, prefix=path, account_number=account) + delete(bucket_name=bucket, prefixed_object_name=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) # delete data, and getting the same data assert (response is None) From a4d2f79a9b965225bc8b4715dcc49d24a0662439 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 13:34:06 +0000 Subject: [PATCH 18/49] Bump cryptography from 3.2 to 3.2.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.2 to 3.2.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.2...3.2.1) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f3d169f3..20808dff 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests colorama==0.4.3 # via twine -cryptography==3.2 # via secretstorage +cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv diff --git a/requirements-docs.txt b/requirements-docs.txt index 6ae23a4c..ceb62e6c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -26,7 +26,7 @@ cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynac chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask cloudflare==2.8.13 # via -r requirements.txt -cryptography==3.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 docutils==0.15.2 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 0bbee7c2..04e9af94 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -19,7 +19,7 @@ cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.2 # via black, flask coverage==5.3 # via -r requirements-tests.in -cryptography==3.2 # via moto, python-jose, sshpubkeys +cryptography==3.2.1 # via moto, python-jose, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys diff --git a/requirements.txt b/requirements.txt index d53241fa..165149d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask cloudflare==2.8.13 # via -r requirements.in -cryptography==3.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 dyn==1.8.1 # via -r requirements.in From a15e1831d072ab8dce4042f561d62c995ee240e9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 2 Nov 2020 09:43:09 -0800 Subject: [PATCH 19/49] turning off Travis-ci notification on success --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 67a1d0b1..d753c241 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,3 +48,5 @@ after_success: notifications: email: lemur@netflix.com + on_success: never + on_failure: always From 771c272895395ba730c616d5b76ead08a9950d03 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 18:51:46 +0000 Subject: [PATCH 20/49] Bump pre-commit from 2.7.1 to 2.8.2 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.7.1 to 2.8.2. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.7.1...v2.8.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 20808dff..180be728 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,7 +24,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.7.1 # via -r requirements-dev.in +pre-commit==2.8.2 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 From 2331638ed189fd5e147c204b257e0c0496459547 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:01:19 +0000 Subject: [PATCH 21/49] Bump sphinx from 3.2.1 to 3.3.0 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.2.1 to 3.3.0. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.2.1...v3.3.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index ceb62e6c..8d4750e4 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -92,7 +92,7 @@ six==1.15.0 # via -r requirements.txt, acme, bcrypt, cryptography, snowballstemmer==2.0.0 # via sphinx soupsieve==2.0.1 # via -r requirements.txt, beautifulsoup4 sphinx-rtd-theme==0.5.0 # via -r requirements-docs.in -sphinx==3.2.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.3.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From fc2fce6c0b6ddab4882ed1fd6b310792a544c3ac Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:10:18 +0000 Subject: [PATCH 22/49] Bump pytest from 6.1.1 to 6.1.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.1.1 to 6.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.1.1...6.1.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 04e9af94..6566a57c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -61,7 +61,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in -pytest==6.1.1 # via -r requirements-tests.in, pytest-flask, pytest-mock +pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose[cryptography]==3.1.0 # via moto pytz==2019.3 # via moto From fa620e539d2b985038821a18dccb994142351b2d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:21:05 +0000 Subject: [PATCH 23/49] Bump boto3 from 1.16.5 to 1.16.9 Bumps [boto3](https://github.com/boto/boto3) from 1.16.5 to 1.16.9. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.5...1.16.9) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 8d4750e4..d0f5a47c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.5 # via -r requirements.txt -botocore==1.19.5 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.9 # via -r requirements.txt +botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 6566a57c..fcc219e9 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.5 # via aws-sam-translator, moto +boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.5 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 165149d5..4e641cfd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.5 # via -r requirements.in -botocore==1.19.5 # via -r requirements.in, boto3, s3transfer +boto3==1.16.9 # via -r requirements.in +botocore==1.19.9 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 4659df42d55080fca09653c2986bb8d3a213c5db Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 2 Nov 2020 15:49:11 -0800 Subject: [PATCH 24/49] better formatting --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d753c241..f9db8d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,7 @@ after_success: notifications: email: - lemur@netflix.com + recipients: + - lemur@netflix.com on_success: never on_failure: always From 634339eac6e0ead00ad8e7bce604643e8d3d43ef Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 14:35:37 -0700 Subject: [PATCH 25/49] replacing imp (deprecated) with importlib --- lemur/factory.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lemur/factory.py b/lemur/factory.py index 0563d873..edea571a 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -10,7 +10,7 @@ """ import os -import imp +import importlib import errno import pkg_resources import socket @@ -73,8 +73,9 @@ def from_file(file_path, silent=False): :param file_path: :param silent: """ - d = imp.new_module("config") - d.__file__ = file_path + module_spec = importlib.util.spec_from_file_location("config", file_path) + d = importlib.util.module_from_spec(module_spec) + try: with open(file_path) as config_file: exec( # nosec: config file safe From 4ffced70f82af0ab98a079ded4f2e472e02184fc Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 14:37:09 -0700 Subject: [PATCH 26/49] backref cannot be set for viewonly relationship will be deprecated in SQLAlchemy 1.4, and will be disallowed in a future release --- lemur/certificates/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index f6562b3f..65918d73 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -184,7 +184,6 @@ class Certificate(db.Model): "PendingCertificate", secondary=pending_cert_replacement_associations, backref="pending_replace", - viewonly=True, ) logs = relationship("Log", backref="certificate") From 2dac95c6fb1737da8739efe8e2c28a746f33cb86 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 17:00:35 -0700 Subject: [PATCH 27/49] Replacing PassiveDefault (deprecated) with DefaultClause --- lemur/authorities/models.py | 4 ++-- lemur/certificates/models.py | 4 ++-- lemur/logs/models.py | 4 ++-- lemur/pending_certificates/models.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index 61ab779e..94985cc9 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -18,7 +18,7 @@ from sqlalchemy import ( func, ForeignKey, DateTime, - PassiveDefault, + DefaultClause, Boolean, ) from sqlalchemy.dialects.postgresql import JSON @@ -39,7 +39,7 @@ class Authority(db.Model): plugin_name = Column(String(64)) description = Column(Text) options = Column(JSON) - date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False) + date_created = Column(DateTime, DefaultClause(func.now()), nullable=False) roles = relationship( "Role", secondary=roles_authorities, diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 65918d73..94e3a42e 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -16,7 +16,7 @@ from sqlalchemy import ( Integer, ForeignKey, String, - PassiveDefault, + DefaultClause, func, Column, Text, @@ -138,7 +138,7 @@ class Certificate(db.Model): not_after = Column(ArrowType) not_after_ix = Index("ix_certificates_not_after", not_after.desc()) - date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False) + date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False) signing_algorithm = Column(String(128)) status = Column(String(128)) diff --git a/lemur/logs/models.py b/lemur/logs/models.py index 07a2ded3..30cc204a 100644 --- a/lemur/logs/models.py +++ b/lemur/logs/models.py @@ -7,7 +7,7 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy import Column, Integer, ForeignKey, PassiveDefault, func, Enum +from sqlalchemy import Column, Integer, ForeignKey, DefaultClause, func, Enum from sqlalchemy_utils.types.arrow import ArrowType @@ -29,5 +29,5 @@ class Log(db.Model): ), nullable=False, ) - logged_at = Column(ArrowType(), PassiveDefault(func.now()), nullable=False) + logged_at = Column(ArrowType(), DefaultClause(func.now()), nullable=False) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index fa6be073..ee3e5e97 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -9,7 +9,7 @@ from sqlalchemy import ( Integer, ForeignKey, String, - PassiveDefault, + DefaultClause, func, Column, Text, @@ -76,14 +76,14 @@ class PendingCertificate(db.Model): chain = Column(Text()) private_key = Column(Vault, nullable=True) - date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False) + date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False) dns_provider_id = Column( Integer, ForeignKey("dns_providers.id", ondelete="CASCADE") ) status = Column(Text(), nullable=True) last_updated = Column( - ArrowType, PassiveDefault(func.now()), onupdate=func.now(), nullable=False + ArrowType, DefaultClause(func.now()), onupdate=func.now(), nullable=False ) rotation = Column(Boolean, default=False) From d821024e35cc2c7c90d1b7f74d01804725c3ef65 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 17:51:13 -0700 Subject: [PATCH 28/49] Fixing DeprecationWarning: callable is None --- lemur/plugins/lemur_acme/tests/test_acme.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 89ca6ee1..39600eb2 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -130,9 +130,8 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge = [] dns_challenge = Mock() mock_authz.dns_challenge.append(dns_challenge) - self.assertRaises( - ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz) - ) + with self.assertRaises(ValueError): + self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From d88da028b1f69bd90940f1100b1a31f755ae7fd6 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 18:12:22 -0700 Subject: [PATCH 29/49] Replace binary with LargeBinary https://flask-appbuilder.readthedocs.io/en/latest/_modules/sqlalchemy/sql/sqltypes.html --- lemur/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/utils.py b/lemur/utils.py index 909d959a..d49a2feb 100644 --- a/lemur/utils.py +++ b/lemur/utils.py @@ -81,7 +81,7 @@ class Vault(types.TypeDecorator): """ # required by SQLAlchemy. defines the underlying column type - impl = types.Binary + impl = types.LargeBinary def process_bind_param(self, value, dialect): """ From 825a001a8b1a49428aeb62626022453992a003cf Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 16:47:40 -0800 Subject: [PATCH 30/49] pass algorithm to jwt.decode() during login api_jwt.py : pass "algorithms" argument when calling decode(). This argument will be mandatory in a future version --- lemur/auth/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/auth/service.py b/lemur/auth/service.py index 0e1521b3..f954ce51 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -101,7 +101,8 @@ def login_required(f): return dict(message="Token is invalid"), 403 try: - payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"]) + header_data = fetch_token_header(token) + payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"], algorithms=[header_data["alg"]]) except jwt.DecodeError: return dict(message="Token is invalid"), 403 except jwt.ExpiredSignatureError: From 6922d348257a2806bb10b8f45b0d4c241b922bd7 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:16:15 -0800 Subject: [PATCH 31/49] invalid escape sequence \ --- lemur/notifications/service.py | 4 ++-- lemur/plugins/bases/notification.py | 2 +- lemur/plugins/lemur_email/plugin.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 34edccc0..5bc5f3e1 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -43,7 +43,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): "name": "recipients", "type": "str", "required": True, - "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", + "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", "helpMessage": "Comma delimited list of email addresses", "value": ",".join(recipients), }, @@ -63,7 +63,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): "name": "interval", "type": "int", "required": True, - "validation": "^\d+$", + "validation": r"^\d+$", "helpMessage": "Number of days to be alert before expiration.", "value": i, } diff --git a/lemur/plugins/bases/notification.py b/lemur/plugins/bases/notification.py index 0da0dad2..76aa33de 100644 --- a/lemur/plugins/bases/notification.py +++ b/lemur/plugins/bases/notification.py @@ -42,7 +42,7 @@ class ExpirationNotificationPlugin(NotificationPlugin): "name": "interval", "type": "int", "required": True, - "validation": "^\d+$", + "validation": r"^\d+$", "helpMessage": "Number of days to be alert before expiration.", }, { diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index f380c82e..041b27ec 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -91,7 +91,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): "name": "recipients", "type": "str", "required": True, - "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", + "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", "helpMessage": "Comma delimited list of email addresses", } ] From b75bd56546b9d2c525a3e6f337ca9f54969045db Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:29:22 -0800 Subject: [PATCH 32/49] Check if ValueError assert works old way --- lemur/plugins/lemur_acme/tests/test_acme.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 39600eb2..6a81d85d 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -130,8 +130,9 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge = [] dns_challenge = Mock() mock_authz.dns_challenge.append(dns_challenge) - with self.assertRaises(ValueError): - self.acme.complete_dns_challenge(mock_acme, mock_authz) + # with self.assertRaises(ValueError): + # self.acme.complete_dns_challenge(mock_acme, mock_authz) + self.assertRaises(ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz)) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From 86b2cfbe4add04692b7d24e2068acfea9f4382b2 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:45:38 -0800 Subject: [PATCH 33/49] invalid escape sequence \ --- lemur/plugins/lemur_acme/plugin.py | 4 ++-- lemur/plugins/lemur_sftp/plugin.py | 2 +- lemur/plugins/lemur_slack/plugin.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e0e5b495..1835971b 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -481,7 +481,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "name": "acme_url", "type": "str", "required": True, - "validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", + "validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", "helpMessage": "Must be a valid web url starting with http[s]://", }, { @@ -494,7 +494,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "name": "email", "type": "str", "default": "", - "validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", + "validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", "helpMessage": "Email to use", }, { diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 66784048..2447cc4e 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -47,7 +47,7 @@ class SFTPDestinationPlugin(DestinationPlugin): "type": "int", "required": True, "helpMessage": "The SFTP port, default is 22.", - "validation": "^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})", + "validation": r"^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})", "default": "22", }, { diff --git a/lemur/plugins/lemur_slack/plugin.py b/lemur/plugins/lemur_slack/plugin.py index 70d97aa5..3ad22bca 100644 --- a/lemur/plugins/lemur_slack/plugin.py +++ b/lemur/plugins/lemur_slack/plugin.py @@ -89,7 +89,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): "name": "webhook", "type": "str", "required": True, - "validation": "^https:\/\/hooks\.slack\.com\/services\/.+$", + "validation": r"^https:\/\/hooks\.slack\.com\/services\/.+$", "helpMessage": "The url Slack told you to use for this integration", }, { From 3d64aa8d114b81ab933f9c67a161114795c3e1f2 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:58:38 -0800 Subject: [PATCH 34/49] Fixing DeprecationWarning: callable is None: another syntax --- lemur/plugins/lemur_acme/tests/test_acme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 6a81d85d..cbb4d314 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -132,7 +132,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.append(dns_challenge) # with self.assertRaises(ValueError): # self.acme.complete_dns_challenge(mock_acme, mock_authz) - self.assertRaises(ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz)) + self.assertRaises(ValueError, self.acme.complete_dns_challenge, mock_acme, mock_authz) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From dc7497e29d66214d88b358ae6781bf67fe4aae64 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 3 Nov 2020 19:05:18 -0800 Subject: [PATCH 35/49] Fix Working outside of application context Test Failures in dev --- lemur/plugins/lemur_acme/tests/test_acme.py | 10 ++++++++++ lemur/plugins/lemur_acme/tests/test_powerdns.py | 11 +++++++++++ lemur/plugins/lemur_acme/tests/test_ultradns.py | 10 ++++++++++ 3 files changed, 31 insertions(+) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index cbb4d314..0e7ed9e3 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -3,6 +3,7 @@ from unittest.mock import patch, Mock import josepy as jose from cryptography.x509 import DNSName +from flask import Flask from lemur.plugins.lemur_acme import plugin from lemur.common.utils import generate_private_key from mock import MagicMock @@ -22,6 +23,15 @@ class TestAcme(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) def test_get_dns_challenges(self, mock_len): assert mock_len diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 37e4968e..ca344d8d 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -1,5 +1,7 @@ import unittest from unittest.mock import patch, Mock + +from flask import Flask from lemur.plugins.lemur_acme import plugin, powerdns @@ -17,6 +19,15 @@ class TestPowerdns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.powerdns.current_app") def test_get_zones(self, mock_current_app): account_number = "1234567890" diff --git a/lemur/plugins/lemur_acme/tests/test_ultradns.py b/lemur/plugins/lemur_acme/tests/test_ultradns.py index f1d61e68..fa695a95 100644 --- a/lemur/plugins/lemur_acme/tests/test_ultradns.py +++ b/lemur/plugins/lemur_acme/tests/test_ultradns.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, Mock +from flask import Flask from lemur.plugins.lemur_acme import plugin, ultradns from requests.models import Response @@ -19,6 +20,15 @@ class TestUltradns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.ultradns.requests") @patch("lemur.plugins.lemur_acme.ultradns.current_app") def test_ultradns_get_token(self, mock_current_app, mock_requests): From 003779a1126f6fff16812ed9f306573fb89c9cdf Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 3 Nov 2020 19:27:41 -0800 Subject: [PATCH 36/49] Mock fix for DeprecationWarning: callable is None --- lemur/plugins/lemur_acme/tests/test_acme.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 0e7ed9e3..29148002 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -127,22 +127,24 @@ class TestAcme(unittest.TestCase): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) + mock_dns_challenge = Mock() + response = Mock() + response.simple_verify = Mock(return_value=False) + mock_dns_challenge.response = Mock(return_value=response) + mock_authz = Mock() - mock_authz.dns_challenge.response = Mock() - mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False) - mock_authz.authz = [] + mock_authz.dns_challenge = [] + mock_authz.dns_challenge.append(mock_dns_challenge) + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" + mock_authz.authz = [] mock_authz.authz.append(mock_authz_record) mock_authz.change_id = [] mock_authz.change_id.append("123") - mock_authz.dns_challenge = [] - dns_challenge = Mock() - mock_authz.dns_challenge.append(dns_challenge) - # with self.assertRaises(ValueError): - # self.acme.complete_dns_challenge(mock_acme, mock_authz) - self.assertRaises(ValueError, self.acme.complete_dns_challenge, mock_acme, mock_authz) + with self.assertRaises(ValueError): + self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From ab014873d0a53a8d4fc88c578c1a4f32ac67dd51 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 3 Nov 2020 19:33:13 -0800 Subject: [PATCH 37/49] invalid escape sequence warning for not an escape char --- lemur/tests/test_dns_providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/tests/test_dns_providers.py b/lemur/tests/test_dns_providers.py index 83315be5..9b8fdb5a 100644 --- a/lemur/tests/test_dns_providers.py +++ b/lemur/tests/test_dns_providers.py @@ -13,7 +13,7 @@ class TestDNSProvider(unittest.TestCase): self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com')) self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com')) self.assertFalse(dnsutil.is_valid_domain('e/xample.com')) - self.assertFalse(dnsutil.is_valid_domain('exam\ple.com')) + self.assertFalse(dnsutil.is_valid_domain('exam\\ple.com')) self.assertFalse(dnsutil.is_valid_domain(' Date: Wed, 4 Nov 2020 10:53:27 -0800 Subject: [PATCH 38/49] Stop repeating certs when sending expiration notifications to security team email --- lemur/notifications/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 3928689e..3fa339d2 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -137,11 +137,11 @@ def send_expiration_notifications(exclude): # security team gets all security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") - security_data = [] for owner, notification_group in get_eligible_certificates(exclude=exclude).items(): for notification_label, certificates in notification_group.items(): notification_data = [] + security_data = [] notification = certificates[0][0] From 7d2ce61303a9b6e77f6012908706cb79f1a6c278 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 4 Nov 2020 18:03:43 -0800 Subject: [PATCH 39/49] Updating comment for application context --- lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++- lemur/plugins/lemur_acme/tests/test_powerdns.py | 3 ++- lemur/plugins/lemur_acme/tests/test_ultradns.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 29148002..4ee56396 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -23,7 +23,8 @@ class TestAcme(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } - # Creates a new Flask application for a test duration. + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. _app = Flask('lemur_test_acme') self.ctx = _app.app_context() assert self.ctx diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index ca344d8d..cf850970 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -19,7 +19,8 @@ class TestPowerdns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } - # Creates a new Flask application for a test duration. + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. _app = Flask('lemur_test_acme') self.ctx = _app.app_context() assert self.ctx diff --git a/lemur/plugins/lemur_acme/tests/test_ultradns.py b/lemur/plugins/lemur_acme/tests/test_ultradns.py index fa695a95..7616459e 100644 --- a/lemur/plugins/lemur_acme/tests/test_ultradns.py +++ b/lemur/plugins/lemur_acme/tests/test_ultradns.py @@ -20,7 +20,8 @@ class TestUltradns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } - # Creates a new Flask application for a test duration. + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. _app = Flask('lemur_test_acme') self.ctx = _app.app_context() assert self.ctx From 206d010c9a01e48894e22d31d83cc208bdacc83d Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 4 Nov 2020 18:23:39 -0800 Subject: [PATCH 40/49] Version updates and making lint happy --- lemur/certificates/service.py | 9 +++++---- lemur/certificates/utils.py | 2 +- lemur/notifications/messaging.py | 9 +++++---- requirements-dev.txt | 6 +++--- requirements.txt | 6 +++--- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 167425cc..ac844120 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -12,6 +12,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from flask import current_app from sqlalchemy import func, or_, not_, cast, Integer +from sqlalchemy.sql.expression import false, true from lemur import database from lemur.authorities.models import Authority @@ -150,7 +151,7 @@ def get_all_certs_attached_to_endpoint_without_autorotate(): """ return ( Certificate.query.filter(Certificate.endpoints.any()) - .filter(Certificate.rotation == False) + .filter(Certificate.rotation == false()) .filter(Certificate.not_after >= arrow.now()) .filter(not_(Certificate.replaced.any())) .all() # noqa @@ -205,9 +206,9 @@ def get_all_pending_reissue(): :return: """ return ( - Certificate.query.filter(Certificate.rotation == True) + Certificate.query.filter(Certificate.rotation == true()) .filter(not_(Certificate.replaced.any())) - .filter(Certificate.in_rotation_window == True) + .filter(Certificate.in_rotation_window == true()) .all() ) # noqa @@ -525,7 +526,7 @@ def render(args): ) if current_app.config.get("ALLOW_CERT_DELETION", False): - query = query.filter(Certificate.deleted == False) # noqa + query = query.filter(Certificate.deleted == false()) result = database.sort_and_page(query, Certificate, args) return result diff --git a/lemur/certificates/utils.py b/lemur/certificates/utils.py index e642e058..18f1c967 100644 --- a/lemur/certificates/utils.py +++ b/lemur/certificates/utils.py @@ -82,4 +82,4 @@ def get_key_type_from_csr(data): raise Exception("Unsupported key type") except NotImplemented: - raise NotImplemented() + raise NotImplementedError diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 3fa339d2..75d227b1 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -16,6 +16,7 @@ from itertools import groupby import arrow from flask import current_app from sqlalchemy import and_ +from sqlalchemy.sql.expression import false, true from lemur import database from lemur.certificates.models import Certificate @@ -40,10 +41,10 @@ def get_certificates(exclude=None): q = ( database.db.session.query(Certificate) .filter(Certificate.not_after <= max) - .filter(Certificate.notify == True) - .filter(Certificate.expired == False) - .filter(Certificate.revoked == False) - ) # noqa + .filter(Certificate.notify == true()) + .filter(Certificate.expired == false()) + .filter(Certificate.revoked == false()) + ) exclude_conditions = [] if exclude: diff --git a/requirements-dev.txt b/requirements-dev.txt index 180be728..5b4c4fe4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,7 +15,7 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv -flake8==3.5.0 # via -r requirements-dev.in +flake8==3.8.0 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests invoke==1.4.1 # via -r requirements-dev.in @@ -25,9 +25,9 @@ mccabe==0.6.1 # via flake8 nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine pre-commit==2.8.2 # via -r requirements-dev.in -pycodestyle==2.3.1 # via flake8 +pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi -pyflakes==1.6.0 # via flake8 +pyflakes==2.2.0 # via flake8 pygments==2.6.1 # via readme-renderer pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit readme-renderer==25.0 # via twine diff --git a/requirements.txt b/requirements.txt index 4e641cfd..ceaafc85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,14 +15,14 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.9 # via -r requirements.in -botocore==1.19.9 # via -r requirements.in, boto3, s3transfer +boto3==1.16.10 # via -r requirements.in +botocore==1.19.10 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.1.1 # via flask +click==7.1.2 # black 20.8b1 has requirement click>=7.1.2 cloudflare==2.8.13 # via -r requirements.in cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in From 320667935df6fe74b09e6bf04f3d5ac3ecebc451 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 4 Nov 2020 19:09:34 -0800 Subject: [PATCH 41/49] flake8 version 3.8.4 --- requirements-dev.in | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index 2ffc5488..53a7586c 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,6 @@ # Run `make up-reqs` to update pinned dependencies in requirement text files -flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors. +flake8==3.8.4 # flake8 latest version pre-commit invoke twine diff --git a/requirements-dev.txt b/requirements-dev.txt index 5b4c4fe4..1a9980b7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv -flake8==3.8.0 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests +flake8==3.8.4 # via -r requirements-dev.in invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine From 519411b309dd239b3adbe59988cbcf8a5141e851 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 6 Nov 2020 22:40:55 -0800 Subject: [PATCH 42/49] regex --- lemur/acme_providers/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 313876e6..7efa196e 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -144,7 +144,7 @@ def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name) "value": bucket_name, "type": "str", "required": True, - "validation": "[0-9a-z.-]{3,63}", + "validation": r"[0-9a-z.-]{3,63}", "helpMessage": "Must be a valid S3 bucket name!", }, { @@ -152,7 +152,7 @@ def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name) "type": "str", "value": account_number, "required": True, - "validation": "[0-9]{12}", + "validation": r"[0-9]{12}", "helpMessage": "A valid AWS account number with permission to access S3", }, { From 7c779d6283be0cfe7e2571ee3264df2989fbdee9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 6 Nov 2020 22:41:48 -0800 Subject: [PATCH 43/49] regex --- lemur/plugins/lemur_aws/tests/test_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index a3227296..be9b14fd 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -28,7 +28,7 @@ def test_upload_acme_token(app): "value": bucket, "type": "str", "required": True, - "validation": "[0-9a-z.-]{3,63}", + "validation": r"[0-9a-z.-]{3,63}", "helpMessage": "Must be a valid S3 bucket name!", }, { @@ -36,7 +36,7 @@ def test_upload_acme_token(app): "type": "str", "value": account, "required": True, - "validation": "[0-9]{12}", + "validation": r"[0-9]{12}", "helpMessage": "A valid AWS account number with permission to access S3", }, { From 57208fe198e9ba17661bbf4c40ccec937623daeb Mon Sep 17 00:00:00 2001 From: Frederic Brin Date: Mon, 9 Nov 2020 09:40:28 +0100 Subject: [PATCH 44/49] Fix group lookup when AD DNS Referal is in lookup path Fix an issue when the DNS AD referal is in the path. An Exception is raised, with the following stacktrace:: Traceback (most recent call last): File "/www/lemur/lemur/auth/views.py", line 317, in post user = ldap_principal.authenticate() File "/www/lemur/lemur/auth/ldap.py", line 147, in authenticate self._bind() File "/www/lemur/lemur/auth/ldap.py", line 216, in _bind self.ldap_groups.append(values["cn"][0].decode("ascii")) TypeError: list indices must be integers or slices, not str This is issue is trigerred by some extra rows that referrences the DNS subtree:: ['ldaps://DomainDnsZones.xxxx'] Limiting the extraction to the expected dicts fix this issue. --- lemur/auth/ldap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index ed87b76c..030c7c78 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -210,7 +210,8 @@ class LdapPrincipal: self.ldap_groups = [] for group in lgroups: (dn, values) = group - self.ldap_groups.append(values["cn"][0].decode("ascii")) + if type(values) == dict: + self.ldap_groups.append(values["cn"][0].decode("ascii")) else: lgroups = self.ldap_client.search_s( self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs From d3e8921731c51a7701344f343b2748d869eda0fb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:20:48 +0000 Subject: [PATCH 45/49] Bump pytest-flask from 1.0.0 to 1.1.0 Bumps [pytest-flask](https://github.com/pytest-dev/pytest-flask) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/pytest-dev/pytest-flask/releases) - [Changelog](https://github.com/pytest-dev/pytest-flask/blob/master/docs/changelog.rst) - [Commits](https://github.com/pytest-dev/pytest-flask/compare/1.0.0...1.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index fcc219e9..a7c746dc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -59,7 +59,7 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema -pytest-flask==1.0.0 # via -r requirements-tests.in +pytest-flask==1.1.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto From a74b8aed152b1b22398621484d5a46084dff5e75 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:29:51 +0000 Subject: [PATCH 46/49] Bump faker from 4.14.0 to 4.14.2 Bumps [faker](https://github.com/joke2k/faker) from 4.14.0 to 4.14.2. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.14.0...v4.14.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index a7c746dc..a7e5a479 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in -faker==4.14.0 # via -r requirements-tests.in, factory-boy +faker==4.14.2 # via -r requirements-tests.in, factory-boy fakeredis==1.4.4 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in From 4c6645ca04fa43b7ac4c92a7b0fb030842e94897 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:40:24 +0000 Subject: [PATCH 47/49] Bump certifi from 2020.6.20 to 2020.11.8 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.6.20 to 2020.11.8. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.06.20...2020.11.08) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 8 ++++---- requirements-tests.txt | 2 +- requirements.txt | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1a9980b7..e2eb7051 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # appdirs==1.4.3 # via virtualenv bleach==3.1.4 # via readme-renderer -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests @@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv +flake8==3.8.4 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests -flake8==3.8.4 # via -r requirements-dev.in invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index d0f5a47c..f9d4cbfa 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,14 +17,14 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.9 # via -r requirements.txt -botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.10 # via -r requirements.txt +botocore==1.19.10 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt -certifi==2020.6.20 # via -r requirements.txt, requests +certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements.txt, requests -click==7.1.1 # via -r requirements.txt, flask +click==7.1.2 # via -r requirements.txt, flask cloudflare==2.8.13 # via -r requirements.txt cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index a7e5a479..a91aaa30 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -13,7 +13,7 @@ black==20.8b1 # via -r requirements-tests.in boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests diff --git a/requirements.txt b/requirements.txt index ceaafc85..3c1a8aba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,11 +18,11 @@ blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.in botocore==1.19.10 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in -certifi==2020.6.20 # via -r requirements.in, requests +certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.1.2 # black 20.8b1 has requirement click>=7.1.2 +click==7.1.2 # via flask cloudflare==2.8.13 # via -r requirements.in cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in From 7ec2860f886be6986c3f1823d883f13250d8c6dd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:49:57 +0000 Subject: [PATCH 48/49] Bump botocore from 1.19.9 to 1.19.14 Bumps [botocore](https://github.com/boto/botocore) from 1.19.9 to 1.19.14. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.19.9...1.19.14) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index f9d4cbfa..e9fd1566 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,7 +18,7 @@ beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.txt -botocore==1.19.10 # via -r requirements.txt, boto3, s3transfer +botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index a91aaa30..d055da83 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 3c1a8aba..73bc348e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.in -botocore==1.19.10 # via -r requirements.in, boto3, s3transfer +botocore==1.19.14 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 65d9ac6a0f83c1bc447a6335d64ee0c0fa6ee016 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 22:22:52 +0000 Subject: [PATCH 49/49] Bump boto3 from 1.16.9 to 1.16.14 Bumps [boto3](https://github.com/boto/boto3) from 1.16.9 to 1.16.14. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.9...1.16.14) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index e9fd1566..1fcf06ab 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,7 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.10 # via -r requirements.txt +boto3==1.16.14 # via -r requirements.txt botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests diff --git a/requirements-tests.txt b/requirements-tests.txt index d055da83..b82e2ac8 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.9 # via aws-sam-translator, moto +boto3==1.16.14 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests diff --git a/requirements.txt b/requirements.txt index 73bc348e..d7b56f2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.10 # via -r requirements.in +boto3==1.16.14 # via -r requirements.in botocore==1.19.14 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests