From bfe89e131e71115a779359dd9a389ba8ec306bdb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 15 Oct 2020 18:13:50 -0700 Subject: [PATCH 01/24] adding delete and put interfaces for the S3 plugin --- lemur/plugins/lemur_aws/s3.py | 40 ++++++++++++++++++++++-- lemur/plugins/lemur_aws/tests/test_s3.py | 38 ++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 lemur/plugins/lemur_aws/tests/test_s3.py diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 43faa28f..c868c7a3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -9,9 +9,12 @@ from flask import current_app from .sts import sts_client +from botocore.exceptions import ClientError +from lemur.extensions import sentry + @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ @@ -32,4 +35,37 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs): ServerSideEncryption="AES256", ) else: - bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + try: + bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + except ClientError: + sentry.captureException() + + +@sts_client("s3", service_type="client") +def delete(bucket_name, prefix, **kwargs): + """ + Use STS to delete an object + """ + try: + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix) + current_app.logger.debug(f"Delete data from S3." + f"Bucket: {bucket_name}," + f"Prefix: {prefix}," + f"Status_code: {response}") + return response['ResponseMetadata']['HTTPStatusCode'] < 300 + except ClientError: + sentry.captureException() + + +@sts_client("s3", service_type="client") +def get(bucket_name, prefix, **kwargs): + """ + Use STS to get an object + """ + try: + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix) + current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," + f"Prefix: {prefix}") + return response['Body'].read().decode("utf-8") + except ClientError: + sentry.captureException() diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py new file mode 100644 index 00000000..f7a36496 --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -0,0 +1,38 @@ +import boto3 +from moto import mock_sts, mock_s3 + + +@mock_sts() +@mock_s3() +def test_put_delete_s3_object(app): + from lemur.plugins.lemur_aws.s3 import put, delete, get + + bucket = "public-bucket" + account = "123456789012" + path = "some_path/foo" + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + + data = "dummy data" + put(bucket_name=bucket, + prefix=path, + data=data, + encrypt=None, + account_number=account) + + response = get(bucket_name=bucket, prefix=path, account_number=account) + + # put data, and getting the same data + assert (response == data) + + response = get(bucket_name="wrong-bucket", prefix=path, account_number=account) + + # attempting to get thccle wrong data + assert (response is None) + + delete(bucket_name=bucket, prefix=path, account_number=account) + response = get(bucket_name=bucket, prefix=path, account_number=account) + + # delete data, and getting the same data + assert (response is None) From d73db59d2352ad54d230948cb4e323cbc46cd30b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:48:47 -0700 Subject: [PATCH 02/24] revsering removing region --- lemur/plugins/lemur_aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index c868c7a3..0e9db182 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -14,7 +14,7 @@ from lemur.extensions import sentry @sts_client("s3", service_type="resource") -def put(bucket_name, prefix, data, encrypt, **kwargs): +def put(bucket_name, region, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ From 6aad37e1f9d115da065c321d88e21b38045ae80c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:00 -0700 Subject: [PATCH 03/24] cleaning up code --- lemur/plugins/lemur_aws/s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 0e9db182..186b715d 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -6,12 +6,12 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from flask import current_app -from .sts import sts_client - from botocore.exceptions import ClientError +from flask import current_app from lemur.extensions import sentry +from .sts import sts_client + @sts_client("s3", service_type="resource") def put(bucket_name, region, prefix, data, encrypt, **kwargs): From 7d8eb1c61edba2656257fbe2c42f864ce9b87107 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:26 -0700 Subject: [PATCH 04/24] improving test --- lemur/plugins/lemur_aws/tests/test_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index f7a36496..7b8b4ac3 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -9,16 +9,16 @@ def test_put_delete_s3_object(app): bucket = "public-bucket" account = "123456789012" - path = "some_path/foo" + path = "some-path/foo" + data = "dummy data" s3_client = boto3.client('s3') s3_client.create_bucket(Bucket=bucket) - data = "dummy data" put(bucket_name=bucket, prefix=path, data=data, - encrypt=None, + encrypt=False, account_number=account) response = get(bucket_name=bucket, prefix=path, account_number=account) From d705e3ae3b5cce2fd8cfd34ebc2283217d180557 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:56 -0700 Subject: [PATCH 05/24] expanding the S3 destination plugin to support the acme token upload inteface --- lemur/plugins/lemur_aws/plugin.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8692348a..6d161ac3 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -406,3 +406,25 @@ class S3DestinationPlugin(ExportDestinationPlugin): self.get_option("encrypt", options), account_number=self.get_option("accountNumber", options), ) + + def upload_acme_token(self, token_path, token, options, **kwargs): + """ + This is called from the acme http challenge + :param self: + :param token_path: + :param token: + :param options: + :param kwargs: + :return: + """ + current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge") + + account_number = self.get_option("accountNumber", options) + bucket_name = self.get_option("bucket", options) + prefix = self.get_option("prefix", options) + region = self.get_option("region", options) + filename = token_path.split("/")[-1] + if not prefix.endswith("/"): + prefix + "/" + + s3.put(bucket_name, region, prefix + filename, token, encrypt=False, account_number=account_number) From 17e528b5dd56284b097f85bb2f6d1e895ac7bfc6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:50:35 -0700 Subject: [PATCH 06/24] adding testing for acme_upload method --- lemur/plugins/lemur_aws/tests/test_plugin.py | 77 ++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index dbad7b02..a471f7c8 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -1,5 +1,82 @@ +import boto3 +from moto import mock_sts, mock_s3 + + def test_get_certificates(app): from lemur.plugins.base import plugins p = plugins.get("aws-s3") assert p + + +@mock_sts() +@mock_s3() +def test_upload_acme_token(app): + from lemur.plugins.base import plugins + from lemur.plugins.lemur_aws.s3 import get + + bucket = "public-bucket" + account = "123456789012" + prefix = "some-path/more-path/" + token_content = "Challenge" + token_name = "TOKEN" + token_path = ".well-known/acme-challenge/" + token_name + + additional_options = [ + { + "name": "bucket", + "value": bucket, + "type": "str", + "required": True, + "validation": "[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account, + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + p = plugins.get("aws-s3") + + p.upload_acme_token(token_path=token_path, + token_content=token_content, + token=token_content, + options=additional_options) + + response = get(bucket_name=bucket, + prefix=prefix + token_name, + encrypt=False, + account_number=account) + + # put data, and getting the same data + assert (response == token_content) From 9c04a888d8122f100aefe646137cb8c0908ae489 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:52:04 -0700 Subject: [PATCH 07/24] adjusting the S3 test --- lemur/plugins/lemur_aws/tests/test_s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index 7b8b4ac3..bfb5a9f9 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -16,6 +16,7 @@ def test_put_delete_s3_object(app): s3_client.create_bucket(Bucket=bucket) put(bucket_name=bucket, + region=None, prefix=path, data=data, encrypt=False, From 11ce540246676b9441fde40b198d8ca0ea7ac2a7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 10:31:19 -0700 Subject: [PATCH 08/24] formatting --- lemur/plugins/lemur_aws/plugin.py | 7 ++++++- lemur/plugins/lemur_aws/s3.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 6d161ac3..ad80d87f 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -427,4 +427,9 @@ class S3DestinationPlugin(ExportDestinationPlugin): if not prefix.endswith("/"): prefix + "/" - s3.put(bucket_name, region, prefix + filename, token, encrypt=False, account_number=account_number) + s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 186b715d..7c4177ff 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -14,7 +14,7 @@ from .sts import sts_client @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ From 503530e93512291985d5d865b9b7af55d2e05488 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 10:32:10 -0700 Subject: [PATCH 09/24] the test requires region param for sts --- lemur/plugins/lemur_aws/tests/test_s3.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index bfb5a9f9..88bd30d2 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -8,6 +8,7 @@ def test_put_delete_s3_object(app): from lemur.plugins.lemur_aws.s3 import put, delete, get bucket = "public-bucket" + region = "us-east-1" account = "123456789012" path = "some-path/foo" data = "dummy data" @@ -16,11 +17,12 @@ def test_put_delete_s3_object(app): s3_client.create_bucket(Bucket=bucket) put(bucket_name=bucket, - region=None, + region_name=region, prefix=path, data=data, encrypt=False, - account_number=account) + account_number=account, + region=region) response = get(bucket_name=bucket, prefix=path, account_number=account) From c5769378cf74356f094af5587b6d0861e3df6798 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 15:21:22 -0700 Subject: [PATCH 10/24] making lint happy --- lemur/plugins/lemur_aws/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8c94dc45..489fa823 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -408,7 +408,6 @@ class S3DestinationPlugin(ExportDestinationPlugin): account_number=self.get_option("accountNumber", options), ) - def upload_acme_token(self, token_path, token, options, **kwargs): """ This is called from the acme http challenge @@ -483,4 +482,3 @@ class SNSNotificationPlugin(ExpirationNotificationPlugin): current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}") sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) - From ba8eb7a3f59e8201810f63e6a174fc2e3b0c700d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:17:02 -0700 Subject: [PATCH 11/24] better logging and metrics --- lemur/plugins/lemur_aws/plugin.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 489fa823..b54787ac 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -33,6 +33,7 @@ .. moduleauthor:: Harm Weites """ +import sys from acme.errors import ClientError from flask import current_app @@ -420,6 +421,8 @@ class S3DestinationPlugin(ExportDestinationPlugin): """ current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge") + function = f"{__name__}.{sys._getframe().f_code.co_name}" + account_number = self.get_option("accountNumber", options) bucket_name = self.get_option("bucket", options) prefix = self.get_option("prefix", options) @@ -428,12 +431,24 @@ class S3DestinationPlugin(ExportDestinationPlugin): if not prefix.endswith("/"): prefix + "/" - s3.put(bucket_name=bucket_name, - region_name=region, - prefix=prefix + filename, - data=token, - encrypt=False, - account_number=account_number) + res = s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) + res = "Success" if res else "Failure" + log_data = { + "function": function, + "message": "check if any valid certificate is revoked", + "result": res, + "bucket_name": bucket_name, + "filename": filename + } + current_app.logger.info(log_data) + metrics.send(f"{function}", "counter", 1, metric_tags={"result": res, + "bucket_name": bucket_name, + "filename": filename}) class SNSNotificationPlugin(ExpirationNotificationPlugin): From cc2aa5c1de131ec389df3ffd57059d0fd9ebc134 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:17:34 -0700 Subject: [PATCH 12/24] cli for live testing --- lemur/acme_providers/cli.py | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 310efad1..56301aae 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -1,12 +1,16 @@ import time import json +import arrow from flask_script import Manager from flask import current_app from lemur.extensions import sentry from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler +from lemur.plugins.lemur_aws import s3 +from lemur.utils import get_random_secret manager = Manager( usage="Handles all ACME related tasks" @@ -84,3 +88,96 @@ def dnstest(domain, token): status = SUCCESS_METRIC_STATUS print("[+] Done with ACME Tests.") + + +@manager.option( + "-t", + "--token", + dest="token", + default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="Value of the Token", +) +@manager.option( + "-n", + "--token_name", + dest="token_name", + default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="path", +) +@manager.option( + "-p", + "--prefix", + dest="prefix", + default="test/", + required=False, + help="S3 bucket prefix", +) +@manager.option( + "-a", + "--account_number", + dest="account_number", + required=True, + help="AWS Account", +) +@manager.option( + "-b", + "--bucket_name", + dest="bucket_name", + required=True, + help="Bucket Name", +) +def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): + + additional_options = [ + { + "name": "bucket", + "value": bucket_name, + "type": "str", + "required": True, + "validation": "[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account_number, + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + p = plugins.get("aws-s3") + p.upload_acme_token(token_name, token, additional_options) + + if not prefix.endswith("/"): + prefix + "/" + + token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number) + assert(token_res == token) + s3.delete(bucket_name, prefix + token_name, account_number=account_number) From e1ff89eb2d6c3e6fbc8149020c689c0a530a0675 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:18:14 -0700 Subject: [PATCH 13/24] better return arguments --- lemur/plugins/lemur_aws/s3.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 7c4177ff..e15f6b6e 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -37,8 +37,10 @@ def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): else: try: bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + return True except ClientError: sentry.captureException() + return False @sts_client("s3", service_type="client") @@ -55,6 +57,7 @@ def delete(bucket_name, prefix, **kwargs): return response['ResponseMetadata']['HTTPStatusCode'] < 300 except ClientError: sentry.captureException() + return False @sts_client("s3", service_type="client") @@ -69,3 +72,4 @@ def get(bucket_name, prefix, **kwargs): return response['Body'].read().decode("utf-8") except ClientError: sentry.captureException() + return None From add0960579e7e4bdfbf87a18cc0cf5b1afac4b30 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:18:37 -0700 Subject: [PATCH 14/24] more meaningful variable naming --- lemur/plugins/lemur_aws/s3.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index e15f6b6e..1b0831b3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -44,15 +44,15 @@ def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): @sts_client("s3", service_type="client") -def delete(bucket_name, prefix, **kwargs): +def delete(bucket_name, prefixed_object_name, **kwargs): """ Use STS to delete an object """ try: - response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix) + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name) current_app.logger.debug(f"Delete data from S3." f"Bucket: {bucket_name}," - f"Prefix: {prefix}," + f"Prefix: {prefixed_object_name}," f"Status_code: {response}") return response['ResponseMetadata']['HTTPStatusCode'] < 300 except ClientError: @@ -61,14 +61,14 @@ def delete(bucket_name, prefix, **kwargs): @sts_client("s3", service_type="client") -def get(bucket_name, prefix, **kwargs): +def get(bucket_name, prefixed_object_name, **kwargs): """ Use STS to get an object """ try: - response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix) + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name) current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," - f"Prefix: {prefix}") + f"object_name: {prefixed_object_name}") return response['Body'].read().decode("utf-8") except ClientError: sentry.captureException() From 3dfafa00218b384ef420cf48cbd4bbeece8743f7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:28:10 -0700 Subject: [PATCH 15/24] making lint happy --- lemur/acme_providers/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 56301aae..ec6326bd 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -10,7 +10,6 @@ from lemur.constants import SUCCESS_METRIC_STATUS from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler from lemur.plugins.lemur_aws import s3 -from lemur.utils import get_random_secret manager = Manager( usage="Handles all ACME related tasks" From 7bca42776b6e0d50825dd7d07222076b18ef32f6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:28:34 -0700 Subject: [PATCH 16/24] better comments --- lemur/acme_providers/cli.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index ec6326bd..313876e6 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -128,7 +128,16 @@ def dnstest(domain, token): help="Bucket Name", ) def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): - + """ + This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it. + It mainly serves for testing purposes. + :param token: + :param token_name: + :param prefix: + :param account_number: + :param bucket_name: + :return: + """ additional_options = [ { "name": "bucket", From 9c6856bcdd7bf93705e02a887bfef3df9cdab542 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:36:32 -0700 Subject: [PATCH 17/24] adjusting the tests to the better naming --- lemur/plugins/lemur_aws/tests/test_plugin.py | 2 +- lemur/plugins/lemur_aws/tests/test_s3.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index a471f7c8..a3227296 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -74,7 +74,7 @@ def test_upload_acme_token(app): options=additional_options) response = get(bucket_name=bucket, - prefix=prefix + token_name, + prefixed_object_name=prefix + token_name, encrypt=False, account_number=account) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index 88bd30d2..7d0fa843 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -24,18 +24,18 @@ def test_put_delete_s3_object(app): account_number=account, region=region) - response = get(bucket_name=bucket, prefix=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) # put data, and getting the same data assert (response == data) - response = get(bucket_name="wrong-bucket", prefix=path, account_number=account) + response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account) # attempting to get thccle wrong data assert (response is None) - delete(bucket_name=bucket, prefix=path, account_number=account) - response = get(bucket_name=bucket, prefix=path, account_number=account) + delete(bucket_name=bucket, prefixed_object_name=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) # delete data, and getting the same data assert (response is None) From 519411b309dd239b3adbe59988cbcf8a5141e851 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 6 Nov 2020 22:40:55 -0800 Subject: [PATCH 18/24] regex --- lemur/acme_providers/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 313876e6..7efa196e 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -144,7 +144,7 @@ def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name) "value": bucket_name, "type": "str", "required": True, - "validation": "[0-9a-z.-]{3,63}", + "validation": r"[0-9a-z.-]{3,63}", "helpMessage": "Must be a valid S3 bucket name!", }, { @@ -152,7 +152,7 @@ def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name) "type": "str", "value": account_number, "required": True, - "validation": "[0-9]{12}", + "validation": r"[0-9]{12}", "helpMessage": "A valid AWS account number with permission to access S3", }, { From 7c779d6283be0cfe7e2571ee3264df2989fbdee9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 6 Nov 2020 22:41:48 -0800 Subject: [PATCH 19/24] regex --- lemur/plugins/lemur_aws/tests/test_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index a3227296..be9b14fd 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -28,7 +28,7 @@ def test_upload_acme_token(app): "value": bucket, "type": "str", "required": True, - "validation": "[0-9a-z.-]{3,63}", + "validation": r"[0-9a-z.-]{3,63}", "helpMessage": "Must be a valid S3 bucket name!", }, { @@ -36,7 +36,7 @@ def test_upload_acme_token(app): "type": "str", "value": account, "required": True, - "validation": "[0-9]{12}", + "validation": r"[0-9]{12}", "helpMessage": "A valid AWS account number with permission to access S3", }, { From d3e8921731c51a7701344f343b2748d869eda0fb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:20:48 +0000 Subject: [PATCH 20/24] Bump pytest-flask from 1.0.0 to 1.1.0 Bumps [pytest-flask](https://github.com/pytest-dev/pytest-flask) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/pytest-dev/pytest-flask/releases) - [Changelog](https://github.com/pytest-dev/pytest-flask/blob/master/docs/changelog.rst) - [Commits](https://github.com/pytest-dev/pytest-flask/compare/1.0.0...1.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index fcc219e9..a7c746dc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -59,7 +59,7 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema -pytest-flask==1.0.0 # via -r requirements-tests.in +pytest-flask==1.1.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto From a74b8aed152b1b22398621484d5a46084dff5e75 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:29:51 +0000 Subject: [PATCH 21/24] Bump faker from 4.14.0 to 4.14.2 Bumps [faker](https://github.com/joke2k/faker) from 4.14.0 to 4.14.2. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.14.0...v4.14.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index a7c746dc..a7e5a479 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in -faker==4.14.0 # via -r requirements-tests.in, factory-boy +faker==4.14.2 # via -r requirements-tests.in, factory-boy fakeredis==1.4.4 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in From 4c6645ca04fa43b7ac4c92a7b0fb030842e94897 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:40:24 +0000 Subject: [PATCH 22/24] Bump certifi from 2020.6.20 to 2020.11.8 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.6.20 to 2020.11.8. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.06.20...2020.11.08) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 8 ++++---- requirements-tests.txt | 2 +- requirements.txt | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1a9980b7..e2eb7051 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # appdirs==1.4.3 # via virtualenv bleach==3.1.4 # via readme-renderer -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests @@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv +flake8==3.8.4 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests -flake8==3.8.4 # via -r requirements-dev.in invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index d0f5a47c..f9d4cbfa 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,14 +17,14 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.9 # via -r requirements.txt -botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.10 # via -r requirements.txt +botocore==1.19.10 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt -certifi==2020.6.20 # via -r requirements.txt, requests +certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements.txt, requests -click==7.1.1 # via -r requirements.txt, flask +click==7.1.2 # via -r requirements.txt, flask cloudflare==2.8.13 # via -r requirements.txt cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index a7e5a479..a91aaa30 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -13,7 +13,7 @@ black==20.8b1 # via -r requirements-tests.in boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests diff --git a/requirements.txt b/requirements.txt index ceaafc85..3c1a8aba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,11 +18,11 @@ blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.in botocore==1.19.10 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in -certifi==2020.6.20 # via -r requirements.in, requests +certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.1.2 # black 20.8b1 has requirement click>=7.1.2 +click==7.1.2 # via flask cloudflare==2.8.13 # via -r requirements.in cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in From 7ec2860f886be6986c3f1823d883f13250d8c6dd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:49:57 +0000 Subject: [PATCH 23/24] Bump botocore from 1.19.9 to 1.19.14 Bumps [botocore](https://github.com/boto/botocore) from 1.19.9 to 1.19.14. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.19.9...1.19.14) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index f9d4cbfa..e9fd1566 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,7 +18,7 @@ beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.txt -botocore==1.19.10 # via -r requirements.txt, boto3, s3transfer +botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index a91aaa30..d055da83 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 3c1a8aba..73bc348e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.in -botocore==1.19.10 # via -r requirements.in, boto3, s3transfer +botocore==1.19.14 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 65d9ac6a0f83c1bc447a6335d64ee0c0fa6ee016 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 22:22:52 +0000 Subject: [PATCH 24/24] Bump boto3 from 1.16.9 to 1.16.14 Bumps [boto3](https://github.com/boto/boto3) from 1.16.9 to 1.16.14. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.9...1.16.14) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index e9fd1566..1fcf06ab 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,7 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.10 # via -r requirements.txt +boto3==1.16.14 # via -r requirements.txt botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests diff --git a/requirements-tests.txt b/requirements-tests.txt index d055da83..b82e2ac8 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.9 # via aws-sam-translator, moto +boto3==1.16.14 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests diff --git a/requirements.txt b/requirements.txt index 73bc348e..d7b56f2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.10 # via -r requirements.in +boto3==1.16.14 # via -r requirements.in botocore==1.19.14 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests