diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 310efad1..7efa196e 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -1,12 +1,15 @@ import time import json +import arrow from flask_script import Manager from flask import current_app from lemur.extensions import sentry from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler +from lemur.plugins.lemur_aws import s3 manager = Manager( usage="Handles all ACME related tasks" @@ -84,3 +87,105 @@ def dnstest(domain, token): status = SUCCESS_METRIC_STATUS print("[+] Done with ACME Tests.") + + +@manager.option( + "-t", + "--token", + dest="token", + default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="Value of the Token", +) +@manager.option( + "-n", + "--token_name", + dest="token_name", + default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="path", +) +@manager.option( + "-p", + "--prefix", + dest="prefix", + default="test/", + required=False, + help="S3 bucket prefix", +) +@manager.option( + "-a", + "--account_number", + dest="account_number", + required=True, + help="AWS Account", +) +@manager.option( + "-b", + "--bucket_name", + dest="bucket_name", + required=True, + help="Bucket Name", +) +def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): + """ + This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it. + It mainly serves for testing purposes. + :param token: + :param token_name: + :param prefix: + :param account_number: + :param bucket_name: + :return: + """ + additional_options = [ + { + "name": "bucket", + "value": bucket_name, + "type": "str", + "required": True, + "validation": r"[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account_number, + "required": True, + "validation": r"[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + p = plugins.get("aws-s3") + p.upload_acme_token(token_name, token, additional_options) + + if not prefix.endswith("/"): + prefix + "/" + + token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number) + assert(token_res == token) + s3.delete(bucket_name, prefix + token_name, account_number=account_number) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 1be641b0..b54787ac 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -33,6 +33,7 @@ .. moduleauthor:: Harm Weites """ +import sys from acme.errors import ClientError from flask import current_app @@ -408,6 +409,47 @@ class S3DestinationPlugin(ExportDestinationPlugin): account_number=self.get_option("accountNumber", options), ) + def upload_acme_token(self, token_path, token, options, **kwargs): + """ + This is called from the acme http challenge + :param self: + :param token_path: + :param token: + :param options: + :param kwargs: + :return: + """ + current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge") + + function = f"{__name__}.{sys._getframe().f_code.co_name}" + + account_number = self.get_option("accountNumber", options) + bucket_name = self.get_option("bucket", options) + prefix = self.get_option("prefix", options) + region = self.get_option("region", options) + filename = token_path.split("/")[-1] + if not prefix.endswith("/"): + prefix + "/" + + res = s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) + res = "Success" if res else "Failure" + log_data = { + "function": function, + "message": "check if any valid certificate is revoked", + "result": res, + "bucket_name": bucket_name, + "filename": filename + } + current_app.logger.info(log_data) + metrics.send(f"{function}", "counter", 1, metric_tags={"result": res, + "bucket_name": bucket_name, + "filename": filename}) + class SNSNotificationPlugin(ExpirationNotificationPlugin): title = "AWS SNS" diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 43faa28f..1b0831b3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -6,12 +6,15 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from botocore.exceptions import ClientError from flask import current_app +from lemur.extensions import sentry + from .sts import sts_client @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ @@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs): ServerSideEncryption="AES256", ) else: - bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + try: + bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + return True + except ClientError: + sentry.captureException() + return False + + +@sts_client("s3", service_type="client") +def delete(bucket_name, prefixed_object_name, **kwargs): + """ + Use STS to delete an object + """ + try: + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name) + current_app.logger.debug(f"Delete data from S3." + f"Bucket: {bucket_name}," + f"Prefix: {prefixed_object_name}," + f"Status_code: {response}") + return response['ResponseMetadata']['HTTPStatusCode'] < 300 + except ClientError: + sentry.captureException() + return False + + +@sts_client("s3", service_type="client") +def get(bucket_name, prefixed_object_name, **kwargs): + """ + Use STS to get an object + """ + try: + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name) + current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," + f"object_name: {prefixed_object_name}") + return response['Body'].read().decode("utf-8") + except ClientError: + sentry.captureException() + return None diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index dbad7b02..be9b14fd 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -1,5 +1,82 @@ +import boto3 +from moto import mock_sts, mock_s3 + + def test_get_certificates(app): from lemur.plugins.base import plugins p = plugins.get("aws-s3") assert p + + +@mock_sts() +@mock_s3() +def test_upload_acme_token(app): + from lemur.plugins.base import plugins + from lemur.plugins.lemur_aws.s3 import get + + bucket = "public-bucket" + account = "123456789012" + prefix = "some-path/more-path/" + token_content = "Challenge" + token_name = "TOKEN" + token_path = ".well-known/acme-challenge/" + token_name + + additional_options = [ + { + "name": "bucket", + "value": bucket, + "type": "str", + "required": True, + "validation": r"[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account, + "required": True, + "validation": r"[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + p = plugins.get("aws-s3") + + p.upload_acme_token(token_path=token_path, + token_content=token_content, + token=token_content, + options=additional_options) + + response = get(bucket_name=bucket, + prefixed_object_name=prefix + token_name, + encrypt=False, + account_number=account) + + # put data, and getting the same data + assert (response == token_content) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py new file mode 100644 index 00000000..7d0fa843 --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -0,0 +1,41 @@ +import boto3 +from moto import mock_sts, mock_s3 + + +@mock_sts() +@mock_s3() +def test_put_delete_s3_object(app): + from lemur.plugins.lemur_aws.s3 import put, delete, get + + bucket = "public-bucket" + region = "us-east-1" + account = "123456789012" + path = "some-path/foo" + data = "dummy data" + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + + put(bucket_name=bucket, + region_name=region, + prefix=path, + data=data, + encrypt=False, + account_number=account, + region=region) + + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) + + # put data, and getting the same data + assert (response == data) + + response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account) + + # attempting to get thccle wrong data + assert (response is None) + + delete(bucket_name=bucket, prefixed_object_name=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) + + # delete data, and getting the same data + assert (response is None) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1a9980b7..e2eb7051 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # appdirs==1.4.3 # via virtualenv bleach==3.1.4 # via readme-renderer -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests @@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv +flake8==3.8.4 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests -flake8==3.8.4 # via -r requirements-dev.in invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index d0f5a47c..1fcf06ab 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,14 +17,14 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.9 # via -r requirements.txt -botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.14 # via -r requirements.txt +botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt -certifi==2020.6.20 # via -r requirements.txt, requests +certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements.txt, requests -click==7.1.1 # via -r requirements.txt, flask +click==7.1.2 # via -r requirements.txt, flask cloudflare==2.8.13 # via -r requirements.txt cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index fcc219e9..b82e2ac8 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,10 +10,10 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.9 # via aws-sam-translator, moto +boto3==1.16.14 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2020.6.20 # via requests +botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests @@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in -faker==4.14.0 # via -r requirements-tests.in, factory-boy +faker==4.14.2 # via -r requirements-tests.in, factory-boy fakeredis==1.4.4 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in @@ -59,7 +59,7 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema -pytest-flask==1.0.0 # via -r requirements-tests.in +pytest-flask==1.1.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto diff --git a/requirements.txt b/requirements.txt index ceaafc85..d7b56f2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,14 +15,14 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.10 # via -r requirements.in -botocore==1.19.10 # via -r requirements.in, boto3, s3transfer +boto3==1.16.14 # via -r requirements.in +botocore==1.19.14 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in -certifi==2020.6.20 # via -r requirements.in, requests +certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.1.2 # black 20.8b1 has requirement click>=7.1.2 +click==7.1.2 # via flask cloudflare==2.8.13 # via -r requirements.in cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in