Merge pull request #3192 from hosseinsh/expanding-S3-plugin

Expanding the S3 plugin with acme-http01-token support

lemur/acme_providers/cli.py

@@ -1,12 +1,15 @@
 import time
 import json
+import arrow
 
 from flask_script import Manager
 from flask import current_app
 
 from lemur.extensions import sentry
 from lemur.constants import SUCCESS_METRIC_STATUS
+from lemur.plugins import plugins
 from lemur.plugins.lemur_acme.plugin import AcmeHandler
+from lemur.plugins.lemur_aws import s3
 
 manager = Manager(
     usage="Handles all ACME related tasks"
@@ -84,3 +87,105 @@ def dnstest(domain, token):
 
     status = SUCCESS_METRIC_STATUS
     print("[+] Done with ACME Tests.")
+
+
+@manager.option(
+    "-t",
+    "--token",
+    dest="token",
+    default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="Value of the Token",
+)
+@manager.option(
+    "-n",
+    "--token_name",
+    dest="token_name",
+    default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
+    required=False,
+    help="path",
+)
+@manager.option(
+    "-p",
+    "--prefix",
+    dest="prefix",
+    default="test/",
+    required=False,
+    help="S3 bucket prefix",
+)
+@manager.option(
+    "-a",
+    "--account_number",
+    dest="account_number",
+    required=True,
+    help="AWS Account",
+)
+@manager.option(
+    "-b",
+    "--bucket_name",
+    dest="bucket_name",
+    required=True,
+    help="Bucket Name",
+)
+def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
+    """
+    This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
+    It mainly serves for testing purposes.
+    :param token:
+    :param token_name:
+    :param prefix:
+    :param account_number:
+    :param bucket_name:
+    :return:
+    """
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket_name,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account_number,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    p = plugins.get("aws-s3")
+    p.upload_acme_token(token_name, token, additional_options)
+
+    if not prefix.endswith("/"):
+        prefix + "/"
+
+    token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
+    assert(token_res == token)
+    s3.delete(bucket_name, prefix + token_name, account_number=account_number)

lemur/plugins/lemur_aws/plugin.py

@@ -33,6 +33,7 @@
 .. moduleauthor:: Harm Weites <harm@weites.com>
 """
 
+import sys
 from acme.errors import ClientError
 from flask import current_app
 
@@ -408,6 +409,47 @@ class S3DestinationPlugin(ExportDestinationPlugin):
                 account_number=self.get_option("accountNumber", options),
             )
 
+    def upload_acme_token(self, token_path, token, options, **kwargs):
+        """
+         This is called from the acme http challenge
+        :param self:
+        :param token_path:
+        :param token:
+        :param options:
+        :param kwargs:
+        :return:
+        """
+        current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
+
+        function = f"{__name__}.{sys._getframe().f_code.co_name}"
+
+        account_number = self.get_option("accountNumber", options)
+        bucket_name = self.get_option("bucket", options)
+        prefix = self.get_option("prefix", options)
+        region = self.get_option("region", options)
+        filename = token_path.split("/")[-1]
+        if not prefix.endswith("/"):
+            prefix + "/"
+
+        res = s3.put(bucket_name=bucket_name,
+                     region_name=region,
+                     prefix=prefix + filename,
+                     data=token,
+                     encrypt=False,
+                     account_number=account_number)
+        res = "Success" if res else "Failure"
+        log_data = {
+            "function": function,
+            "message": "check if any valid certificate is revoked",
+            "result": res,
+            "bucket_name": bucket_name,
+            "filename": filename
+        }
+        current_app.logger.info(log_data)
+        metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
+                                                               "bucket_name": bucket_name,
+                                                               "filename": filename})
+
 
 class SNSNotificationPlugin(ExpirationNotificationPlugin):
     title = "AWS SNS"

lemur/plugins/lemur_aws/s3.py

@@ -6,12 +6,15 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
+from botocore.exceptions import ClientError
 from flask import current_app
+from lemur.extensions import sentry
+
 from .sts import sts_client
 
 
 @sts_client("s3", service_type="resource")
-def put(bucket_name, region, prefix, data, encrypt, **kwargs):
+def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
     """
     Use STS to write to an S3 bucket
     """
@@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs):
             ServerSideEncryption="AES256",
         )
     else:
+        try:
             bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
+            return True
+        except ClientError:
+            sentry.captureException()
+            return False
+
+
+@sts_client("s3", service_type="client")
+def delete(bucket_name, prefixed_object_name, **kwargs):
+    """
+    Use STS to delete an object
+    """
+    try:
+        response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name)
+        current_app.logger.debug(f"Delete data from S3."
+                                 f"Bucket: {bucket_name},"
+                                 f"Prefix: {prefixed_object_name},"
+                                 f"Status_code: {response}")
+        return response['ResponseMetadata']['HTTPStatusCode'] < 300
+    except ClientError:
+        sentry.captureException()
+        return False
+
+
+@sts_client("s3", service_type="client")
+def get(bucket_name, prefixed_object_name, **kwargs):
+    """
+    Use STS to get an object
+    """
+    try:
+        response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name)
+        current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
+                                 f"object_name: {prefixed_object_name}")
+        return response['Body'].read().decode("utf-8")
+    except ClientError:
+        sentry.captureException()
+        return None

lemur/plugins/lemur_aws/tests/test_plugin.py

@@ -1,5 +1,82 @@
+import boto3
+from moto import mock_sts, mock_s3
+
+
 def test_get_certificates(app):
     from lemur.plugins.base import plugins
 
     p = plugins.get("aws-s3")
     assert p
+
+
+@mock_sts()
+@mock_s3()
+def test_upload_acme_token(app):
+    from lemur.plugins.base import plugins
+    from lemur.plugins.lemur_aws.s3 import get
+
+    bucket = "public-bucket"
+    account = "123456789012"
+    prefix = "some-path/more-path/"
+    token_content = "Challenge"
+    token_name = "TOKEN"
+    token_path = ".well-known/acme-challenge/" + token_name
+
+    additional_options = [
+        {
+            "name": "bucket",
+            "value": bucket,
+            "type": "str",
+            "required": True,
+            "validation": r"[0-9a-z.-]{3,63}",
+            "helpMessage": "Must be a valid S3 bucket name!",
+        },
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "value": account,
+            "required": True,
+            "validation": r"[0-9]{12}",
+            "helpMessage": "A valid AWS account number with permission to access S3",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+        {
+            "name": "encrypt",
+            "type": "bool",
+            "value": False,
+            "required": False,
+            "helpMessage": "Enable server side encryption",
+            "default": True,
+        },
+        {
+            "name": "prefix",
+            "type": "str",
+            "value": prefix,
+            "required": False,
+            "helpMessage": "Must be a valid S3 object prefix!",
+        },
+    ]
+
+    s3_client = boto3.client('s3')
+    s3_client.create_bucket(Bucket=bucket)
+    p = plugins.get("aws-s3")
+
+    p.upload_acme_token(token_path=token_path,
+                        token_content=token_content,
+                        token=token_content,
+                        options=additional_options)
+
+    response = get(bucket_name=bucket,
+                   prefixed_object_name=prefix + token_name,
+                   encrypt=False,
+                   account_number=account)
+
+    # put data, and getting the same data
+    assert (response == token_content)

lemur/plugins/lemur_aws/tests/test_s3.py

@@ -0,0 +1,41 @@
+import boto3
+from moto import mock_sts, mock_s3
+
+
+@mock_sts()
+@mock_s3()
+def test_put_delete_s3_object(app):
+    from lemur.plugins.lemur_aws.s3 import put, delete, get
+
+    bucket = "public-bucket"
+    region = "us-east-1"
+    account = "123456789012"
+    path = "some-path/foo"
+    data = "dummy data"
+
+    s3_client = boto3.client('s3')
+    s3_client.create_bucket(Bucket=bucket)
+
+    put(bucket_name=bucket,
+        region_name=region,
+        prefix=path,
+        data=data,
+        encrypt=False,
+        account_number=account,
+        region=region)
+
+    response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+
+    # put data, and getting the same data
+    assert (response == data)
+
+    response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account)
+
+    # attempting to get thccle wrong data
+    assert (response is None)
+
+    delete(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+    response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
+
+    # delete data, and getting the same data
+    assert (response is None)