Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #3192 from hosseinsh/expanding-S3-plugin 

Expanding the S3 plugin with acme-http01-token support
This commit is contained in:
Hossein Shafagh 2020-11-09 13:18:38 -08:00 committed by GitHub
parent cafc2c1d80 7c779d6283
commit 40ef6d06d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 307 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
lemur/acme_providers/cli.py
View File

@ -1,12 +1,15 @@
import time
import json
import arrow
from flask_script import Manager
from flask import current_app
from lemur.extensions import sentry
from lemur.constants import SUCCESS_METRIC_STATUS
from lemur.plugins import plugins
from lemur.plugins.lemur_acme.plugin import AcmeHandler
from lemur.plugins.lemur_aws import s3
manager = Manager(
usage="Handles all ACME related tasks"
@ -84,3 +87,105 @@ def dnstest(domain, token):
status = SUCCESS_METRIC_STATUS
print("[+] Done with ACME Tests.")
@manager.option(
"-t",
"--token",
dest="token",
default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
required=False,
help="Value of the Token",
)
@manager.option(
"-n",
"--token_name",
dest="token_name",
default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
required=False,
help="path",
)
@manager.option(
"-p",
"--prefix",
dest="prefix",
default="test/",
required=False,
help="S3 bucket prefix",
)
@manager.option(
"-a",
"--account_number",
dest="account_number",
required=True,
help="AWS Account",
)
@manager.option(
"-b",
"--bucket_name",
dest="bucket_name",
required=True,
help="Bucket Name",
)
def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
"""
This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
It mainly serves for testing purposes.
:param token:
:param token_name:
:param prefix:
:param account_number:
:param bucket_name:
:return:
"""
additional_options = [
{
"name": "bucket",
"value": bucket_name,
"type": "str",
"required": True,
"validation": r"[0-9a-z.-]{3,63}",
"helpMessage": "Must be a valid S3 bucket name!",
},
{
"name": "accountNumber",
"type": "str",
"value": account_number,
"required": True,
"validation": r"[0-9]{12}",
"helpMessage": "A valid AWS account number with permission to access S3",
},
{
"name": "region",
"type": "str",
"default": "us-east-1",
"required": False,
"helpMessage": "Region bucket exists",
"available": ["us-east-1", "us-west-2", "eu-west-1"],
},
{
"name": "encrypt",
"type": "bool",
"value": False,
"required": False,
"helpMessage": "Enable server side encryption",
"default": True,
},
{
"name": "prefix",
"type": "str",
"value": prefix,
"required": False,
"helpMessage": "Must be a valid S3 object prefix!",
},
]
p = plugins.get("aws-s3")
p.upload_acme_token(token_name, token, additional_options)
if not prefix.endswith("/"):
prefix + "/"
token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
assert(token_res == token)
s3.delete(bucket_name, prefix + token_name, account_number=account_number)

42
lemur/plugins/lemur_aws/plugin.py
View File

@ -33,6 +33,7 @@
.. moduleauthor:: Harm Weites <harm@weites.com>
"""
import sys
from acme.errors import ClientError
from flask import current_app
@ -408,6 +409,47 @@ class S3DestinationPlugin(ExportDestinationPlugin):
account_number=self.get_option("accountNumber", options),
)
def upload_acme_token(self, token_path, token, options, **kwargs):
"""
This is called from the acme http challenge
:param self:
:param token_path:
:param token:
:param options:
:param kwargs:
:return:
"""
current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
function = f"{__name__}.{sys._getframe().f_code.co_name}"
account_number = self.get_option("accountNumber", options)
bucket_name = self.get_option("bucket", options)
prefix = self.get_option("prefix", options)
region = self.get_option("region", options)
filename = token_path.split("/")[-1]
if not prefix.endswith("/"):
prefix + "/"
res = s3.put(bucket_name=bucket_name,
region_name=region,
prefix=prefix + filename,
data=token,
encrypt=False,
account_number=account_number)
res = "Success" if res else "Failure"
log_data = {
"function": function,
"message": "check if any valid certificate is revoked",
"result": res,
"bucket_name": bucket_name,
"filename": filename
}
current_app.logger.info(log_data)
metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
"bucket_name": bucket_name,
"filename": filename})
class SNSNotificationPlugin(ExpirationNotificationPlugin):
title = "AWS SNS"

44
lemur/plugins/lemur_aws/s3.py
View File

@ -6,12 +6,15 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from botocore.exceptions import ClientError
from flask import current_app
from lemur.extensions import sentry
from .sts import sts_client
@sts_client("s3", service_type="resource")
def put(bucket_name, region, prefix, data, encrypt, **kwargs):
def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
"""
Use STS to write to an S3 bucket
"""
@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs):
ServerSideEncryption="AES256",
)
else:
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
try:
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
return True
except ClientError:
sentry.captureException()
return False
@sts_client("s3", service_type="client")
def delete(bucket_name, prefixed_object_name, **kwargs):
"""
Use STS to delete an object
"""
try:
response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name)
current_app.logger.debug(f"Delete data from S3."
f"Bucket: {bucket_name},"
f"Prefix: {prefixed_object_name},"
f"Status_code: {response}")
return response['ResponseMetadata']['HTTPStatusCode'] < 300
except ClientError:
sentry.captureException()
return False
@sts_client("s3", service_type="client")
def get(bucket_name, prefixed_object_name, **kwargs):
"""
Use STS to get an object
"""
try:
response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name)
current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
f"object_name: {prefixed_object_name}")
return response['Body'].read().decode("utf-8")
except ClientError:
sentry.captureException()
return None

77
lemur/plugins/lemur_aws/tests/test_plugin.py
View File

@ -1,5 +1,82 @@
import boto3
from moto import mock_sts, mock_s3
def test_get_certificates(app):
from lemur.plugins.base import plugins
p = plugins.get("aws-s3")
assert p
@mock_sts()
@mock_s3()
def test_upload_acme_token(app):
from lemur.plugins.base import plugins
from lemur.plugins.lemur_aws.s3 import get
bucket = "public-bucket"
account = "123456789012"
prefix = "some-path/more-path/"
token_content = "Challenge"
token_name = "TOKEN"
token_path = ".well-known/acme-challenge/" + token_name
additional_options = [
{
"name": "bucket",
"value": bucket,
"type": "str",
"required": True,
"validation": r"[0-9a-z.-]{3,63}",
"helpMessage": "Must be a valid S3 bucket name!",
},
{
"name": "accountNumber",
"type": "str",
"value": account,
"required": True,
"validation": r"[0-9]{12}",
"helpMessage": "A valid AWS account number with permission to access S3",
},
{
"name": "region",
"type": "str",
"default": "us-east-1",
"required": False,
"helpMessage": "Region bucket exists",
"available": ["us-east-1", "us-west-2", "eu-west-1"],
},
{
"name": "encrypt",
"type": "bool",
"value": False,
"required": False,
"helpMessage": "Enable server side encryption",
"default": True,
},
{
"name": "prefix",
"type": "str",
"value": prefix,
"required": False,
"helpMessage": "Must be a valid S3 object prefix!",
},
]
s3_client = boto3.client('s3')
s3_client.create_bucket(Bucket=bucket)
p = plugins.get("aws-s3")
p.upload_acme_token(token_path=token_path,
token_content=token_content,
token=token_content,
options=additional_options)
response = get(bucket_name=bucket,
prefixed_object_name=prefix + token_name,
encrypt=False,
account_number=account)
# put data, and getting the same data
assert (response == token_content)

41
lemur/plugins/lemur_aws/tests/test_s3.py Normal file
View File

@ -0,0 +1,41 @@
import boto3
from moto import mock_sts, mock_s3
@mock_sts()
@mock_s3()
def test_put_delete_s3_object(app):
from lemur.plugins.lemur_aws.s3 import put, delete, get
bucket = "public-bucket"
region = "us-east-1"
account = "123456789012"
path = "some-path/foo"
data = "dummy data"
s3_client = boto3.client('s3')
s3_client.create_bucket(Bucket=bucket)
put(bucket_name=bucket,
region_name=region,
prefix=path,
data=data,
encrypt=False,
account_number=account,
region=region)
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
# put data, and getting the same data
assert (response == data)
response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account)
# attempting to get thccle wrong data
assert (response is None)
delete(bucket_name=bucket, prefixed_object_name=path, account_number=account)
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
# delete data, and getting the same data
assert (response is None)