Merge branch 'master' into bugfix/group-lookup-fix-referral

This commit is contained in:
Hossein Shafagh 2020-11-09 14:30:01 -08:00 committed by GitHub
commit 018f4a4b77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 322 additions and 17 deletions

View File

@ -1,12 +1,15 @@
import time import time
import json import json
import arrow
from flask_script import Manager from flask_script import Manager
from flask import current_app from flask import current_app
from lemur.extensions import sentry from lemur.extensions import sentry
from lemur.constants import SUCCESS_METRIC_STATUS from lemur.constants import SUCCESS_METRIC_STATUS
from lemur.plugins import plugins
from lemur.plugins.lemur_acme.plugin import AcmeHandler from lemur.plugins.lemur_acme.plugin import AcmeHandler
from lemur.plugins.lemur_aws import s3
manager = Manager( manager = Manager(
usage="Handles all ACME related tasks" usage="Handles all ACME related tasks"
@ -84,3 +87,105 @@ def dnstest(domain, token):
status = SUCCESS_METRIC_STATUS status = SUCCESS_METRIC_STATUS
print("[+] Done with ACME Tests.") print("[+] Done with ACME Tests.")
@manager.option(
"-t",
"--token",
dest="token",
default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
required=False,
help="Value of the Token",
)
@manager.option(
"-n",
"--token_name",
dest="token_name",
default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
required=False,
help="path",
)
@manager.option(
"-p",
"--prefix",
dest="prefix",
default="test/",
required=False,
help="S3 bucket prefix",
)
@manager.option(
"-a",
"--account_number",
dest="account_number",
required=True,
help="AWS Account",
)
@manager.option(
"-b",
"--bucket_name",
dest="bucket_name",
required=True,
help="Bucket Name",
)
def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
"""
This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
It mainly serves for testing purposes.
:param token:
:param token_name:
:param prefix:
:param account_number:
:param bucket_name:
:return:
"""
additional_options = [
{
"name": "bucket",
"value": bucket_name,
"type": "str",
"required": True,
"validation": r"[0-9a-z.-]{3,63}",
"helpMessage": "Must be a valid S3 bucket name!",
},
{
"name": "accountNumber",
"type": "str",
"value": account_number,
"required": True,
"validation": r"[0-9]{12}",
"helpMessage": "A valid AWS account number with permission to access S3",
},
{
"name": "region",
"type": "str",
"default": "us-east-1",
"required": False,
"helpMessage": "Region bucket exists",
"available": ["us-east-1", "us-west-2", "eu-west-1"],
},
{
"name": "encrypt",
"type": "bool",
"value": False,
"required": False,
"helpMessage": "Enable server side encryption",
"default": True,
},
{
"name": "prefix",
"type": "str",
"value": prefix,
"required": False,
"helpMessage": "Must be a valid S3 object prefix!",
},
]
p = plugins.get("aws-s3")
p.upload_acme_token(token_name, token, additional_options)
if not prefix.endswith("/"):
prefix + "/"
token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
assert(token_res == token)
s3.delete(bucket_name, prefix + token_name, account_number=account_number)

View File

@ -33,6 +33,7 @@
.. moduleauthor:: Harm Weites <harm@weites.com> .. moduleauthor:: Harm Weites <harm@weites.com>
""" """
import sys
from acme.errors import ClientError from acme.errors import ClientError
from flask import current_app from flask import current_app
@ -408,6 +409,47 @@ class S3DestinationPlugin(ExportDestinationPlugin):
account_number=self.get_option("accountNumber", options), account_number=self.get_option("accountNumber", options),
) )
def upload_acme_token(self, token_path, token, options, **kwargs):
"""
This is called from the acme http challenge
:param self:
:param token_path:
:param token:
:param options:
:param kwargs:
:return:
"""
current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
function = f"{__name__}.{sys._getframe().f_code.co_name}"
account_number = self.get_option("accountNumber", options)
bucket_name = self.get_option("bucket", options)
prefix = self.get_option("prefix", options)
region = self.get_option("region", options)
filename = token_path.split("/")[-1]
if not prefix.endswith("/"):
prefix + "/"
res = s3.put(bucket_name=bucket_name,
region_name=region,
prefix=prefix + filename,
data=token,
encrypt=False,
account_number=account_number)
res = "Success" if res else "Failure"
log_data = {
"function": function,
"message": "check if any valid certificate is revoked",
"result": res,
"bucket_name": bucket_name,
"filename": filename
}
current_app.logger.info(log_data)
metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
"bucket_name": bucket_name,
"filename": filename})
class SNSNotificationPlugin(ExpirationNotificationPlugin): class SNSNotificationPlugin(ExpirationNotificationPlugin):
title = "AWS SNS" title = "AWS SNS"

View File

@ -6,12 +6,15 @@
:license: Apache, see LICENSE for more details. :license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com> .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
""" """
from botocore.exceptions import ClientError
from flask import current_app from flask import current_app
from lemur.extensions import sentry
from .sts import sts_client from .sts import sts_client
@sts_client("s3", service_type="resource") @sts_client("s3", service_type="resource")
def put(bucket_name, region, prefix, data, encrypt, **kwargs): def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
""" """
Use STS to write to an S3 bucket Use STS to write to an S3 bucket
""" """
@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs):
ServerSideEncryption="AES256", ServerSideEncryption="AES256",
) )
else: else:
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") try:
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
return True
except ClientError:
sentry.captureException()
return False
@sts_client("s3", service_type="client")
def delete(bucket_name, prefixed_object_name, **kwargs):
"""
Use STS to delete an object
"""
try:
response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name)
current_app.logger.debug(f"Delete data from S3."
f"Bucket: {bucket_name},"
f"Prefix: {prefixed_object_name},"
f"Status_code: {response}")
return response['ResponseMetadata']['HTTPStatusCode'] < 300
except ClientError:
sentry.captureException()
return False
@sts_client("s3", service_type="client")
def get(bucket_name, prefixed_object_name, **kwargs):
"""
Use STS to get an object
"""
try:
response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name)
current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
f"object_name: {prefixed_object_name}")
return response['Body'].read().decode("utf-8")
except ClientError:
sentry.captureException()
return None

View File

@ -1,5 +1,82 @@
import boto3
from moto import mock_sts, mock_s3
def test_get_certificates(app): def test_get_certificates(app):
from lemur.plugins.base import plugins from lemur.plugins.base import plugins
p = plugins.get("aws-s3") p = plugins.get("aws-s3")
assert p assert p
@mock_sts()
@mock_s3()
def test_upload_acme_token(app):
from lemur.plugins.base import plugins
from lemur.plugins.lemur_aws.s3 import get
bucket = "public-bucket"
account = "123456789012"
prefix = "some-path/more-path/"
token_content = "Challenge"
token_name = "TOKEN"
token_path = ".well-known/acme-challenge/" + token_name
additional_options = [
{
"name": "bucket",
"value": bucket,
"type": "str",
"required": True,
"validation": r"[0-9a-z.-]{3,63}",
"helpMessage": "Must be a valid S3 bucket name!",
},
{
"name": "accountNumber",
"type": "str",
"value": account,
"required": True,
"validation": r"[0-9]{12}",
"helpMessage": "A valid AWS account number with permission to access S3",
},
{
"name": "region",
"type": "str",
"default": "us-east-1",
"required": False,
"helpMessage": "Region bucket exists",
"available": ["us-east-1", "us-west-2", "eu-west-1"],
},
{
"name": "encrypt",
"type": "bool",
"value": False,
"required": False,
"helpMessage": "Enable server side encryption",
"default": True,
},
{
"name": "prefix",
"type": "str",
"value": prefix,
"required": False,
"helpMessage": "Must be a valid S3 object prefix!",
},
]
s3_client = boto3.client('s3')
s3_client.create_bucket(Bucket=bucket)
p = plugins.get("aws-s3")
p.upload_acme_token(token_path=token_path,
token_content=token_content,
token=token_content,
options=additional_options)
response = get(bucket_name=bucket,
prefixed_object_name=prefix + token_name,
encrypt=False,
account_number=account)
# put data, and getting the same data
assert (response == token_content)

View File

@ -0,0 +1,41 @@
import boto3
from moto import mock_sts, mock_s3
@mock_sts()
@mock_s3()
def test_put_delete_s3_object(app):
from lemur.plugins.lemur_aws.s3 import put, delete, get
bucket = "public-bucket"
region = "us-east-1"
account = "123456789012"
path = "some-path/foo"
data = "dummy data"
s3_client = boto3.client('s3')
s3_client.create_bucket(Bucket=bucket)
put(bucket_name=bucket,
region_name=region,
prefix=path,
data=data,
encrypt=False,
account_number=account,
region=region)
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
# put data, and getting the same data
assert (response == data)
response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account)
# attempting to get thccle wrong data
assert (response is None)
delete(bucket_name=bucket, prefixed_object_name=path, account_number=account)
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
# delete data, and getting the same data
assert (response is None)

View File

@ -6,7 +6,7 @@
# #
appdirs==1.4.3 # via virtualenv appdirs==1.4.3 # via virtualenv
bleach==3.1.4 # via readme-renderer bleach==3.1.4 # via readme-renderer
certifi==2020.6.20 # via requests certifi==2020.11.8 # via requests
cffi==1.14.0 # via cryptography cffi==1.14.0 # via cryptography
cfgv==3.1.0 # via pre-commit cfgv==3.1.0 # via pre-commit
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage
distlib==0.3.0 # via virtualenv distlib==0.3.0 # via virtualenv
docutils==0.16 # via readme-renderer docutils==0.16 # via readme-renderer
filelock==3.0.12 # via virtualenv filelock==3.0.12 # via virtualenv
flake8==3.8.4 # via -r requirements-dev.in
identify==1.4.14 # via pre-commit identify==1.4.14 # via pre-commit
idna==2.9 # via requests idna==2.9 # via requests
flake8==3.8.4 # via -r requirements-dev.in
invoke==1.4.1 # via -r requirements-dev.in invoke==1.4.1 # via -r requirements-dev.in
jeepney==0.4.3 # via keyring, secretstorage jeepney==0.4.3 # via keyring, secretstorage
keyring==21.2.0 # via twine keyring==21.2.0 # via twine

View File

@ -17,14 +17,14 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko
beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare
billiard==3.6.3.0 # via -r requirements.txt, celery billiard==3.6.3.0 # via -r requirements.txt, celery
blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven
boto3==1.16.9 # via -r requirements.txt boto3==1.16.14 # via -r requirements.txt
botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.txt celery[redis]==4.4.2 # via -r requirements.txt
certifi==2020.6.20 # via -r requirements.txt, requests certifi==2020.11.8 # via -r requirements.txt, requests
certsrv==2.1.1 # via -r requirements.txt certsrv==2.1.1 # via -r requirements.txt
cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl
chardet==3.0.4 # via -r requirements.txt, requests chardet==3.0.4 # via -r requirements.txt, requests
click==7.1.1 # via -r requirements.txt, flask click==7.1.2 # via -r requirements.txt, flask
cloudflare==2.8.13 # via -r requirements.txt cloudflare==2.8.13 # via -r requirements.txt
cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests
dnspython3==1.15.0 # via -r requirements.txt dnspython3==1.15.0 # via -r requirements.txt

View File

@ -10,10 +10,10 @@ aws-sam-translator==1.22.0 # via cfn-lint
aws-xray-sdk==2.5.0 # via moto aws-xray-sdk==2.5.0 # via moto
bandit==1.6.2 # via -r requirements-tests.in bandit==1.6.2 # via -r requirements-tests.in
black==20.8b1 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in
boto3==1.16.9 # via aws-sam-translator, moto boto3==1.16.14 # via aws-sam-translator, moto
boto==2.49.0 # via moto boto==2.49.0 # via moto
botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2020.6.20 # via requests certifi==2020.11.8 # via requests
cffi==1.14.0 # via cryptography cffi==1.14.0 # via cryptography
cfn-lint==0.29.5 # via moto cfn-lint==0.29.5 # via moto
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx
docker==4.2.0 # via moto docker==4.2.0 # via moto
ecdsa==0.14.1 # via moto, python-jose, sshpubkeys ecdsa==0.14.1 # via moto, python-jose, sshpubkeys
factory-boy==3.1.0 # via -r requirements-tests.in factory-boy==3.1.0 # via -r requirements-tests.in
faker==4.14.0 # via -r requirements-tests.in, factory-boy faker==4.14.2 # via -r requirements-tests.in, factory-boy
fakeredis==1.4.4 # via -r requirements-tests.in fakeredis==1.4.4 # via -r requirements-tests.in
flask==1.1.2 # via pytest-flask flask==1.1.2 # via pytest-flask
freezegun==1.0.0 # via -r requirements-tests.in freezegun==1.0.0 # via -r requirements-tests.in
@ -59,7 +59,7 @@ pycparser==2.20 # via cffi
pyflakes==2.2.0 # via -r requirements-tests.in pyflakes==2.2.0 # via -r requirements-tests.in
pyparsing==2.4.7 # via packaging pyparsing==2.4.7 # via packaging
pyrsistent==0.16.0 # via jsonschema pyrsistent==0.16.0 # via jsonschema
pytest-flask==1.0.0 # via -r requirements-tests.in pytest-flask==1.1.0 # via -r requirements-tests.in
pytest-mock==3.3.1 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in
pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock
python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-dateutil==2.8.1 # via botocore, faker, freezegun, moto

View File

@ -15,14 +15,14 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko
beautifulsoup4==4.9.1 # via cloudflare beautifulsoup4==4.9.1 # via cloudflare
billiard==3.6.3.0 # via celery billiard==3.6.3.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.16.10 # via -r requirements.in boto3==1.16.14 # via -r requirements.in
botocore==1.19.10 # via -r requirements.in, boto3, s3transfer botocore==1.19.14 # via -r requirements.in, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.in celery[redis]==4.4.2 # via -r requirements.in
certifi==2020.6.20 # via -r requirements.in, requests certifi==2020.11.8 # via -r requirements.in, requests
certsrv==2.1.1 # via -r requirements.in certsrv==2.1.1 # via -r requirements.in
cffi==1.14.0 # via bcrypt, cryptography, pynacl cffi==1.14.0 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
click==7.1.2 # black 20.8b1 has requirement click>=7.1.2 click==7.1.2 # via flask
cloudflare==2.8.13 # via -r requirements.in cloudflare==2.8.13 # via -r requirements.in
cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests
dnspython3==1.15.0 # via -r requirements.in dnspython3==1.15.0 # via -r requirements.in