Merge branch 'master' into bugfix/group-lookup-fix-referral
This commit is contained in:
Hossein Shafagh
GitHub
commit
018f4a4b77
@ -1,12 +1,15 @@
|
||||
import time
|
||||
import json
|
||||
import arrow
|
||||
|
||||
from flask_script import Manager
|
||||
from flask import current_app
|
||||
|
||||
from lemur.extensions import sentry
|
||||
from lemur.constants import SUCCESS_METRIC_STATUS
|
||||
from lemur.plugins import plugins
|
||||
from lemur.plugins.lemur_acme.plugin import AcmeHandler
|
||||
from lemur.plugins.lemur_aws import s3
|
||||
|
||||
manager = Manager(
|
||||
usage="Handles all ACME related tasks"
|
||||
@ -84,3 +87,105 @@ def dnstest(domain, token):
|
||||
|
||||
status = SUCCESS_METRIC_STATUS
|
||||
print("[+] Done with ACME Tests.")
|
||||
|
||||
|
||||
@manager.option(
|
||||
"-t",
|
||||
"--token",
|
||||
dest="token",
|
||||
default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
|
||||
required=False,
|
||||
help="Value of the Token",
|
||||
)
|
||||
@manager.option(
|
||||
"-n",
|
||||
"--token_name",
|
||||
dest="token_name",
|
||||
default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"),
|
||||
required=False,
|
||||
help="path",
|
||||
)
|
||||
@manager.option(
|
||||
"-p",
|
||||
"--prefix",
|
||||
dest="prefix",
|
||||
default="test/",
|
||||
required=False,
|
||||
help="S3 bucket prefix",
|
||||
)
|
||||
@manager.option(
|
||||
"-a",
|
||||
"--account_number",
|
||||
dest="account_number",
|
||||
required=True,
|
||||
help="AWS Account",
|
||||
)
|
||||
@manager.option(
|
||||
"-b",
|
||||
"--bucket_name",
|
||||
dest="bucket_name",
|
||||
required=True,
|
||||
help="Bucket Name",
|
||||
)
|
||||
def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name):
|
||||
"""
|
||||
This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it.
|
||||
It mainly serves for testing purposes.
|
||||
:param token:
|
||||
:param token_name:
|
||||
:param prefix:
|
||||
:param account_number:
|
||||
:param bucket_name:
|
||||
:return:
|
||||
"""
|
||||
additional_options = [
|
||||
{
|
||||
"name": "bucket",
|
||||
"value": bucket_name,
|
||||
"type": "str",
|
||||
"required": True,
|
||||
"validation": r"[0-9a-z.-]{3,63}",
|
||||
"helpMessage": "Must be a valid S3 bucket name!",
|
||||
},
|
||||
{
|
||||
"name": "accountNumber",
|
||||
"type": "str",
|
||||
"value": account_number,
|
||||
"required": True,
|
||||
"validation": r"[0-9]{12}",
|
||||
"helpMessage": "A valid AWS account number with permission to access S3",
|
||||
},
|
||||
{
|
||||
"name": "region",
|
||||
"type": "str",
|
||||
"default": "us-east-1",
|
||||
"required": False,
|
||||
"helpMessage": "Region bucket exists",
|
||||
"available": ["us-east-1", "us-west-2", "eu-west-1"],
|
||||
},
|
||||
{
|
||||
"name": "encrypt",
|
||||
"type": "bool",
|
||||
"value": False,
|
||||
"required": False,
|
||||
"helpMessage": "Enable server side encryption",
|
||||
"default": True,
|
||||
},
|
||||
{
|
||||
"name": "prefix",
|
||||
"type": "str",
|
||||
"value": prefix,
|
||||
"required": False,
|
||||
"helpMessage": "Must be a valid S3 object prefix!",
|
||||
},
|
||||
]
|
||||
|
||||
p = plugins.get("aws-s3")
|
||||
p.upload_acme_token(token_name, token, additional_options)
|
||||
|
||||
if not prefix.endswith("/"):
|
||||
prefix + "/"
|
||||
|
||||
token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number)
|
||||
assert(token_res == token)
|
||||
s3.delete(bucket_name, prefix + token_name, account_number=account_number)
|
||||
|
||||
@ -33,6 +33,7 @@
|
||||
.. moduleauthor:: Harm Weites <harm@weites.com>
|
||||
"""
|
||||
|
||||
import sys
|
||||
from acme.errors import ClientError
|
||||
from flask import current_app
|
||||
|
||||
@ -408,6 +409,47 @@ class S3DestinationPlugin(ExportDestinationPlugin):
|
||||
account_number=self.get_option("accountNumber", options),
|
||||
)
|
||||
|
||||
def upload_acme_token(self, token_path, token, options, **kwargs):
|
||||
"""
|
||||
This is called from the acme http challenge
|
||||
:param self:
|
||||
:param token_path:
|
||||
:param token:
|
||||
:param options:
|
||||
:param kwargs:
|
||||
:return:
|
||||
"""
|
||||
current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
|
||||
|
||||
function = f"{__name__}.{sys._getframe().f_code.co_name}"
|
||||
|
||||
account_number = self.get_option("accountNumber", options)
|
||||
bucket_name = self.get_option("bucket", options)
|
||||
prefix = self.get_option("prefix", options)
|
||||
region = self.get_option("region", options)
|
||||
filename = token_path.split("/")[-1]
|
||||
if not prefix.endswith("/"):
|
||||
prefix + "/"
|
||||
|
||||
res = s3.put(bucket_name=bucket_name,
|
||||
region_name=region,
|
||||
prefix=prefix + filename,
|
||||
data=token,
|
||||
encrypt=False,
|
||||
account_number=account_number)
|
||||
res = "Success" if res else "Failure"
|
||||
log_data = {
|
||||
"function": function,
|
||||
"message": "check if any valid certificate is revoked",
|
||||
"result": res,
|
||||
"bucket_name": bucket_name,
|
||||
"filename": filename
|
||||
}
|
||||
current_app.logger.info(log_data)
|
||||
metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
|
||||
"bucket_name": bucket_name,
|
||||
"filename": filename})
|
||||
|
||||
|
||||
class SNSNotificationPlugin(ExpirationNotificationPlugin):
|
||||
title = "AWS SNS"
|
||||
|
||||
@ -6,12 +6,15 @@
|
||||
:license: Apache, see LICENSE for more details.
|
||||
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
|
||||
"""
|
||||
from botocore.exceptions import ClientError
|
||||
from flask import current_app
|
||||
from lemur.extensions import sentry
|
||||
|
||||
from .sts import sts_client
|
||||
|
||||
|
||||
@sts_client("s3", service_type="resource")
|
||||
def put(bucket_name, region, prefix, data, encrypt, **kwargs):
|
||||
def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
|
||||
"""
|
||||
Use STS to write to an S3 bucket
|
||||
"""
|
||||
@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs):
|
||||
ServerSideEncryption="AES256",
|
||||
)
|
||||
else:
|
||||
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
|
||||
try:
|
||||
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
|
||||
return True
|
||||
except ClientError:
|
||||
sentry.captureException()
|
||||
return False
|
||||
|
||||
|
||||
@sts_client("s3", service_type="client")
|
||||
def delete(bucket_name, prefixed_object_name, **kwargs):
|
||||
"""
|
||||
Use STS to delete an object
|
||||
"""
|
||||
try:
|
||||
response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name)
|
||||
current_app.logger.debug(f"Delete data from S3."
|
||||
f"Bucket: {bucket_name},"
|
||||
f"Prefix: {prefixed_object_name},"
|
||||
f"Status_code: {response}")
|
||||
return response['ResponseMetadata']['HTTPStatusCode'] < 300
|
||||
except ClientError:
|
||||
sentry.captureException()
|
||||
return False
|
||||
|
||||
|
||||
@sts_client("s3", service_type="client")
|
||||
def get(bucket_name, prefixed_object_name, **kwargs):
|
||||
"""
|
||||
Use STS to get an object
|
||||
"""
|
||||
try:
|
||||
response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name)
|
||||
current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
|
||||
f"object_name: {prefixed_object_name}")
|
||||
return response['Body'].read().decode("utf-8")
|
||||
except ClientError:
|
||||
sentry.captureException()
|
||||
return None
|
||||
|
||||
@ -1,5 +1,82 @@
|
||||
import boto3
|
||||
from moto import mock_sts, mock_s3
|
||||
|
||||
|
||||
def test_get_certificates(app):
|
||||
from lemur.plugins.base import plugins
|
||||
|
||||
p = plugins.get("aws-s3")
|
||||
assert p
|
||||
|
||||
|
||||
@mock_sts()
|
||||
@mock_s3()
|
||||
def test_upload_acme_token(app):
|
||||
from lemur.plugins.base import plugins
|
||||
from lemur.plugins.lemur_aws.s3 import get
|
||||
|
||||
bucket = "public-bucket"
|
||||
account = "123456789012"
|
||||
prefix = "some-path/more-path/"
|
||||
token_content = "Challenge"
|
||||
token_name = "TOKEN"
|
||||
token_path = ".well-known/acme-challenge/" + token_name
|
||||
|
||||
additional_options = [
|
||||
{
|
||||
"name": "bucket",
|
||||
"value": bucket,
|
||||
"type": "str",
|
||||
"required": True,
|
||||
"validation": r"[0-9a-z.-]{3,63}",
|
||||
"helpMessage": "Must be a valid S3 bucket name!",
|
||||
},
|
||||
{
|
||||
"name": "accountNumber",
|
||||
"type": "str",
|
||||
"value": account,
|
||||
"required": True,
|
||||
"validation": r"[0-9]{12}",
|
||||
"helpMessage": "A valid AWS account number with permission to access S3",
|
||||
},
|
||||
{
|
||||
"name": "region",
|
||||
"type": "str",
|
||||
"default": "us-east-1",
|
||||
"required": False,
|
||||
"helpMessage": "Region bucket exists",
|
||||
"available": ["us-east-1", "us-west-2", "eu-west-1"],
|
||||
},
|
||||
{
|
||||
"name": "encrypt",
|
||||
"type": "bool",
|
||||
"value": False,
|
||||
"required": False,
|
||||
"helpMessage": "Enable server side encryption",
|
||||
"default": True,
|
||||
},
|
||||
{
|
||||
"name": "prefix",
|
||||
"type": "str",
|
||||
"value": prefix,
|
||||
"required": False,
|
||||
"helpMessage": "Must be a valid S3 object prefix!",
|
||||
},
|
||||
]
|
||||
|
||||
s3_client = boto3.client('s3')
|
||||
s3_client.create_bucket(Bucket=bucket)
|
||||
p = plugins.get("aws-s3")
|
||||
|
||||
p.upload_acme_token(token_path=token_path,
|
||||
token_content=token_content,
|
||||
token=token_content,
|
||||
options=additional_options)
|
||||
|
||||
response = get(bucket_name=bucket,
|
||||
prefixed_object_name=prefix + token_name,
|
||||
encrypt=False,
|
||||
account_number=account)
|
||||
|
||||
# put data, and getting the same data
|
||||
assert (response == token_content)
|
||||
|
||||
41
lemur/plugins/lemur_aws/tests/test_s3.py
Normal file
41
lemur/plugins/lemur_aws/tests/test_s3.py
Normal file
@ -0,0 +1,41 @@
|
||||
import boto3
|
||||
from moto import mock_sts, mock_s3
|
||||
|
||||
|
||||
@mock_sts()
|
||||
@mock_s3()
|
||||
def test_put_delete_s3_object(app):
|
||||
from lemur.plugins.lemur_aws.s3 import put, delete, get
|
||||
|
||||
bucket = "public-bucket"
|
||||
region = "us-east-1"
|
||||
account = "123456789012"
|
||||
path = "some-path/foo"
|
||||
data = "dummy data"
|
||||
|
||||
s3_client = boto3.client('s3')
|
||||
s3_client.create_bucket(Bucket=bucket)
|
||||
|
||||
put(bucket_name=bucket,
|
||||
region_name=region,
|
||||
prefix=path,
|
||||
data=data,
|
||||
encrypt=False,
|
||||
account_number=account,
|
||||
region=region)
|
||||
|
||||
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
|
||||
|
||||
# put data, and getting the same data
|
||||
assert (response == data)
|
||||
|
||||
response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account)
|
||||
|
||||
# attempting to get thccle wrong data
|
||||
assert (response is None)
|
||||
|
||||
delete(bucket_name=bucket, prefixed_object_name=path, account_number=account)
|
||||
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
|
||||
|
||||
# delete data, and getting the same data
|
||||
assert (response is None)
|
||||
@ -6,7 +6,7 @@
|
||||
#
|
||||
appdirs==1.4.3 # via virtualenv
|
||||
bleach==3.1.4 # via readme-renderer
|
||||
certifi==2020.6.20 # via requests
|
||||
certifi==2020.11.8 # via requests
|
||||
cffi==1.14.0 # via cryptography
|
||||
cfgv==3.1.0 # via pre-commit
|
||||
chardet==3.0.4 # via requests
|
||||
@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage
|
||||
distlib==0.3.0 # via virtualenv
|
||||
docutils==0.16 # via readme-renderer
|
||||
filelock==3.0.12 # via virtualenv
|
||||
flake8==3.8.4 # via -r requirements-dev.in
|
||||
identify==1.4.14 # via pre-commit
|
||||
idna==2.9 # via requests
|
||||
flake8==3.8.4 # via -r requirements-dev.in
|
||||
invoke==1.4.1 # via -r requirements-dev.in
|
||||
jeepney==0.4.3 # via keyring, secretstorage
|
||||
keyring==21.2.0 # via twine
|
||||
|
||||
@ -17,14 +17,14 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko
|
||||
beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare
|
||||
billiard==3.6.3.0 # via -r requirements.txt, celery
|
||||
blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven
|
||||
boto3==1.16.9 # via -r requirements.txt
|
||||
botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer
|
||||
boto3==1.16.14 # via -r requirements.txt
|
||||
botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer
|
||||
celery[redis]==4.4.2 # via -r requirements.txt
|
||||
certifi==2020.6.20 # via -r requirements.txt, requests
|
||||
certifi==2020.11.8 # via -r requirements.txt, requests
|
||||
certsrv==2.1.1 # via -r requirements.txt
|
||||
cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl
|
||||
chardet==3.0.4 # via -r requirements.txt, requests
|
||||
click==7.1.1 # via -r requirements.txt, flask
|
||||
click==7.1.2 # via -r requirements.txt, flask
|
||||
cloudflare==2.8.13 # via -r requirements.txt
|
||||
cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests
|
||||
dnspython3==1.15.0 # via -r requirements.txt
|
||||
|
||||
@ -10,10 +10,10 @@ aws-sam-translator==1.22.0 # via cfn-lint
|
||||
aws-xray-sdk==2.5.0 # via moto
|
||||
bandit==1.6.2 # via -r requirements-tests.in
|
||||
black==20.8b1 # via -r requirements-tests.in
|
||||
boto3==1.16.9 # via aws-sam-translator, moto
|
||||
boto3==1.16.14 # via aws-sam-translator, moto
|
||||
boto==2.49.0 # via moto
|
||||
botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer
|
||||
certifi==2020.6.20 # via requests
|
||||
botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer
|
||||
certifi==2020.11.8 # via requests
|
||||
cffi==1.14.0 # via cryptography
|
||||
cfn-lint==0.29.5 # via moto
|
||||
chardet==3.0.4 # via requests
|
||||
@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx
|
||||
docker==4.2.0 # via moto
|
||||
ecdsa==0.14.1 # via moto, python-jose, sshpubkeys
|
||||
factory-boy==3.1.0 # via -r requirements-tests.in
|
||||
faker==4.14.0 # via -r requirements-tests.in, factory-boy
|
||||
faker==4.14.2 # via -r requirements-tests.in, factory-boy
|
||||
fakeredis==1.4.4 # via -r requirements-tests.in
|
||||
flask==1.1.2 # via pytest-flask
|
||||
freezegun==1.0.0 # via -r requirements-tests.in
|
||||
@ -59,7 +59,7 @@ pycparser==2.20 # via cffi
|
||||
pyflakes==2.2.0 # via -r requirements-tests.in
|
||||
pyparsing==2.4.7 # via packaging
|
||||
pyrsistent==0.16.0 # via jsonschema
|
||||
pytest-flask==1.0.0 # via -r requirements-tests.in
|
||||
pytest-flask==1.1.0 # via -r requirements-tests.in
|
||||
pytest-mock==3.3.1 # via -r requirements-tests.in
|
||||
pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock
|
||||
python-dateutil==2.8.1 # via botocore, faker, freezegun, moto
|
||||
|
||||
@ -15,14 +15,14 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko
|
||||
beautifulsoup4==4.9.1 # via cloudflare
|
||||
billiard==3.6.3.0 # via celery
|
||||
blinker==1.4 # via flask-mail, flask-principal, raven
|
||||
boto3==1.16.10 # via -r requirements.in
|
||||
botocore==1.19.10 # via -r requirements.in, boto3, s3transfer
|
||||
boto3==1.16.14 # via -r requirements.in
|
||||
botocore==1.19.14 # via -r requirements.in, boto3, s3transfer
|
||||
celery[redis]==4.4.2 # via -r requirements.in
|
||||
certifi==2020.6.20 # via -r requirements.in, requests
|
||||
certifi==2020.11.8 # via -r requirements.in, requests
|
||||
certsrv==2.1.1 # via -r requirements.in
|
||||
cffi==1.14.0 # via bcrypt, cryptography, pynacl
|
||||
chardet==3.0.4 # via requests
|
||||
click==7.1.2 # black 20.8b1 has requirement click>=7.1.2
|
||||
click==7.1.2 # via flask
|
||||
cloudflare==2.8.13 # via -r requirements.in
|
||||
cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests
|
||||
dnspython3==1.15.0 # via -r requirements.in
|
||||
|
||||
Loading…
Reference in New Issue
Block a user