Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into feature/acme-http-challenge

Browse Source
This commit is contained in:
Mathias Petermann
2020-11-10 15:20:47 +01:00
parent c71dbcb0a0 f42d9539fc
commit 4a181aff6e
31 changed files with 409 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lemur/plugins/bases/notification.py
View File

@ -42,7 +42,7 @@ class ExpirationNotificationPlugin(NotificationPlugin):
"name": "interval",
"type": "int",
"required": True,
"validation": "^\d+$",
"validation": r"^\d+$",
"helpMessage": "Number of days to be alert before expiration.",
},
{

4
lemur/plugins/lemur_acme/plugin.py
View File

@ -46,7 +46,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
"name": "acme_url",
"type": "str",
"required": True,
"validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/",
"validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/",
"helpMessage": "Must be a valid web url starting with http[s]://",
},
{
@ -59,7 +59,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
"name": "email",
"type": "str",
"default": "",
"validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/",
"validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/",
"helpMessage": "Email to use",
},
{

31
lemur/plugins/lemur_acme/tests/test_acme_dns.py
View File

@ -3,6 +3,7 @@ from unittest.mock import patch, Mock
import josepy as jose
from cryptography.x509 import DNSName
from flask import Flask
from lemur.plugins.lemur_acme import plugin
from lemur.plugins.lemur_acme.acme_handlers import AuthorizationRecord
from lemur.common.utils import generate_private_key
@ -23,6 +24,16 @@ class TestAcmeDns(unittest.TestCase):
"test.fakedomain.net": [mock_dns_provider],
}
# Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
# needed to run tests in dev environment without getting error 'Working outside of application context'.
_app = Flask('lemur_test_acme')
self.ctx = _app.app_context()
assert self.ctx
self.ctx.push()
def tearDown(self):
self.ctx.pop()
@patch("lemur.plugins.lemur_acme.plugin.len", return_value=1)
def test_get_dns_challenges(self, mock_len):
assert mock_len
@ -105,22 +116,24 @@ class TestAcmeDns(unittest.TestCase):
mock_dns_provider = Mock()
mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
mock_dns_challenge = Mock()
response = Mock()
response.simple_verify = Mock(return_value=False)
mock_dns_challenge.response = Mock(return_value=response)
mock_authz = Mock()
mock_authz.dns_challenge.response = Mock()
mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False)
mock_authz.authz = []
mock_authz.dns_challenge = []
mock_authz.dns_challenge.append(mock_dns_challenge)
mock_authz.target_domain = "www.test.com"
mock_authz_record = Mock()
mock_authz_record.body.identifier.value = "test"
mock_authz.authz = []
mock_authz.authz.append(mock_authz_record)
mock_authz.change_id = []
mock_authz.change_id.append("123")
mock_authz.dns_challenge = []
dns_challenge = Mock()
mock_authz.dns_challenge.append(dns_challenge)
self.assertRaises(
ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz)
)
with self.assertRaises(ValueError):
self.acme.complete_dns_challenge(mock_acme, mock_authz)
@patch("acme.client.Client")
@patch("OpenSSL.crypto", return_value="mock_cert")

12
lemur/plugins/lemur_acme/tests/test_powerdns.py
View File

@ -1,5 +1,7 @@
import unittest
from unittest.mock import patch, Mock
from flask import Flask
from lemur.plugins.lemur_acme import plugin, powerdns
@ -17,6 +19,16 @@ class TestPowerdns(unittest.TestCase):
"test.fakedomain.net": [mock_dns_provider],
}
# Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
# needed to run tests in dev environment without getting error 'Working outside of application context'.
_app = Flask('lemur_test_acme')
self.ctx = _app.app_context()
assert self.ctx
self.ctx.push()
def tearDown(self):
self.ctx.pop()
@patch("lemur.plugins.lemur_acme.powerdns.current_app")
def test_get_zones(self, mock_current_app):
account_number = "1234567890"

11
lemur/plugins/lemur_acme/tests/test_ultradns.py
View File

@ -1,6 +1,7 @@
import unittest
from unittest.mock import patch, Mock
from flask import Flask
from lemur.plugins.lemur_acme import plugin, ultradns
from requests.models import Response
@ -19,6 +20,16 @@ class TestUltradns(unittest.TestCase):
"test.fakedomain.net": [mock_dns_provider],
}
# Creates a new Flask application for a test duration. In python 3.8, manual push of application context is
# needed to run tests in dev environment without getting error 'Working outside of application context'.
_app = Flask('lemur_test_acme')
self.ctx = _app.app_context()
assert self.ctx
self.ctx.push()
def tearDown(self):
self.ctx.pop()
@patch("lemur.plugins.lemur_acme.ultradns.requests")
@patch("lemur.plugins.lemur_acme.ultradns.current_app")
def test_ultradns_get_token(self, mock_current_app, mock_requests):

42
lemur/plugins/lemur_aws/plugin.py
View File

@ -33,6 +33,7 @@
.. moduleauthor:: Harm Weites <harm@weites.com>
"""
import sys
from acme.errors import ClientError
from flask import current_app
@ -408,6 +409,47 @@ class S3DestinationPlugin(ExportDestinationPlugin):
account_number=self.get_option("accountNumber", options),
)
def upload_acme_token(self, token_path, token, options, **kwargs):
"""
This is called from the acme http challenge
:param self:
:param token_path:
:param token:
:param options:
:param kwargs:
:return:
"""
current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge")
function = f"{__name__}.{sys._getframe().f_code.co_name}"
account_number = self.get_option("accountNumber", options)
bucket_name = self.get_option("bucket", options)
prefix = self.get_option("prefix", options)
region = self.get_option("region", options)
filename = token_path.split("/")[-1]
if not prefix.endswith("/"):
prefix + "/"
res = s3.put(bucket_name=bucket_name,
region_name=region,
prefix=prefix + filename,
data=token,
encrypt=False,
account_number=account_number)
res = "Success" if res else "Failure"
log_data = {
"function": function,
"message": "check if any valid certificate is revoked",
"result": res,
"bucket_name": bucket_name,
"filename": filename
}
current_app.logger.info(log_data)
metrics.send(f"{function}", "counter", 1, metric_tags={"result": res,
"bucket_name": bucket_name,
"filename": filename})
class SNSNotificationPlugin(ExpirationNotificationPlugin):
title = "AWS SNS"

44
lemur/plugins/lemur_aws/s3.py
View File

@ -6,12 +6,15 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from botocore.exceptions import ClientError
from flask import current_app
from lemur.extensions import sentry
from .sts import sts_client
@sts_client("s3", service_type="resource")
def put(bucket_name, region, prefix, data, encrypt, **kwargs):
def put(bucket_name, region_name, prefix, data, encrypt, **kwargs):
"""
Use STS to write to an S3 bucket
"""
@ -32,4 +35,41 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs):
ServerSideEncryption="AES256",
)
else:
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
try:
bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control")
return True
except ClientError:
sentry.captureException()
return False
@sts_client("s3", service_type="client")
def delete(bucket_name, prefixed_object_name, **kwargs):
"""
Use STS to delete an object
"""
try:
response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name)
current_app.logger.debug(f"Delete data from S3."
f"Bucket: {bucket_name},"
f"Prefix: {prefixed_object_name},"
f"Status_code: {response}")
return response['ResponseMetadata']['HTTPStatusCode'] < 300
except ClientError:
sentry.captureException()
return False
@sts_client("s3", service_type="client")
def get(bucket_name, prefixed_object_name, **kwargs):
"""
Use STS to get an object
"""
try:
response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name)
current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name},"
f"object_name: {prefixed_object_name}")
return response['Body'].read().decode("utf-8")
except ClientError:
sentry.captureException()
return None

77
lemur/plugins/lemur_aws/tests/test_plugin.py
View File

@ -1,5 +1,82 @@
import boto3
from moto import mock_sts, mock_s3
def test_get_certificates(app):
from lemur.plugins.base import plugins
p = plugins.get("aws-s3")
assert p
@mock_sts()
@mock_s3()
def test_upload_acme_token(app):
from lemur.plugins.base import plugins
from lemur.plugins.lemur_aws.s3 import get
bucket = "public-bucket"
account = "123456789012"
prefix = "some-path/more-path/"
token_content = "Challenge"
token_name = "TOKEN"
token_path = ".well-known/acme-challenge/" + token_name
additional_options = [
{
"name": "bucket",
"value": bucket,
"type": "str",
"required": True,
"validation": r"[0-9a-z.-]{3,63}",
"helpMessage": "Must be a valid S3 bucket name!",
},
{
"name": "accountNumber",
"type": "str",
"value": account,
"required": True,
"validation": r"[0-9]{12}",
"helpMessage": "A valid AWS account number with permission to access S3",
},
{
"name": "region",
"type": "str",
"default": "us-east-1",
"required": False,
"helpMessage": "Region bucket exists",
"available": ["us-east-1", "us-west-2", "eu-west-1"],
},
{
"name": "encrypt",
"type": "bool",
"value": False,
"required": False,
"helpMessage": "Enable server side encryption",
"default": True,
},
{
"name": "prefix",
"type": "str",
"value": prefix,
"required": False,
"helpMessage": "Must be a valid S3 object prefix!",
},
]
s3_client = boto3.client('s3')
s3_client.create_bucket(Bucket=bucket)
p = plugins.get("aws-s3")
p.upload_acme_token(token_path=token_path,
token_content=token_content,
token=token_content,
options=additional_options)
response = get(bucket_name=bucket,
prefixed_object_name=prefix + token_name,
encrypt=False,
account_number=account)
# put data, and getting the same data
assert (response == token_content)

41
lemur/plugins/lemur_aws/tests/test_s3.py Normal file
View File

@ -0,0 +1,41 @@
import boto3
from moto import mock_sts, mock_s3
@mock_sts()
@mock_s3()
def test_put_delete_s3_object(app):
from lemur.plugins.lemur_aws.s3 import put, delete, get
bucket = "public-bucket"
region = "us-east-1"
account = "123456789012"
path = "some-path/foo"
data = "dummy data"
s3_client = boto3.client('s3')
s3_client.create_bucket(Bucket=bucket)
put(bucket_name=bucket,
region_name=region,
prefix=path,
data=data,
encrypt=False,
account_number=account,
region=region)
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
# put data, and getting the same data
assert (response == data)
response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account)
# attempting to get thccle wrong data
assert (response is None)
delete(bucket_name=bucket, prefixed_object_name=path, account_number=account)
response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account)
# delete data, and getting the same data
assert (response is None)

2
lemur/plugins/lemur_email/plugin.py
View File

@ -91,7 +91,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin):
"name": "recipients",
"type": "str",
"required": True,
"validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$",
"validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$",
"helpMessage": "Comma delimited list of email addresses",
}
]

2
lemur/plugins/lemur_sftp/plugin.py
View File

@ -48,7 +48,7 @@ class SFTPDestinationPlugin(DestinationPlugin):
"type": "int",
"required": True,
"helpMessage": "The SFTP port, default is 22.",
"validation": "^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})",
"validation": r"^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})",
"default": "22",
},
{

2
lemur/plugins/lemur_slack/plugin.py
View File

@ -89,7 +89,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin):
"name": "webhook",
"type": "str",
"required": True,
"validation": "^https:\/\/hooks\.slack\.com\/services\/.+$",
"validation": r"^https:\/\/hooks\.slack\.com\/services\/.+$",
"helpMessage": "The url Slack told you to use for this integration",
},
{