Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into ecc_changes

This commit is contained in:
charhate 2020-10-29 18:05:38 -07:00 committed by GitHub
parent 03dfbf535d 4330a42dd3
commit 69aa98c1c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 159 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
docs/administration.rst
View File

@ -292,6 +292,25 @@ Lemur supports sending certificate expiration notifications through SES and SMTP
you can send any mail. See: `Verifying Email Address in Amazon SES <http://docs.aws.amazon.com/ses/latest/DeveloperGuide/verify-email-addresses.html>`_
.. data:: LEMUR_SES_SOURCE_ARN
:noindex:
Specifies an ARN to use as the SourceArn when sending emails via SES.
.. note::
This parameter is only required if you're using a sending authorization with SES.
See: `Using sending authorization with Amazon SES <https://docs.aws.amazon.com/ses/latest/DeveloperGuide/sending-authorization.html>`_
.. data:: LEMUR_SES_REGION
:noindex:
Specifies a region for sending emails via SES.
.. note::
This parameter defaults to us-east-1 and is only required if you wish to use a different region.
.. data:: LEMUR_EMAIL
:noindex:
@ -671,6 +690,20 @@ If you are not using a metric provider you do not need to configure any of these
Plugin Specific Options
-----------------------
ACME Plugin
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. data:: ACME_DNS_PROVIDER_TYPES
:noindex:
Dictionary of ACME DNS Providers and their requirements.
.. data:: ACME_ENABLE_DELEGATED_CNAME
:noindex:
Enables delegated DNS domain validation using CNAMES. When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges.
Active Directory Certificate Services Plugin
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

5
gulp/build.js
View File

@ -237,7 +237,7 @@ gulp.task('addUrlContextPath',['addUrlContextPath:revreplace'], function(){
.forEach(function(file){
return gulp.src(file)
.pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
.pipe(gulpif(urlContextPathExists, replace('angular/', argv.urlContextPath + '/angular/')))
.pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/')))
.pipe(gulp.dest(function(file){
return file.base;
}))
@ -256,10 +256,9 @@ gulp.task('addUrlContextPath:revreplace', ['addUrlContextPath:revision'], functi
var manifest = gulp.src("lemur/static/dist/rev-manifest.json");
var urlContextPathExists = argv.urlContextPath ? true : false;
return gulp.src( "lemur/static/dist/index.html")
.pipe(gulpif(urlContextPathExists, revReplace({prefix: argv.urlContextPath + '/', manifest: manifest}, revReplace({manifest: manifest}))))
.pipe(gulp.dest('lemur/static/dist'));
})
gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']);
gulp.task('package', ['addUrlContextPath', 'package:strip']);
gulp.task('package', ['addUrlContextPath', 'package:strip']);

1
lemur/certificates/views.py
View File

@ -1155,6 +1155,7 @@ class NotificationCertificatesList(AuthenticatedResource):
)
parser.add_argument("creator", type=str, location="args")
parser.add_argument("show", type=str, location="args")
parser.add_argument("showExpired", type=int, location="args")
args = parser.parse_args()
args["notification_id"] = notification_id

3
lemur/destinations/schemas.py
View File

@ -31,6 +31,9 @@ class DestinationOutputSchema(LemurOutputSchema):
def fill_object(self, data):
if data:
data["plugin"]["pluginOptions"] = data["options"]
for option in data["plugin"]["pluginOptions"]:
if "export-plugin" in option["type"]:
option["value"]["pluginOptions"] = option["value"]["plugin_options"]
return data

9
lemur/destinations/service.py
View File

@ -41,12 +41,14 @@ def create(label, plugin_name, options, description=None):
return database.create(destination)
def update(destination_id, label, options, description):
def update(destination_id, label, plugin_name, options, description):
"""
Updates an existing destination.
:param destination_id: Lemur assigned ID
:param label: Destination common name
:param plugin_name:
:param options:
:param description:
:rtype : Destination
:return:
@ -54,6 +56,11 @@ def update(destination_id, label, options, description):
destination = get(destination_id)
destination.label = label
destination.plugin_name = plugin_name
# remove any sub-plugin objects before try to save the json options
for option in options:
if "plugin" in option["type"]:
del option["value"]["plugin_object"]
destination.options = options
destination.description = description

1
lemur/destinations/views.py
View File

@ -338,6 +338,7 @@ class Destinations(AuthenticatedResource):
return service.update(
destination_id,
data["label"],
data["plugin"]["slug"],
data["plugin"]["plugin_options"],
data["description"],
)

4
lemur/notifications/service.py
View File

@ -104,12 +104,13 @@ def create(label, plugin_name, options, description, certificates):
return database.create(notification)
def update(notification_id, label, options, description, active, certificates):
def update(notification_id, label, plugin_name, options, description, active, certificates):
"""
Updates an existing notification.
:param notification_id:
:param label: Notification label
:param plugin_name:
:param options:
:param description:
:param active:
@ -120,6 +121,7 @@ def update(notification_id, label, options, description, active, certificates):
notification = get(notification_id)
notification.label = label
notification.plugin_name = plugin_name
notification.options = options
notification.description = description
notification.active = active

1
lemur/notifications/views.py
View File

@ -340,6 +340,7 @@ class Notifications(AuthenticatedResource):
return service.update(
notification_id,
data["label"],
data["plugin"]["slug"],
data["plugin"]["plugin_options"],
data["description"],
data["active"],

87
lemur/plugins/lemur_acme/plugin.py
View File

@ -16,6 +16,7 @@ import json
import time
import OpenSSL.crypto
import dns.resolver
import josepy as jose
from acme import challenges, errors, messages
from acme.client import BackwardsCompatibleClientV2, ClientNetwork
@ -23,7 +24,6 @@ from acme.errors import PollError, TimeoutError, WildcardUnsupportedError
from acme.messages import Error as AcmeError
from botocore.exceptions import ClientError
from flask import current_app
from lemur.authorizations import service as authorization_service
from lemur.common.utils import generate_private_key
from lemur.dns_providers import service as dns_provider_service
@ -37,8 +37,9 @@ from retrying import retry
class AuthorizationRecord(object):
def __init__(self, host, authz, dns_challenge, change_id):
self.host = host
def __init__(self, domain, target_domain, authz, dns_challenge, change_id):
self.domain = domain
self.target_domain = target_domain
self.authz = authz
self.dns_challenge = dns_challenge
self.change_id = change_id
@ -91,19 +92,18 @@ class AcmeHandler(object):
self,
acme_client,
account_number,
host,
domain,
target_domain,
dns_provider,
order,
dns_provider_options,
):
current_app.logger.debug("Starting DNS challenge for {0}".format(host))
current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.")
change_ids = []
dns_challenges = self.get_dns_challenges(host, order.authorizations)
host_to_validate, _ = self.strip_wildcard(host)
host_to_validate = self.maybe_add_extension(
host_to_validate, dns_provider_options
)
dns_challenges = self.get_dns_challenges(domain, order.authorizations)
host_to_validate, _ = self.strip_wildcard(target_domain)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
if not dns_challenges:
sentry.captureException()
@ -111,15 +111,20 @@ class AcmeHandler(object):
raise Exception("Unable to determine DNS challenges from authorizations")
for dns_challenge in dns_challenges:
# Only prepend '_acme-challenge' if not using CNAME redirection
if domain == target_domain:
host_to_validate = dns_challenge.validation_domain_name(host_to_validate)
change_id = dns_provider.create_txt_record(
dns_challenge.validation_domain_name(host_to_validate),
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
account_number,
)
change_ids.append(change_id)
return AuthorizationRecord(
host, order.authorizations, dns_challenges, change_ids
domain, target_domain, order.authorizations, dns_challenges, change_ids
)
def complete_dns_challenge(self, acme_client, authz_record):
@ -128,11 +133,11 @@ class AcmeHandler(object):
authz_record.authz[0].body.identifier.value
)
)
dns_providers = self.dns_providers_for_domain.get(authz_record.host)
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
if not dns_providers:
metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1)
raise Exception(
"No DNS providers found for domain: {}".format(authz_record.host)
"No DNS providers found for domain: {}".format(authz_record.target_domain)
)
for dns_provider in dns_providers:
@ -160,7 +165,7 @@ class AcmeHandler(object):
verified = response.simple_verify(
dns_challenge.chall,
authz_record.host,
authz_record.target_domain,
acme_client.client.net.key.public_key(),
)
@ -311,12 +316,24 @@ class AcmeHandler(object):
authorizations = []
for domain in order_info.domains:
if not self.dns_providers_for_domain.get(domain):
# If CNAME exists, set host to the target address
target_domain = domain
if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False):
cname_result, _ = self.strip_wildcard(domain)
cname_result = challenges.DNS01().validation_domain_name(cname_result)
cname_result = self.get_cname(cname_result)
if cname_result:
target_domain = cname_result
self.autodetect_dns_providers(target_domain)
if not self.dns_providers_for_domain.get(target_domain):
metrics.send(
"get_authorizations_no_dns_provider_for_domain", "counter", 1
)
raise Exception("No DNS providers found for domain: {}".format(domain))
for dns_provider in self.dns_providers_for_domain[domain]:
raise Exception("No DNS providers found for domain: {}".format(target_domain))
for dns_provider in self.dns_providers_for_domain[target_domain]:
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
@ -324,6 +341,7 @@ class AcmeHandler(object):
acme_client,
account_number,
domain,
target_domain,
dns_provider_plugin,
order,
dns_provider.options,
@ -358,7 +376,7 @@ class AcmeHandler(object):
for authz_record in authorizations:
dns_challenges = authz_record.dns_challenge
for dns_challenge in dns_challenges:
dns_providers = self.dns_providers_for_domain.get(authz_record.host)
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_plugin = self.get_dns_provider(
@ -366,14 +384,14 @@ class AcmeHandler(object):
)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
host_to_validate, _ = self.strip_wildcard(authz_record.host)
host_to_validate = self.maybe_add_extension(
host_to_validate, dns_provider_options
)
host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
if authz_record.domain == authz_record.target_domain:
host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate)
dns_provider_plugin.delete_txt_record(
authz_record.change_id,
account_number,
dns_challenge.validation_domain_name(host_to_validate),
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
)
@ -392,23 +410,26 @@ class AcmeHandler(object):
:return:
"""
for authz_record in authorizations:
dns_providers = self.dns_providers_for_domain.get(authz_record.host)
dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain)
for dns_provider in dns_providers:
# Grab account number (For Route53)
dns_provider_options = json.loads(dns_provider.credentials)
account_number = dns_provider_options.get("account_id")
dns_challenges = authz_record.dns_challenge
host_to_validate, _ = self.strip_wildcard(authz_record.host)
host_to_validate, _ = self.strip_wildcard(authz_record.target_domain)
host_to_validate = self.maybe_add_extension(
host_to_validate, dns_provider_options
)
dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type)
for dns_challenge in dns_challenges:
if authz_record.domain == authz_record.target_domain:
host_to_validate = dns_challenge.validation_domain_name(host_to_validate),
try:
dns_provider_plugin.delete_txt_record(
authz_record.change_id,
account_number,
dns_challenge.validation_domain_name(host_to_validate),
host_to_validate,
dns_challenge.validation(acme_client.client.net.key),
)
except Exception as e:
@ -431,6 +452,18 @@ class AcmeHandler(object):
raise UnknownProvider("No such DNS provider: {}".format(type))
return provider
def get_cname(self, domain):
"""
:param domain: Domain name to look up a CNAME for.
:return: First CNAME target or False if no CNAME record exists.
"""
try:
result = dns.resolver.query(domain, 'CNAME')
if len(result) > 0:
return str(result[0].target).rstrip('.')
except dns.exception.DNSException:
return False
class ACMEIssuerPlugin(IssuerPlugin):
title = "Acme"

16
lemur/plugins/lemur_acme/tests/test_acme.py
View File

@ -49,7 +49,7 @@ class TestAcme(unittest.TestCase):
self.assertEqual(expected, result)
def test_authz_record(self):
a = plugin.AuthorizationRecord("host", "authz", "challenge", "id")
a = plugin.AuthorizationRecord("domain", "host", "authz", "challenge", "id")
self.assertEqual(type(a), plugin.AuthorizationRecord)
@patch("acme.client.Client")
@ -79,7 +79,7 @@ class TestAcme(unittest.TestCase):
iterator = iter(values)
iterable.__iter__.return_value = iterator
result = self.acme.start_dns_challenge(
mock_acme, "accountid", "host", mock_dns_provider, mock_order, {}
mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {}
)
self.assertEqual(type(result), plugin.AuthorizationRecord)
@ -97,7 +97,7 @@ class TestAcme(unittest.TestCase):
mock_authz.dns_challenge.response = Mock()
mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True)
mock_authz.authz = []
mock_authz.host = "www.test.com"
mock_authz.target_domain = "www.test.com"
mock_authz_record = Mock()
mock_authz_record.body.identifier.value = "test"
mock_authz.authz.append(mock_authz_record)
@ -121,7 +121,7 @@ class TestAcme(unittest.TestCase):
mock_authz.dns_challenge.response = Mock()
mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False)
mock_authz.authz = []
mock_authz.host = "www.test.com"
mock_authz.target_domain = "www.test.com"
mock_authz_record = Mock()
mock_authz_record.body.identifier.value = "test"
mock_authz.authz.append(mock_authz_record)
@ -270,11 +270,9 @@ class TestAcme(unittest.TestCase):
result, [options["common_name"], "test2.netflix.net"]
)
@patch(
"lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge",
return_value="test",
)
def test_get_authorizations(self, mock_start_dns_challenge):
@patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", return_value="test")
@patch("lemur.plugins.lemur_acme.plugin.current_app", return_value=False)
def test_get_authorizations(self, mock_current_app, mock_start_dns_challenge):
mock_order = Mock()
mock_order.body.identifiers = []
mock_domain = Mock()

9
lemur/plugins/lemur_aws/sns.py
View File

@ -15,16 +15,18 @@ from flask import current_app
def publish(topic_arn, certificates, notification_type, **kwargs):
sns_client = boto3.client("sns", **kwargs)
message_ids = {}
subject = "Lemur: {0} Notification".format(notification_type.capitalize())
for certificate in certificates:
message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type)
message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject)
return message_ids
def publish_single(sns_client, topic_arn, certificate, notification_type):
def publish_single(sns_client, topic_arn, certificate, notification_type, subject):
response = sns_client.publish(
TopicArn=topic_arn,
Message=format_message(certificate, notification_type),
Subject=subject,
)
response_code = response["ResponseMetadata"]["HTTPStatusCode"]
@ -48,8 +50,9 @@ def format_message(certificate, notification_type):
json_message = {
"notification_type": notification_type,
"certificate_name": certificate["name"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"), # 2047-12-T22:00:00
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), # 2047-12-31T22:00:00
"endpoints_detected": len(certificate["endpoints"]),
"owner": certificate["owner"],
"details": create_certificate_url(certificate["name"])
}
return json.dumps(json_message)

7
lemur/plugins/lemur_aws/tests/test_sns.py
View File

@ -20,8 +20,9 @@ def test_format(certificate, endpoint):
expected_message = {
"notification_type": "expiration",
"certificate_name": certificate["name"],
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"),
"expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"),
"endpoints_detected": 0,
"owner": certificate["owner"],
"details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"])
}
assert expected_message == json.loads(format_message(certificate, "expiration"))
@ -57,7 +58,9 @@ def test_publish(certificate, endpoint):
expected_message_id = message_ids[certificate["name"]]
actual_message = next(
(m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None)
assert json.loads(actual_message["Body"])["Message"] == format_message(certificate, "expiration")
actual_json = json.loads(actual_message["Body"])
assert actual_json["Message"] == format_message(certificate, "expiration")
assert actual_json["Subject"] == "Lemur: Expiration Notification"
def get_options():

21
lemur/plugins/lemur_email/plugin.py
View File

@ -38,7 +38,7 @@ def render_html(template_name, options, certificates):
def send_via_smtp(subject, body, targets):
"""
Attempts to deliver email notification via SES service.
Attempts to deliver email notification via SMTP.
:param subject:
:param body:
@ -55,21 +55,26 @@ def send_via_smtp(subject, body, targets):
def send_via_ses(subject, body, targets):
"""
Attempts to deliver email notification via SMTP.
Attempts to deliver email notification via SES service.
:param subject:
:param body:
:param targets:
:return:
"""
client = boto3.client("ses", region_name="us-east-1")
client.send_email(
Source=current_app.config.get("LEMUR_EMAIL"),
Destination={"ToAddresses": targets},
Message={
ses_region = current_app.config.get("LEMUR_SES_REGION", "us-east-1")
client = boto3.client("ses", region_name=ses_region)
source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN")
args = {
"Source": current_app.config.get("LEMUR_EMAIL"),
"Destination": {"ToAddresses": targets},
"Message": {
"Subject": {"Data": subject, "Charset": "UTF-8"},
"Body": {"Html": {"Data": body, "Charset": "UTF-8"}},
},
)
}
if source_arn:
args["SourceArn"] = source_arn
client.send_email(**args)
class EmailNotificationPlugin(ExpirationNotificationPlugin):

4
lemur/sources/service.py
View File

@ -264,13 +264,14 @@ def create(label, plugin_name, options, description=None):
return database.create(source)
def update(source_id, label, options, description):
def update(source_id, label, plugin_name, options, description):
"""
Updates an existing source.
:param source_id: Lemur assigned ID
:param label: Source common name
:param options:
:param plugin_name:
:param description:
:rtype : Source
:return:
@ -278,6 +279,7 @@ def update(source_id, label, options, description):
source = get(source_id)
source.label = label
source.plugin_name = plugin_name
source.options = options
source.description = description

1
lemur/sources/views.py
View File

@ -284,6 +284,7 @@ class Sources(AuthenticatedResource):
return service.update(
source_id,
data["label"],
data["plugin"]["slug"],
data["plugin"]["plugin_options"],
data["description"],
)

12
lemur/static/app/angular/destinations/destination/destination.js vendored
View File

@ -52,19 +52,19 @@ angular.module('lemur')
if (plugin.slug === $scope.destination.plugin.slug) {
plugin.pluginOptions = $scope.destination.plugin.pluginOptions;
$scope.destination.plugin = plugin;
_.each($scope.destination.plugin.pluginOptions, function (option) {
if (option.type === 'export-plugin') {
PluginService.getByType('export').then(function (plugins) {
$scope.exportPlugins = plugins;
PluginService.getByType('export').then(function (plugins) {
$scope.exportPlugins = plugins;
_.each($scope.destination.plugin.pluginOptions, function (option) {
if (option.type === 'export-plugin') {
_.each($scope.exportPlugins, function (plugin) {
if (plugin.slug === option.value.slug) {
plugin.pluginOptions = option.value.pluginOptions;
option.value = plugin;
}
});
});
}
}
});
});
}
});

14
lemur/static/app/angular/notifications/notification/notification.js vendored
View File

@ -42,8 +42,8 @@ angular.module('lemur')
PluginService.getByType('notification').then(function (plugins) {
$scope.plugins = plugins;
_.each($scope.plugins, function (plugin) {
if (plugin.slug === $scope.notification.pluginName) {
plugin.pluginOptions = $scope.notification.notificationOptions;
if (plugin.slug === $scope.notification.plugin.slug) {
plugin.pluginOptions = $scope.notification.plugin.pluginOptions;
$scope.notification.plugin = plugin;
}
});
@ -51,16 +51,6 @@ angular.module('lemur')
NotificationService.getCertificates(notification);
});
PluginService.getByType('notification').then(function (plugins) {
$scope.plugins = plugins;
_.each($scope.plugins, function (plugin) {
if (plugin.slug === $scope.notification.pluginName) {
plugin.pluginOptions = $scope.notification.notificationOptions;
$scope.notification.plugin = plugin;
}
});
});
$scope.save = function (notification) {
NotificationService.update(notification).then(
function () {

4
lemur/static/app/angular/notifications/services.js vendored
View File

@ -27,7 +27,7 @@ angular.module('lemur')
};
NotificationService.getCertificates = function (notification) {
notification.getList('certificates').then(function (certificates) {
notification.getList('certificates', {showExpired: 0}).then(function (certificates) {
notification.certificates = certificates;
});
};
@ -40,7 +40,7 @@ angular.module('lemur')
NotificationService.loadMoreCertificates = function (notification, page) {
notification.getList('certificates', {page: page}).then(function (certificates) {
notification.getList('certificates', {page: page, showExpired: 0}).then(function (certificates) {
_.each(certificates, function (certificate) {
notification.roles.push(certificate);
});

12
lemur/static/app/angular/sources/source/source.js vendored
View File

@ -41,22 +41,14 @@ angular.module('lemur')
PluginService.getByType('source').then(function (plugins) {
$scope.plugins = plugins;
_.each($scope.plugins, function (plugin) {
if (plugin.slug === $scope.source.pluginName) {
if (plugin.slug === $scope.source.plugin.slug) {
plugin.pluginOptions = $scope.source.plugin.pluginOptions;
$scope.source.plugin = plugin;
}
});
});
});
PluginService.getByType('source').then(function (plugins) {
$scope.plugins = plugins;
_.each($scope.plugins, function (plugin) {
if (plugin.slug === $scope.source.pluginName) {
$scope.source.plugin = plugin;
}
});
});
$scope.save = function (source) {
SourceService.update(source).then(
function () {