From 591c8cf5249cb5d8b1b075f8b0c52863795913d0 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Mon, 19 Oct 2020 22:03:43 +0200 Subject: [PATCH 01/18] Do not add urlContextPath to relative path --- gulp/build.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gulp/build.js b/gulp/build.js index eed59503..5aca8094 100644 --- a/gulp/build.js +++ b/gulp/build.js @@ -237,7 +237,7 @@ gulp.task('addUrlContextPath',['addUrlContextPath:revreplace'], function(){ .forEach(function(file){ return gulp.src(file) .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/'))) - .pipe(gulpif(urlContextPathExists, replace('angular/', argv.urlContextPath + '/angular/'))) + .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/'))) .pipe(gulp.dest(function(file){ return file.base; })) @@ -256,10 +256,9 @@ gulp.task('addUrlContextPath:revreplace', ['addUrlContextPath:revision'], functi var manifest = gulp.src("lemur/static/dist/rev-manifest.json"); var urlContextPathExists = argv.urlContextPath ? true : false; return gulp.src( "lemur/static/dist/index.html") - .pipe(gulpif(urlContextPathExists, revReplace({prefix: argv.urlContextPath + '/', manifest: manifest}, revReplace({manifest: manifest})))) .pipe(gulp.dest('lemur/static/dist')); }) gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']); -gulp.task('package', ['addUrlContextPath', 'package:strip']); \ No newline at end of file +gulp.task('package', ['addUrlContextPath', 'package:strip']); From 71df6b8560079934802553af929d3b3cc2f0d90c Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 22 Oct 2020 18:15:26 -0700 Subject: [PATCH 02/18] Fix plugin field on notification edit --- .../notifications/notification/notification.js | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/lemur/static/app/angular/notifications/notification/notification.js b/lemur/static/app/angular/notifications/notification/notification.js index d3cfac9b..9cf88cbf 100644 --- a/lemur/static/app/angular/notifications/notification/notification.js +++ b/lemur/static/app/angular/notifications/notification/notification.js @@ -42,8 +42,8 @@ angular.module('lemur') PluginService.getByType('notification').then(function (plugins) { $scope.plugins = plugins; _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.notification.pluginName) { - plugin.pluginOptions = $scope.notification.notificationOptions; + if (plugin.slug === $scope.notification.plugin.slug) { + plugin.pluginOptions = $scope.notification.plugin.pluginOptions; $scope.notification.plugin = plugin; } }); @@ -51,16 +51,6 @@ angular.module('lemur') NotificationService.getCertificates(notification); }); - PluginService.getByType('notification').then(function (plugins) { - $scope.plugins = plugins; - _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.notification.pluginName) { - plugin.pluginOptions = $scope.notification.notificationOptions; - $scope.notification.plugin = plugin; - } - }); - }); - $scope.save = function (notification) { NotificationService.update(notification).then( function () { From 37f05a89f22d041a7d49bfac5f62519b4f5f65f9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2020 09:23:10 +0000 Subject: [PATCH 03/18] Bump botocore from 1.18.16 to 1.18.18 Bumps [botocore](https://github.com/boto/botocore) from 1.18.16 to 1.18.18. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.18.16...1.18.18) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 38b62198..24504233 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,7 +18,7 @@ beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.15.16 # via -r requirements.txt -botocore==1.18.16 # via -r requirements.txt, boto3, s3transfer +botocore==1.18.18 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 8df4f5d1..94bdabc4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in boto3==1.15.16 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.18.16 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.18.18 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index d323b40f..0949fbb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.15.16 # via -r requirements.in -botocore==1.18.16 # via -r requirements.in, boto3, s3transfer +botocore==1.18.18 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From fa62023b2db0a868af31ae380d6f25f30a9f11dc Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:16:23 -0700 Subject: [PATCH 04/18] fixing the time bug, sub-second to second, and month to minute! --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index ee917dac..091539de 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -177,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:ss") + "Z" }, "organization": { "name": options["organization"], From 3290d6634bda9591e76d8f857e5ae8162848f28c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:16:38 -0700 Subject: [PATCH 05/18] fixing testing --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 059cdd82..fd07ea2b 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -123,7 +123,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority): "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } @@ -159,7 +159,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } From 6723e3c80dcdee473d1d23be444180bdb6a9f61b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:18:24 -0700 Subject: [PATCH 06/18] now fixing the month to minute bug --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 091539de..ec3a0792 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -177,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:ss") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "organization": { "name": options["organization"], From 3f765b51efdce6730f608c1762d09212347908d3 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Mon, 26 Oct 2020 11:27:18 -0700 Subject: [PATCH 07/18] Fix sources and destinations, and allow actually updating the notification type --- lemur/destinations/schemas.py | 3 +++ lemur/destinations/service.py | 9 ++++++++- lemur/destinations/views.py | 1 + lemur/notifications/service.py | 4 +++- lemur/notifications/views.py | 1 + lemur/sources/service.py | 4 +++- lemur/sources/views.py | 1 + .../angular/destinations/destination/destination.js | 12 ++++++------ lemur/static/app/angular/sources/source/source.js | 12 ++---------- 9 files changed, 28 insertions(+), 19 deletions(-) diff --git a/lemur/destinations/schemas.py b/lemur/destinations/schemas.py index cc46ecd4..22187a09 100644 --- a/lemur/destinations/schemas.py +++ b/lemur/destinations/schemas.py @@ -31,6 +31,9 @@ class DestinationOutputSchema(LemurOutputSchema): def fill_object(self, data): if data: data["plugin"]["pluginOptions"] = data["options"] + for option in data["plugin"]["pluginOptions"]: + if "export-plugin" in option["type"]: + option["value"]["pluginOptions"] = option["value"]["plugin_options"] return data diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 92162f4b..7bae57f0 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -41,12 +41,14 @@ def create(label, plugin_name, options, description=None): return database.create(destination) -def update(destination_id, label, options, description): +def update(destination_id, label, plugin_name, options, description): """ Updates an existing destination. :param destination_id: Lemur assigned ID :param label: Destination common name + :param plugin_name: + :param options: :param description: :rtype : Destination :return: @@ -54,6 +56,11 @@ def update(destination_id, label, options, description): destination = get(destination_id) destination.label = label + destination.plugin_name = plugin_name + # remove any sub-plugin objects before try to save the json options + for option in options: + if "plugin" in option["type"]: + del option["value"]["plugin_object"] destination.options = options destination.description = description diff --git a/lemur/destinations/views.py b/lemur/destinations/views.py index 0b0559fe..072ff34e 100644 --- a/lemur/destinations/views.py +++ b/lemur/destinations/views.py @@ -338,6 +338,7 @@ class Destinations(AuthenticatedResource): return service.update( destination_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], ) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index ac624d1c..34edccc0 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -104,12 +104,13 @@ def create(label, plugin_name, options, description, certificates): return database.create(notification) -def update(notification_id, label, options, description, active, certificates): +def update(notification_id, label, plugin_name, options, description, active, certificates): """ Updates an existing notification. :param notification_id: :param label: Notification label + :param plugin_name: :param options: :param description: :param active: @@ -120,6 +121,7 @@ def update(notification_id, label, options, description, active, certificates): notification = get(notification_id) notification.label = label + notification.plugin_name = plugin_name notification.options = options notification.description = description notification.active = active diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index cdabb4d4..f6eef655 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -340,6 +340,7 @@ class Notifications(AuthenticatedResource): return service.update( notification_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], data["active"], diff --git a/lemur/sources/service.py b/lemur/sources/service.py index fafa6f5a..be0de049 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -264,13 +264,14 @@ def create(label, plugin_name, options, description=None): return database.create(source) -def update(source_id, label, options, description): +def update(source_id, label, plugin_name, options, description): """ Updates an existing source. :param source_id: Lemur assigned ID :param label: Source common name :param options: + :param plugin_name: :param description: :rtype : Source :return: @@ -278,6 +279,7 @@ def update(source_id, label, options, description): source = get(source_id) source.label = label + source.plugin_name = plugin_name source.options = options source.description = description diff --git a/lemur/sources/views.py b/lemur/sources/views.py index b74c4d80..3b4deab7 100644 --- a/lemur/sources/views.py +++ b/lemur/sources/views.py @@ -284,6 +284,7 @@ class Sources(AuthenticatedResource): return service.update( source_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], ) diff --git a/lemur/static/app/angular/destinations/destination/destination.js b/lemur/static/app/angular/destinations/destination/destination.js index 21f624c8..93a7f80e 100644 --- a/lemur/static/app/angular/destinations/destination/destination.js +++ b/lemur/static/app/angular/destinations/destination/destination.js @@ -52,19 +52,19 @@ angular.module('lemur') if (plugin.slug === $scope.destination.plugin.slug) { plugin.pluginOptions = $scope.destination.plugin.pluginOptions; $scope.destination.plugin = plugin; - _.each($scope.destination.plugin.pluginOptions, function (option) { - if (option.type === 'export-plugin') { - PluginService.getByType('export').then(function (plugins) { - $scope.exportPlugins = plugins; + PluginService.getByType('export').then(function (plugins) { + $scope.exportPlugins = plugins; + _.each($scope.destination.plugin.pluginOptions, function (option) { + if (option.type === 'export-plugin') { _.each($scope.exportPlugins, function (plugin) { if (plugin.slug === option.value.slug) { plugin.pluginOptions = option.value.pluginOptions; option.value = plugin; } }); - }); - } + } + }); }); } }); diff --git a/lemur/static/app/angular/sources/source/source.js b/lemur/static/app/angular/sources/source/source.js index 1d5c1641..8ea381f8 100644 --- a/lemur/static/app/angular/sources/source/source.js +++ b/lemur/static/app/angular/sources/source/source.js @@ -41,22 +41,14 @@ angular.module('lemur') PluginService.getByType('source').then(function (plugins) { $scope.plugins = plugins; _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.source.pluginName) { + if (plugin.slug === $scope.source.plugin.slug) { + plugin.pluginOptions = $scope.source.plugin.pluginOptions; $scope.source.plugin = plugin; } }); }); }); - PluginService.getByType('source').then(function (plugins) { - $scope.plugins = plugins; - _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.source.pluginName) { - $scope.source.plugin = plugin; - } - }); - }); - $scope.save = function (source) { SourceService.update(source).then( function () { From 749aa772ba459a40dd59cc3b3ebd1e73cee99ef2 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 26 Oct 2020 11:57:33 -0700 Subject: [PATCH 08/18] First change to get CNAME redirection working --- docs/administration.rst | 14 ++++++++++++++ lemur/plugins/lemur_acme/plugin.py | 22 +++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 846a4c34..80d88feb 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -620,6 +620,20 @@ If you are not using a metric provider you do not need to configure any of these Plugin Specific Options ----------------------- +ACME Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. data:: ACME_DNS_PROVIDER_TYPES + :noindex: + + Dictionary of ACME DNS Providers and their requirements. + +.. data:: ACME_ENABLE_DELEGATED_CNAME + :noindex: + + Enables delegated DNS domain validation using CNAMES. When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges. + + Active Directory Certificate Services Plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 16d61a0f..9177d6e8 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -16,6 +16,7 @@ import json import time import OpenSSL.crypto +import dns.resolver import josepy as jose from acme import challenges, errors, messages from acme.client import BackwardsCompatibleClientV2, ClientNetwork @@ -23,7 +24,6 @@ from acme.errors import PollError, TimeoutError, WildcardUnsupportedError from acme.messages import Error as AcmeError from botocore.exceptions import ClientError from flask import current_app - from lemur.authorizations import service as authorization_service from lemur.common.utils import generate_private_key from lemur.dns_providers import service as dns_provider_service @@ -287,6 +287,13 @@ class AcmeHandler(object): authorizations = [] for domain in order_info.domains: + + # Replace domain if doing CNAME delegation + if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): + cname = self.get_cname(domain) + if cname: + domain = cname + if not self.dns_providers_for_domain.get(domain): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 @@ -407,6 +414,19 @@ class AcmeHandler(object): raise UnknownProvider("No such DNS provider: {}".format(type)) return provider + def get_cname(self, domain): + """ + :param domain: Domain name to look up a CNAME for. + :param record_type: Type of DNS record to lookup. + :return: First CNAME target or False if no CNAME record exists. + """ + try: + result = dns.resolver.query(domain, 'CNAME') + if len(result) > 0: + return str(result[0].target).rstrip('.') + except dns.exception.DNSException: + return False + class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" From acc95a4b6677548872fdf143333e36e2e551bc9d Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 28 Oct 2020 16:12:27 -0700 Subject: [PATCH 09/18] Fix notification view to actually show associated certs --- lemur/certificates/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 18746636..a066f20f 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -1155,6 +1155,7 @@ class NotificationCertificatesList(AuthenticatedResource): ) parser.add_argument("creator", type=str, location="args") parser.add_argument("show", type=str, location="args") + parser.add_argument("showExpired", type=int, location="args") args = parser.parse_args() args["notification_id"] = notification_id From 5e696f36bf8762da59583e85cb2ad4647417bca2 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 28 Oct 2020 16:34:31 -0700 Subject: [PATCH 10/18] Add ability to override SourceArnn for SES --- docs/administration.rst | 10 ++++++++++ lemur/plugins/lemur_email/plugin.py | 18 +++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 724b136f..a1eba56e 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -285,6 +285,16 @@ Lemur supports sending certificate expiration notifications through SES and SMTP you can send any mail. See: `Verifying Email Address in Amazon SES `_ +.. data:: LEMUR_SES_SOURCE_ARN + :noindex: + + Specifies an ARN to use as the SourceArn when sending emails via SES. + + .. note:: + This parameter is only required if you're using a sending authorization with SES. + See: `Using sending authorization with Amazon SES `_ + + .. data:: LEMUR_EMAIL :noindex: diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 5b9c188e..39e76932 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -38,7 +38,7 @@ def render_html(template_name, options, certificates): def send_via_smtp(subject, body, targets): """ - Attempts to deliver email notification via SES service. + Attempts to deliver email notification via SMTP. :param subject: :param body: @@ -55,21 +55,25 @@ def send_via_smtp(subject, body, targets): def send_via_ses(subject, body, targets): """ - Attempts to deliver email notification via SMTP. + Attempts to deliver email notification via SES service. :param subject: :param body: :param targets: :return: """ client = boto3.client("ses", region_name="us-east-1") - client.send_email( - Source=current_app.config.get("LEMUR_EMAIL"), - Destination={"ToAddresses": targets}, - Message={ + source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN") + args = { + "Source": current_app.config.get("LEMUR_EMAIL"), + "Destination": {"ToAddresses": targets}, + "Message": { "Subject": {"Data": subject, "Charset": "UTF-8"}, "Body": {"Html": {"Data": body, "Charset": "UTF-8"}}, }, - ) + } + if source_arn: + args["SourceArn"] = source_arn + client.send_email(**args) class EmailNotificationPlugin(ExpirationNotificationPlugin): From 3e492e6310496e26c85ca1d42223b3e59c89322c Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 28 Oct 2020 17:09:54 -0700 Subject: [PATCH 11/18] Add ability to override SES region --- docs/administration.rst | 9 +++++++++ lemur/plugins/lemur_email/plugin.py | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index a1eba56e..7e94a4db 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -295,6 +295,15 @@ Lemur supports sending certificate expiration notifications through SES and SMTP See: `Using sending authorization with Amazon SES `_ +.. data:: LEMUR_SES_REGION + :noindex: + + Specifies a region for sending emails via SES. + + .. note:: + This parameter defaults to us-east-1 and is only required if you wish to use a different region. + + .. data:: LEMUR_EMAIL :noindex: diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 39e76932..a9e35d16 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -61,7 +61,10 @@ def send_via_ses(subject, body, targets): :param targets: :return: """ - client = boto3.client("ses", region_name="us-east-1") + ses_region = current_app.config.get("LEMUR_SES_REGION") + if not ses_region: + ses_region = "us-east-1" + client = boto3.client("ses", region_name=ses_region) source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN") args = { "Source": current_app.config.get("LEMUR_EMAIL"), From b47667b73e08f47a649c79427a252e0a628ec831 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 28 Oct 2020 20:51:35 -0700 Subject: [PATCH 12/18] cname redirection working --- lemur/plugins/lemur_acme/plugin.py | 48 ++++++++++++++------- lemur/plugins/lemur_acme/tests/test_acme.py | 12 +++--- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 2a09dbdf..4105465a 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -37,7 +37,8 @@ from retrying import retry class AuthorizationRecord(object): - def __init__(self, host, authz, dns_challenge, change_id): + def __init__(self, domain, host, authz, dns_challenge, change_id): + self.domain = domain self.host = host self.authz = authz self.dns_challenge = dns_challenge @@ -91,6 +92,7 @@ class AcmeHandler(object): self, acme_client, account_number, + domain, host, dns_provider, order, @@ -99,11 +101,9 @@ class AcmeHandler(object): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) change_ids = [] - dns_challenges = self.get_dns_challenges(host, order.authorizations) + dns_challenges = self.get_dns_challenges(domain, order.authorizations) host_to_validate, _ = self.strip_wildcard(host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) if not dns_challenges: sentry.captureException() @@ -111,15 +111,20 @@ class AcmeHandler(object): raise Exception("Unable to determine DNS challenges from authorizations") for dns_challenge in dns_challenges: + + # Only prepend '_acme-challenge' if not using CNAME redirection + if domain == host: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate) + change_id = dns_provider.create_txt_record( - dns_challenge.validation_domain_name(host_to_validate), + host_to_validate, dns_challenge.validation(acme_client.client.net.key), account_number, ) change_ids.append(change_id) return AuthorizationRecord( - host, order.authorizations, dns_challenges, change_ids + domain, host, order.authorizations, dns_challenges, change_ids ) def complete_dns_challenge(self, acme_client, authz_record): @@ -312,18 +317,23 @@ class AcmeHandler(object): for domain in order_info.domains: - # Replace domain if doing CNAME delegation + # If CNAME exists, set host to the target address + host = domain if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): - cname = self.get_cname(domain) - if cname: - domain = cname + val_domain, _ = self.strip_wildcard(domain) + val_domain = challenges.DNS01().validation_domain_name(val_domain) + cname_res = self.get_cname(val_domain) + if cname_res: + host = cname_res + self.autodetect_dns_providers(host) - if not self.dns_providers_for_domain.get(domain): + if not self.dns_providers_for_domain.get(host): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 ) - raise Exception("No DNS providers found for domain: {}".format(domain)) - for dns_provider in self.dns_providers_for_domain[domain]: + raise Exception("No DNS providers found for domain: {}".format(host)) + + for dns_provider in self.dns_providers_for_domain[host]: dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") @@ -331,6 +341,7 @@ class AcmeHandler(object): acme_client, account_number, domain, + host, dns_provider_plugin, order, dns_provider.options, @@ -377,10 +388,12 @@ class AcmeHandler(object): host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) + if authz_record.domain == authz_record.host: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate), dns_provider_plugin.delete_txt_record( authz_record.change_id, account_number, - dns_challenge.validation_domain_name(host_to_validate), + host_to_validate, dns_challenge.validation(acme_client.client.net.key), ) @@ -409,13 +422,16 @@ class AcmeHandler(object): host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) for dns_challenge in dns_challenges: + if authz_record.domain == authz_record.host: + host_to_validate = dns_challenge.validation_domain_name(host_to_validate), try: dns_provider_plugin.delete_txt_record( authz_record.change_id, account_number, - dns_challenge.validation_domain_name(host_to_validate), + host_to_validate, dns_challenge.validation(acme_client.client.net.key), ) except Exception as e: diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index ab246563..cce97d32 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -49,7 +49,7 @@ class TestAcme(unittest.TestCase): self.assertEqual(expected, result) def test_authz_record(self): - a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") + a = plugin.AuthorizationRecord("domain", "host", "authz", "challenge", "id") self.assertEqual(type(a), plugin.AuthorizationRecord) @patch("acme.client.Client") @@ -79,7 +79,7 @@ class TestAcme(unittest.TestCase): iterator = iter(values) iterable.__iter__.return_value = iterator result = self.acme.start_dns_challenge( - mock_acme, "accountid", "host", mock_dns_provider, mock_order, {} + mock_acme, "accountid", "domain", "host", mock_dns_provider, mock_order, {} ) self.assertEqual(type(result), plugin.AuthorizationRecord) @@ -270,11 +270,9 @@ class TestAcme(unittest.TestCase): result, [options["common_name"], "test2.netflix.net"] ) - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", - return_value="test", - ) - def test_get_authorizations(self, mock_start_dns_challenge): + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", return_value="test") + @patch("lemur.plugins.lemur_acme.plugin.current_app", return_value=False) + def test_get_authorizations(self, mock_current_app, mock_start_dns_challenge): mock_order = Mock() mock_order.body.identifiers = [] mock_domain = Mock() From 33a006bbeba013e4e3f99c5f257458d9199c942a Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 28 Oct 2020 22:24:37 -0700 Subject: [PATCH 13/18] fixing delete with optional validation --- lemur/plugins/lemur_acme/plugin.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 4105465a..63fa5626 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -385,11 +385,9 @@ class AcmeHandler(object): dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") host_to_validate, _ = self.strip_wildcard(authz_record.host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) + host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) if authz_record.domain == authz_record.host: - host_to_validate = dns_challenge.validation_domain_name(host_to_validate), + host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate) dns_provider_plugin.delete_txt_record( authz_record.change_id, account_number, From 78afc060aeb1fdbc44890b3571746c9327587175 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 29 Oct 2020 13:41:47 -0700 Subject: [PATCH 14/18] Add subject for SNS messages and correct date format --- lemur/plugins/lemur_aws/sns.py | 9 ++++++--- lemur/plugins/lemur_aws/tests/test_sns.py | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index c98bbc0c..14109c11 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -15,16 +15,18 @@ from flask import current_app def publish(topic_arn, certificates, notification_type, **kwargs): sns_client = boto3.client("sns", **kwargs) message_ids = {} + subject = "Lemur: {0} Notification".format(notification_type.capitalize()) for certificate in certificates: - message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type) + message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject) return message_ids -def publish_single(sns_client, topic_arn, certificate, notification_type): +def publish_single(sns_client, topic_arn, certificate, notification_type, subject): response = sns_client.publish( TopicArn=topic_arn, Message=format_message(certificate, notification_type), + Subject=subject, ) response_code = response["ResponseMetadata"]["HTTPStatusCode"] @@ -48,8 +50,9 @@ def format_message(certificate, notification_type): json_message = { "notification_type": notification_type, "certificate_name": certificate["name"], - "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"), # 2047-12-T22:00:00 + "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), # 2047-12-31T22:00:00 "endpoints_detected": len(certificate["endpoints"]), + "owner": certificate["owner"], "details": create_certificate_url(certificate["name"]) } return json.dumps(json_message) diff --git a/lemur/plugins/lemur_aws/tests/test_sns.py b/lemur/plugins/lemur_aws/tests/test_sns.py index ce05c33c..59ef30f2 100644 --- a/lemur/plugins/lemur_aws/tests/test_sns.py +++ b/lemur/plugins/lemur_aws/tests/test_sns.py @@ -20,8 +20,9 @@ def test_format(certificate, endpoint): expected_message = { "notification_type": "expiration", "certificate_name": certificate["name"], - "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-ddTHH:mm:ss"), + "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), "endpoints_detected": 0, + "owner": certificate["owner"], "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) } assert expected_message == json.loads(format_message(certificate, "expiration")) @@ -57,7 +58,9 @@ def test_publish(certificate, endpoint): expected_message_id = message_ids[certificate["name"]] actual_message = next( (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None) - assert json.loads(actual_message["Body"])["Message"] == format_message(certificate, "expiration") + actual_json = json.loads(actual_message["Body"]) + assert actual_json["Message"] == format_message(certificate, "expiration") + assert actual_json["Subject"] == "Lemur: Expiration Notification" def get_options(): From 45cc9528d28416155197d72653996236c8843180 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 29 Oct 2020 13:48:43 -0700 Subject: [PATCH 15/18] Cleaner syntax for default region --- lemur/plugins/lemur_email/plugin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index a9e35d16..f380c82e 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -61,9 +61,7 @@ def send_via_ses(subject, body, targets): :param targets: :return: """ - ses_region = current_app.config.get("LEMUR_SES_REGION") - if not ses_region: - ses_region = "us-east-1" + ses_region = current_app.config.get("LEMUR_SES_REGION", "us-east-1") client = boto3.client("ses", region_name=ses_region) source_arn = current_app.config.get("LEMUR_SES_SOURCE_ARN") args = { From 2b91077d9217a55a8ca022027091e7e0b88e025f Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 29 Oct 2020 13:51:22 -0700 Subject: [PATCH 16/18] updating variables based on feedback --- lemur/plugins/lemur_acme/plugin.py | 55 ++++++++++----------- lemur/plugins/lemur_acme/tests/test_acme.py | 4 +- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 63fa5626..0baa0478 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -37,9 +37,9 @@ from retrying import retry class AuthorizationRecord(object): - def __init__(self, domain, host, authz, dns_challenge, change_id): + def __init__(self, domain, target_domain, authz, dns_challenge, change_id): self.domain = domain - self.host = host + self.target_domain = target_domain self.authz = authz self.dns_challenge = dns_challenge self.change_id = change_id @@ -93,16 +93,16 @@ class AcmeHandler(object): acme_client, account_number, domain, - host, + target_domain, dns_provider, order, dns_provider_options, ): - current_app.logger.debug("Starting DNS challenge for {0}".format(host)) + current_app.logger.debug("Starting DNS challenge for {0}".format(target_domain)) change_ids = [] dns_challenges = self.get_dns_challenges(domain, order.authorizations) - host_to_validate, _ = self.strip_wildcard(host) + host_to_validate, _ = self.strip_wildcard(target_domain) host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) if not dns_challenges: @@ -113,7 +113,7 @@ class AcmeHandler(object): for dns_challenge in dns_challenges: # Only prepend '_acme-challenge' if not using CNAME redirection - if domain == host: + if domain == target_domain: host_to_validate = dns_challenge.validation_domain_name(host_to_validate) change_id = dns_provider.create_txt_record( @@ -124,7 +124,7 @@ class AcmeHandler(object): change_ids.append(change_id) return AuthorizationRecord( - domain, host, order.authorizations, dns_challenges, change_ids + domain, target_domain, order.authorizations, dns_challenges, change_ids ) def complete_dns_challenge(self, acme_client, authz_record): @@ -133,11 +133,11 @@ class AcmeHandler(object): authz_record.authz[0].body.identifier.value ) ) - dns_providers = self.dns_providers_for_domain.get(authz_record.host) + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) if not dns_providers: metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) raise Exception( - "No DNS providers found for domain: {}".format(authz_record.host) + "No DNS providers found for domain: {}".format(authz_record.target_domain) ) for dns_provider in dns_providers: @@ -165,7 +165,7 @@ class AcmeHandler(object): verified = response.simple_verify( dns_challenge.chall, - authz_record.host, + authz_record.target_domain, acme_client.client.net.key.public_key(), ) @@ -318,22 +318,22 @@ class AcmeHandler(object): for domain in order_info.domains: # If CNAME exists, set host to the target address - host = domain + target_domain = domain if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): - val_domain, _ = self.strip_wildcard(domain) - val_domain = challenges.DNS01().validation_domain_name(val_domain) - cname_res = self.get_cname(val_domain) - if cname_res: - host = cname_res - self.autodetect_dns_providers(host) + cname_result, _ = self.strip_wildcard(domain) + cname_result = challenges.DNS01().validation_domain_name(cname_result) + cname_result = self.get_cname(cname_result) + if cname_result: + target_domain = cname_result + self.autodetect_dns_providers(target_domain) - if not self.dns_providers_for_domain.get(host): + if not self.dns_providers_for_domain.get(target_domain): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 ) - raise Exception("No DNS providers found for domain: {}".format(host)) + raise Exception("No DNS providers found for domain: {}".format(target_domain)) - for dns_provider in self.dns_providers_for_domain[host]: + for dns_provider in self.dns_providers_for_domain[target_domain]: dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") @@ -341,7 +341,7 @@ class AcmeHandler(object): acme_client, account_number, domain, - host, + target_domain, dns_provider_plugin, order, dns_provider.options, @@ -376,7 +376,7 @@ class AcmeHandler(object): for authz_record in authorizations: dns_challenges = authz_record.dns_challenge for dns_challenge in dns_challenges: - dns_providers = self.dns_providers_for_domain.get(authz_record.host) + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) for dns_provider in dns_providers: # Grab account number (For Route53) dns_provider_plugin = self.get_dns_provider( @@ -384,9 +384,9 @@ class AcmeHandler(object): ) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") - host_to_validate, _ = self.strip_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options) - if authz_record.domain == authz_record.host: + if authz_record.domain == authz_record.target_domain: host_to_validate = challenges.DNS01().validation_domain_name(host_to_validate) dns_provider_plugin.delete_txt_record( authz_record.change_id, @@ -410,20 +410,20 @@ class AcmeHandler(object): :return: """ for authz_record in authorizations: - dns_providers = self.dns_providers_for_domain.get(authz_record.host) + dns_providers = self.dns_providers_for_domain.get(authz_record.target_domain) for dns_provider in dns_providers: # Grab account number (For Route53) dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_challenges = authz_record.dns_challenge - host_to_validate, _ = self.strip_wildcard(authz_record.host) + host_to_validate, _ = self.strip_wildcard(authz_record.target_domain) host_to_validate = self.maybe_add_extension( host_to_validate, dns_provider_options ) dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) for dns_challenge in dns_challenges: - if authz_record.domain == authz_record.host: + if authz_record.domain == authz_record.target_domain: host_to_validate = dns_challenge.validation_domain_name(host_to_validate), try: dns_provider_plugin.delete_txt_record( @@ -455,7 +455,6 @@ class AcmeHandler(object): def get_cname(self, domain): """ :param domain: Domain name to look up a CNAME for. - :param record_type: Type of DNS record to lookup. :return: First CNAME target or False if no CNAME record exists. """ try: diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index cce97d32..89ca6ee1 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -97,7 +97,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True) mock_authz.authz = [] - mock_authz.host = "www.test.com" + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" mock_authz.authz.append(mock_authz_record) @@ -121,7 +121,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.response = Mock() mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False) mock_authz.authz = [] - mock_authz.host = "www.test.com" + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" mock_authz.authz.append(mock_authz_record) From 84f8905cf1a0cfb0c15c1c1b5974f54ee9527376 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 29 Oct 2020 14:07:25 -0700 Subject: [PATCH 17/18] Hide expired certs for notifications --- lemur/static/app/angular/notifications/services.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index 02443701..9e8c9b33 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -27,7 +27,7 @@ angular.module('lemur') }; NotificationService.getCertificates = function (notification) { - notification.getList('certificates').then(function (certificates) { + notification.getList('certificates', {showExpired: 0}).then(function (certificates) { notification.certificates = certificates; }); }; @@ -40,7 +40,7 @@ angular.module('lemur') NotificationService.loadMoreCertificates = function (notification, page) { - notification.getList('certificates', {page: page}).then(function (certificates) { + notification.getList('certificates', {page: page, showExpired: 0}).then(function (certificates) { _.each(certificates, function (certificate) { notification.roles.push(certificate); }); From ca465e3c9eb06834e19c477ccd39b45f0ca4fd7e Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 29 Oct 2020 14:42:51 -0700 Subject: [PATCH 18/18] updating debug string with target_domain --- lemur/plugins/lemur_acme/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 0baa0478..07324f35 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -98,7 +98,7 @@ class AcmeHandler(object): order, dns_provider_options, ): - current_app.logger.debug("Starting DNS challenge for {0}".format(target_domain)) + current_app.logger.debug(f"Starting DNS challenge for {domain} using target domain {target_domain}.") change_ids = [] dns_challenges = self.get_dns_challenges(domain, order.authorizations)