From 0129e97dfe779c2d46208979c232c812d795793c Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 13:56:24 -0600 Subject: [PATCH 01/54] address karma CVEs --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9b899176..20e1e356 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "gulp-util": "^3.0.1", "http-proxy": "~1.16.2", "jshint-stylish": "^2.2.1", - "karma": "~1.3.0", + "karma": "^4.4.1", "karma-jasmine": "^1.1.0", "main-bower-files": "^2.13.1", "merge-stream": "^1.0.1", From a0a8c155f6f2bdae75720b1165710c180c540c92 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 13:59:41 -0600 Subject: [PATCH 02/54] patch gulp-protractor --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 20e1e356..2cee77ab 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "gulp-notify": "^2.2.0", "gulp-plumber": "^1.1.0", "gulp-print": "^2.0.1", - "gulp-protractor": "3.0.0", + "gulp-protractor": "^4.1.1", "gulp-replace": "~0.5.3", "gulp-replace-task": "~0.11.0", "gulp-rev": "^7.1.2", From 7a2ca8969eec68fc932dd4c9b8660dbd9fe26d62 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 14:08:22 -0600 Subject: [PATCH 03/54] patch gulp-imagemin --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2cee77ab..df0bbe9a 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "gulp-flatten": "^0.3.1", "gulp-foreach": "0.1.0", "gulp-if": "^2.0.2", - "gulp-imagemin": "^3.1.1", + "gulp-imagemin": "^7.1.0", "gulp-inject": "~4.1.0", "gulp-jshint": "^2.0.4", "gulp-less": "^3.0.3", From 980360b94a1530e049aee88360e374a10eda73d7 Mon Sep 17 00:00:00 2001 From: Andrew Pennebaker Date: Mon, 27 Jan 2020 14:09:57 -0600 Subject: [PATCH 04/54] patch gulp-less --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index df0bbe9a..a34fc745 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "gulp-imagemin": "^7.1.0", "gulp-inject": "~4.1.0", "gulp-jshint": "^2.0.4", - "gulp-less": "^3.0.3", + "gulp-less": "^4.0.1", "gulp-load-plugins": "^1.4.0", "gulp-minify-css": "^1.2.4", "gulp-minify-html": "~1.0.6", From 8ba9ae1148bfdadc3121b8c4ffb69689648f5725 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Mar 2020 16:18:43 +0000 Subject: [PATCH 05/54] Bump bleach from 3.1.1 to 3.1.2 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.1 to 3.1.2. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.1...v3.1.2) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 224789f6..d369cef4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,28 +5,28 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.1 # via readme-renderer +bleach==3.1.2 # via readme-renderer certifi==2019.11.28 # via requests cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit chardet==3.0.4 # via requests cryptography==2.8 # via secretstorage docutils==0.15.2 # via readme-renderer -flake8==3.5.0 +flake8==3.5.0 # via -r requirements-dev.in identify==1.4.9 # via pre-commit idna==2.8 # via requests -invoke==1.3.0 +invoke==1.3.0 # via -r requirements-dev.in jeepney==0.4.2 # via secretstorage keyring==21.0.0 # via twine mccabe==0.6.1 # via flake8 -nodeenv==1.3.3 +nodeenv==1.3.3 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==1.21.0 +pre-commit==1.21.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 pygments==2.5.2 # via readme-renderer -pyyaml==5.2 +pyyaml==5.2 # via -r requirements-dev.in, aspy.yaml, pre-commit readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine @@ -34,7 +34,7 @@ secretstorage==3.1.2 # via keyring six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.41.1 # via twine -twine==3.1.1 +twine==3.1.1 # via -r requirements-dev.in urllib3==1.25.7 # via requests virtualenv==16.7.9 # via pre-commit webencodings==0.5.1 # via bleach From 5206997468a3deb66be0393342d1d20561067605 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 26 Mar 2020 19:01:07 -0700 Subject: [PATCH 06/54] expired is now called for new certs, where the not_after field might be in datetime format, and not comparable to utc --- lemur/certificates/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 2ca88b00..9d7a459c 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -8,6 +8,8 @@ from datetime import timedelta import arrow +import pytz +import datetime from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa from flask import current_app @@ -321,8 +323,13 @@ class Certificate(db.Model): @hybrid_property def expired(self): - if self.not_after <= arrow.utcnow(): - return True + if isinstance(self.not_after, datetime.datetime): + # can't compare offset-naive and offset-aware datetimes + if self.not_after.replace(tzinfo=pytz.UTC) <= arrow.utcnow(): + return True + else: + if self.not_after <= arrow.utcnow(): + return True @expired.expression def expired(cls): From 2a2499a929bd30a80d19f4cca2be35dfdfa67098 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 26 Mar 2020 20:43:52 -0700 Subject: [PATCH 07/54] simplifying code --- lemur/certificates/models.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 9d7a459c..58630ee6 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -8,8 +8,6 @@ from datetime import timedelta import arrow -import pytz -import datetime from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa from flask import current_app @@ -323,13 +321,9 @@ class Certificate(db.Model): @hybrid_property def expired(self): - if isinstance(self.not_after, datetime.datetime): - # can't compare offset-naive and offset-aware datetimes - if self.not_after.replace(tzinfo=pytz.UTC) <= arrow.utcnow(): - return True - else: - if self.not_after <= arrow.utcnow(): - return True + # can't compare offset-naive and offset-aware datetimes + if arrow.Arrow.fromdatetime(self.not_after) <= arrow.utcnow(): + return True @expired.expression def expired(cls): From 0149f8b0d3b9c86739000c8372d92d7ac2d2ace3 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 26 Mar 2020 22:15:10 -0700 Subject: [PATCH 08/54] add support for wildcard and naked domains to PowerDNS module --- lemur/plugins/lemur_acme/powerdns.py | 210 +++++++++++++----- .../plugins/lemur_acme/tests/test_powerdns.py | 63 +++++- 2 files changed, 207 insertions(+), 66 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index a26faaac..e3f7e575 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -49,16 +49,20 @@ class Record: return self._data["name"] @property - def disabled(self): - return self._data["disabled"] + def type(self): + return self._data["type"] + + @property + def ttl(self): + return self._data["ttl"] @property def content(self): return self._data["content"] @property - def ttl(self): - return self._data["ttl"] + def disabled(self): + return self._data["disabled"] def get_zones(account_number): @@ -92,42 +96,32 @@ def get_zones(account_number): def create_txt_record(domain, token, account_number): """ Create a TXT record for the given domain and token and return a change_id tuple """ _check_conf() - zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_id = zone_name + "." - domain_id = domain + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "REPLACE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } + function = sys._getframe().f_code.co_name log_data = { "function": function, "fqdn": domain, "token": token, } + + # Create new record + domain_id = domain + "." + records = [Record({'name': domain_id, 'content': f"\"{token}\"", 'disabled': False})] + + # Get current records + cur_records = _get_txt_records(domain) + for record in cur_records: + if record.content != token: + records.append(record) + try: - _patch(path, payload) - log_data["message"] = "TXT record successfully created" + _patch_txt_records(domain, account_number, records) + log_data["message"] = "TXT record(s) successfully created" current_app.logger.debug(log_data) except Exception as e: sentry.captureException() log_data["Exception"] = e - log_data["message"] = "Unable to create TXT record" + log_data["message"] = "Unable to create TXT record(s)" current_app.logger.debug(log_data) change_id = (domain, token) @@ -173,43 +167,78 @@ def wait_for_dns_change(change_id, account_number=None): def delete_txt_record(change_id, account_number, domain, token): """ Delete the TXT record for the given domain and token """ _check_conf() - zone_name = _get_zone_name(domain, account_number) - server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") - zone_id = zone_name + "." - domain_id = domain + "." - path = f"/api/v1/servers/{server_id}/zones/{zone_id}" - payload = { - "rrsets": [ - { - "name": domain_id, - "type": "TXT", - "ttl": 300, - "changetype": "DELETE", - "records": [ - { - "content": f"\"{token}\"", - "disabled": False - } - ], - "comments": [] - } - ] - } + function = sys._getframe().f_code.co_name log_data = { "function": function, "fqdn": domain, - "token": token + "token": token, } - try: - _patch(path, payload) - log_data["message"] = "TXT record successfully deleted" - current_app.logger.debug(log_data) - except Exception as e: - sentry.captureException() - log_data["Exception"] = e - log_data["message"] = "Unable to delete TXT record" + + # Determine if we can delete whole RRset or just one record + cur_records = _get_txt_records(domain) + found = False + new_records = [] + for record in cur_records: + if record.content == f"\"{token}\"": + found = True + else: + new_records.append(record) + + if not found: # Record not found in DNS + log_data["message"] = "Unable to delete TXT record: TXT record not found" current_app.logger.debug(log_data) + return + + elif new_records: # Removing Record from RRSet via Patch + try: + _patch_txt_records(domain, account_number, new_records) + log_data["message"] = "TXT record successfully deleted" + current_app.logger.debug(log_data) + except Exception as e: + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to delete TXT record: patching exception" + current_app.logger.debug(log_data) + + else: # Delete current records + zone_name = _get_zone_name(domain, account_number) + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + zone_id = zone_name + "." + domain_id = domain + "." + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + payload = { + "rrsets": [ + { + "name": domain_id, + "type": "TXT", + "ttl": 300, + "changetype": "DELETE", + "records": [ + { + "content": f"\"{token}\"", + "disabled": False + } + ], + "comments": [] + } + ] + } + function = sys._getframe().f_code.co_name + log_data = { + "function": function, + "fqdn": domain, + "token": token + } + try: + _patch(path, payload) + log_data["message"] = "TXT record successfully deleted" + current_app.logger.debug(log_data) + except Exception as e: + sentry.captureException() + log_data["Exception"] = e + log_data["message"] = "Unable to delete TXT record" + current_app.logger.debug(log_data) def _check_conf(): @@ -243,6 +272,33 @@ def _get_zone_name(domain, account_number): return zone_name +def _get_txt_records(domain): + """Retrieve TXT records for a given domain and return list of Record Objects""" + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + + path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record" + function = sys._getframe().f_code.co_name + log_data = { + "function": function + } + try: + records = _get(path) + log_data["message"] = "Retrieved TXT Records Successfully" + current_app.logger.debug(log_data) + + except Exception as e: + sentry.captureException() + log_data["message"] = "Failed to Retrieve TXT Records" + current_app.logger.debug(log_data) + raise + + txt_records = [] + for record in records: + cur_record = Record(record) + txt_records.append(cur_record) + return txt_records + + def _get(path, params=None): """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") @@ -257,6 +313,40 @@ def _get(path, params=None): return resp.json() +def _patch_txt_records(domain, account_number, records): + """Send Patch request to PowerDNS Server""" + + domain_id = domain + "." + + # Create records + txt_records = [] + for record in records: + txt_records.append( + {'content': record.content, 'disabled': record.disabled} + ) + + # Create RRSet + payload = { + "rrsets": [ + { + "name": domain_id, + "type": "TXT", + "ttl": 300, + "changetype": "REPLACE", + "records": txt_records, + "comments": [] + } + ] + } + + # Create Txt Records + server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") + zone_name = _get_zone_name(domain, account_number) + zone_id = zone_name + "." + path = f"/api/v1/servers/{server_id}/zones/{zone_id}" + _patch(path, payload) + + def _patch(path, payload): """ Execute a Patch request on the given URL (base_uri + path) with given payload """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index c8b0a11e..d24827bb 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -48,13 +48,14 @@ class TestPowerdns(unittest.TestCase): self.assertEqual(result, zone) @patch("lemur.plugins.lemur_acme.powerdns.current_app") - def test_create_txt_record(self, mock_current_app): + def test_create_txt_record_write_only(self, mock_current_app): domain = "_acme_challenge.test.example.com" zone = "test.example.com" token = "ABCDEFGHIJ" account_number = "1234567890" change_id = (domain, token) powerdns._check_conf = Mock() + powerdns._get_txt_records = Mock(return_value=[]) powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() mock_current_app.config.get = Mock(return_value="localhost") @@ -63,24 +64,74 @@ class TestPowerdns(unittest.TestCase): "function": "create_txt_record", "fqdn": domain, "token": token, - "message": "TXT record successfully created" + "message": "TXT record(s) successfully created" } result = powerdns.create_txt_record(domain, token, account_number) mock_current_app.logger.debug.assert_called_with(log_data) self.assertEqual(result, change_id) + @patch("lemur.plugins.lemur_acme.powerdns.current_app") + def test_create_txt_record_append(self, mock_current_app): + domain = "_acme_challenge.test.example.com" + zone = "test.example.com" + token = "ABCDEFGHIJ" + account_number = "1234567890" + change_id = (domain, token) + powerdns._check_conf = Mock() + cur_token = "123456" + cur_records = [powerdns.Record({'name': domain, 'content': cur_token, 'disabled': False})] + powerdns._get_txt_records = Mock(return_value=cur_records) + powerdns._get_zone_name = Mock(return_value=zone) + mock_current_app.logger.debug = Mock() + mock_current_app.config.get = Mock(return_value="localhost") + powerdns._patch = Mock() + log_data = { + "function": "create_txt_record", + "fqdn": domain, + "token": token, + "message": "TXT record(s) successfully created" + } + expected_path = f"/api/v1/servers/localhost/zones/test.example.com." + expected_payload = { + "rrsets": [ + { + "name": domain + ".", + "type": "TXT", + "ttl": 300, + "changetype": "REPLACE", + "records": [ + { + "content": f"\"{token}\"", + "disabled": False + }, + { + "content": f"\"{cur_token}\"", + "disabled": False + } + ], + "comments": [] + } + ] + } + + result = powerdns.create_txt_record(domain, token, account_number) + mock_current_app.logger.debug.assert_called_with(log_data) + powerdns._patch.assert_called_with(expected_path, expected_payload) + self.assertEqual(result, change_id) + @patch("lemur.plugins.lemur_acme.powerdns.dnsutil") @patch("lemur.plugins.lemur_acme.powerdns.current_app") @patch("lemur.extensions.metrics") @patch("time.sleep") def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil): domain = "_acme-challenge.test.example.com" - token = "ABCDEFG" + token1 = "ABCDEFG" + token2 = "HIJKLMN" zone_name = "test.example.com" nameserver = "1.1.1.1" - change_id = (domain, token) + change_id = (domain, token1) powerdns._check_conf = Mock() - mock_records = (token,) + mock_records = (token2, token1) mock_current_app.config.get = Mock(return_value=1) powerdns._get_zone_name = Mock(return_value=zone_name) mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver) @@ -114,7 +165,7 @@ class TestPowerdns(unittest.TestCase): "function": "delete_txt_record", "fqdn": domain, "token": token, - "message": "TXT record successfully deleted" + "message": "Unable to delete TXT record: TXT record not found" } powerdns.delete_txt_record(change_id, account_number, domain, token) mock_current_app.logger.debug.assert_called_with(log_data) From 0e314d00281fbdcd31769cda2335d27df22d0d0b Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Fri, 27 Mar 2020 10:18:38 -0700 Subject: [PATCH 09/54] adding documentation and final cleanup --- lemur/plugins/lemur_acme/powerdns.py | 97 ++++++++++++++++--- .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- 2 files changed, 82 insertions(+), 17 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index e3f7e575..c988fac9 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -1,11 +1,10 @@ -import time -import requests import json import sys +import time import lemur.common.utils as utils import lemur.dns_providers.util as dnsutil - +import requests from flask import current_app from lemur.extensions import metrics, sentry @@ -17,7 +16,9 @@ REQUIRED_VARIABLES = [ class Zone: - """ This class implements a PowerDNS zone in JSON. """ + """ + This class implements a PowerDNS zone in JSON. + """ def __init__(self, _data): self._data = _data @@ -39,7 +40,9 @@ class Zone: class Record: - """ This class implements a PowerDNS record. """ + """ + This class implements a PowerDNS record. + """ def __init__(self, _data): self._data = _data @@ -66,7 +69,12 @@ class Record: def get_zones(account_number): - """Retrieve authoritative zones from the PowerDNS API and return a list""" + """ + Retrieve authoritative zones from the PowerDNS API and return a list + :param account_number: + :raise: Exception + :return: list of Zone Objects + """ _check_conf() server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") path = f"/api/v1/servers/{server_id}/zones" @@ -94,7 +102,14 @@ def get_zones(account_number): def create_txt_record(domain, token, account_number): - """ Create a TXT record for the given domain and token and return a change_id tuple """ + """ + Create a TXT record for the given domain and token and return a change_id tuple + + :param domain: FQDN + :param token: challenge value + :param account_number: + :return: tuple of domain/token + """ _check_conf() function = sys._getframe().f_code.co_name @@ -130,8 +145,11 @@ def create_txt_record(domain, token, account_number): def wait_for_dns_change(change_id, account_number=None): """ - Checks the authoritative DNS Server to see if changes have propagated to DNS - Retries and waits until successful. + Checks the authoritative DNS Server to see if changes have propagated. + + :param change_id: tuple of domain/token + :param account_number: + :return: """ _check_conf() domain, token = change_id @@ -165,7 +183,15 @@ def wait_for_dns_change(change_id, account_number=None): def delete_txt_record(change_id, account_number, domain, token): - """ Delete the TXT record for the given domain and token """ + """ + Delete the TXT record for the given domain and token + + :param change_id: tuple of domain/token + :param account_number: + :param domain: FQDN + :param token: challenge to delete + :return: + """ _check_conf() function = sys._getframe().f_code.co_name @@ -242,11 +268,20 @@ def delete_txt_record(change_id, account_number, domain, token): def _check_conf(): + """ + Verifies required configuration variables are set + + :return: + """ utils.validate_conf(current_app, REQUIRED_VARIABLES) def _generate_header(): - """Generate a PowerDNS API header and return it as a dictionary""" + """ + Generate a PowerDNS API header and return it as a dictionary + + :return: Dict of header parameters + """ api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME") api_key = current_app.config.get("ACME_POWERDNS_APIKEY") headers = {api_key_name: api_key} @@ -254,7 +289,13 @@ def _generate_header(): def _get_zone_name(domain, account_number): - """Get most specific matching zone for the given domain and return as a String""" + """ + Get most specific matching zone for the given domain and return as a String + + :param domain: FQDN + :param account_number: + :return: FQDN of domain + """ zones = get_zones(account_number) zone_name = "" for z in zones: @@ -273,7 +314,13 @@ def _get_zone_name(domain, account_number): def _get_txt_records(domain): - """Retrieve TXT records for a given domain and return list of Record Objects""" + """ + Retrieve TXT records for a given domain and return list of Record Objects + + :param domain: FQDN + :raise: Exception + :return: list of Record objects + """ server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record" @@ -300,7 +347,13 @@ def _get_txt_records(domain): def _get(path, params=None): - """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """ + """ + Execute a GET request on the given URL (base_uri + path) and return response as JSON object + + :param path: Relative URL path + :param params: additional parameters + :return: json response + """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.get( @@ -314,8 +367,14 @@ def _get(path, params=None): def _patch_txt_records(domain, account_number, records): - """Send Patch request to PowerDNS Server""" + """ + Send Patch request to PowerDNS Server + :param domain: FQDN + :param account_number: + :param records: List of Record objects + :return: + """ domain_id = domain + "." # Create records @@ -348,7 +407,13 @@ def _patch_txt_records(domain, account_number, records): def _patch(path, payload): - """ Execute a Patch request on the given URL (base_uri + path) with given payload """ + """ + Execute a Patch request on the given URL (base_uri + path) with given payload + + :param path: + :param payload: + :return: + """ base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN") verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True) resp = requests.patch( diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index d24827bb..707ce9e4 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -79,7 +79,7 @@ class TestPowerdns(unittest.TestCase): change_id = (domain, token) powerdns._check_conf = Mock() cur_token = "123456" - cur_records = [powerdns.Record({'name': domain, 'content': cur_token, 'disabled': False})] + cur_records = [powerdns.Record({'name': domain, 'content': f"\"{cur_token}\"", 'disabled': False})] powerdns._get_txt_records = Mock(return_value=cur_records) powerdns._get_zone_name = Mock(return_value=zone) mock_current_app.logger.debug = Mock() From d6cc8a8a9a86f6c1353d9ab15c9c0391856a139f Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 30 Mar 2020 09:01:28 -0700 Subject: [PATCH 10/54] fixing whitespace --- lemur/plugins/lemur_acme/powerdns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index c988fac9..1542ad7b 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -71,6 +71,7 @@ class Record: def get_zones(account_number): """ Retrieve authoritative zones from the PowerDNS API and return a list + :param account_number: :raise: Exception :return: list of Zone Objects From 6f3ba23fa0bb3f046fd76ac04cc2629dcf48eac7 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 30 Mar 2020 13:34:24 -0700 Subject: [PATCH 11/54] updating sinlge line of comments --- lemur/plugins/lemur_acme/powerdns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index 1542ad7b..c1e5e19b 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -70,8 +70,8 @@ class Record: def get_zones(account_number): """ - Retrieve authoritative zones from the PowerDNS API and return a list - + Retrieve authoritative zones from the PowerDNS API and return a list of zones + :param account_number: :raise: Exception :return: list of Zone Objects From 67d24caef586920ee0d91f790596dead5f119bb5 Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:31:12 -0700 Subject: [PATCH 12/54] Remove equivalent destinations when cleaning certificates Remove equivalent destinations when cleaning certificates. This will prevent Lemur from attempting to re-upload a certificate after it has been cleaned. --- lemur/sources/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lemur/sources/cli.py b/lemur/sources/cli.py index 0d537500..c415b567 100644 --- a/lemur/sources/cli.py +++ b/lemur/sources/cli.py @@ -58,6 +58,13 @@ def execute_clean(plugin, certificate, source): try: plugin.clean(certificate, source.options) certificate.sources.remove(source) + + # If we want to remove the source from the certificate, we also need to clear any equivalent destinations to + # prevent Lemur from re-uploading the certificate. + for destination in certificate.destinations: + if destination.label == source.label: + certificate.destinations.remove(destination) + certificate_service.database.update(certificate) return SUCCESS_METRIC_STATUS except Exception as e: From e25f97fce7da04b316ba2f6fc7359630f236f99f Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:50:24 -0700 Subject: [PATCH 13/54] Bump time limit for clean_source Celery job For larger accounts, I've hit SoftTimeLimit exceptions before completion of this celery job. Bumping up the time limit on this job. --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 4af33d86..ebf85ed7 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -332,7 +332,7 @@ def clean_all_sources(): metrics.send(f"{function}.success", 'counter', 1) -@celery.task(soft_time_limit=600) +@celery.task(soft_time_limit=3600) def clean_source(source): """ This celery task will clean the specified source. This is a destructive operation that will delete unused From d825616ea6cef6fd02ee80eedd7a75f5fc4cea61 Mon Sep 17 00:00:00 2001 From: Curtis Date: Wed, 1 Apr 2020 10:53:17 -0700 Subject: [PATCH 14/54] No need to retry 25 times on DeleteConflict errors --- lemur/plugins/lemur_aws/iam.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 13590ddd..8d80e020 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -24,6 +24,12 @@ def retry_throttled(exception): if exception.response["Error"]["Code"] == "NoSuchEntity": return False + # No need to retry deletion requests if there is a DeleteConflict error. + # This error indicates that the certificate is still attached to an entity + # and cannot be deleted. + if exception.response["Error"]["Code"] == "DeleteConflict": + return False + metrics.send("iam_retry", "counter", 1, metric_tags={"exception": str(exception)}) return True From 2e1b58c70a233ad656fa88f69004b786d00dd4b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2020 18:27:38 +0000 Subject: [PATCH 15/54] Bump bleach from 3.1.2 to 3.1.4 Bumps [bleach](https://github.com/mozilla/bleach) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/mozilla/bleach/releases) - [Changelog](https://github.com/mozilla/bleach/blob/master/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v3.1.2...v3.1.4) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d369cef4..b5521d38 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # aspy.yaml==1.3.0 # via pre-commit -bleach==3.1.2 # via readme-renderer +bleach==3.1.4 # via readme-renderer certifi==2019.11.28 # via requests cffi==1.14.0 # via cryptography cfgv==2.0.1 # via pre-commit From 5add64714883d5c07b688a50f5fc5bbc9722839f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 3 Apr 2020 16:51:24 -0700 Subject: [PATCH 16/54] # emitting the count of certificates on the source --- lemur/sources/service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f4783313..408d411a 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -193,6 +193,11 @@ def sync_certificates(source, user): s = plugins.get(source.plugin_name) certificates = s.get_certificates(source.options) + # emitting the count of certificates on the source + metrics.send("sync_certificates_count", + "gauge", len(certificates), + metric_tags={"source": source.label}) + for certificate in certificates: exists, updated_by_hash = find_cert(certificate) From 5c2a2f8ff24939b297788abff8322001e9b82513 Mon Sep 17 00:00:00 2001 From: David Stipp Date: Sat, 4 Apr 2020 11:24:04 -0400 Subject: [PATCH 17/54] OAUTH2 fixes * Use OAUTH2 variable instead of PING while using OAUTH * Some IDPs require a POST instead of a GET to user data --- lemur/auth/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lemur/auth/views.py b/lemur/auth/views.py index e7f87356..eaed419d 100644 --- a/lemur/auth/views.py +++ b/lemur/auth/views.py @@ -127,6 +127,10 @@ def retrieve_user(user_api_url, access_token): # retrieve information about the current user. r = requests.get(user_api_url, params=user_params, headers=headers) + # Some IDPs, like "Keycloak", require a POST instead of a GET + if r.status_code == 400: + r = requests.post(user_api_url, data=user_params, headers=headers) + profile = r.json() user = user_service.get_by_email(profile["email"]) @@ -434,7 +438,7 @@ class OAuth2(Resource): verify_cert=verify_cert, ) - jwks_url = current_app.config.get("PING_JWKS_URL") + jwks_url = current_app.config.get("OAUTH2_JWKS_URL") error_code = validate_id_token(id_token, args["clientId"], jwks_url) if error_code: return error_code From f82ec24dfaf6a1e8fb56fac3a394f2990085b711 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Sun, 5 Apr 2020 21:46:33 -0700 Subject: [PATCH 18/54] updating _get_txt_records return values and docstrings --- lemur/plugins/lemur_acme/powerdns.py | 24 +++++++++++++------ .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/powerdns.py b/lemur/plugins/lemur_acme/powerdns.py index c1e5e19b..a5d02353 100644 --- a/lemur/plugins/lemur_acme/powerdns.py +++ b/lemur/plugins/lemur_acme/powerdns.py @@ -202,7 +202,11 @@ def delete_txt_record(change_id, account_number, domain, token): "token": token, } - # Determine if we can delete whole RRset or just one record + """ + Get existing TXT records matching the domain from DNS + The token to be deleted should already exist + There may be other records with different tokens as well + """ cur_records = _get_txt_records(domain) found = False new_records = [] @@ -212,12 +216,16 @@ def delete_txt_record(change_id, account_number, domain, token): else: new_records.append(record) - if not found: # Record not found in DNS - log_data["message"] = "Unable to delete TXT record: TXT record not found" + # Since the matching token is not in DNS, there is nothing to delete + if not found: + log_data["message"] = "Unable to delete TXT record: Token not found in existing TXT records" current_app.logger.debug(log_data) return - elif new_records: # Removing Record from RRSet via Patch + # The record to delete has been found AND there are other tokens set on the same domain + # Since we only want to delete one token value from the RRSet, we need to use the Patch command to + # overwrite the current RRSet with the existing records. + elif new_records: try: _patch_txt_records(domain, account_number, new_records) log_data["message"] = "TXT record successfully deleted" @@ -228,7 +236,9 @@ def delete_txt_record(change_id, account_number, domain, token): log_data["message"] = "Unable to delete TXT record: patching exception" current_app.logger.debug(log_data) - else: # Delete current records + # The record to delete has been found AND there are no other token values set on the same domain + # Use the Delete command to delete the whole RRSet. + else: zone_name = _get_zone_name(domain, account_number) server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") zone_id = zone_name + "." @@ -319,7 +329,6 @@ def _get_txt_records(domain): Retrieve TXT records for a given domain and return list of Record Objects :param domain: FQDN - :raise: Exception :return: list of Record objects """ server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost") @@ -336,9 +345,10 @@ def _get_txt_records(domain): except Exception as e: sentry.captureException() + log_data["Exception"] = e log_data["message"] = "Failed to Retrieve TXT Records" current_app.logger.debug(log_data) - raise + return [] txt_records = [] for record in records: diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 707ce9e4..167381f2 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -165,7 +165,7 @@ class TestPowerdns(unittest.TestCase): "function": "delete_txt_record", "fqdn": domain, "token": token, - "message": "Unable to delete TXT record: TXT record not found" + "message": "Unable to delete TXT record: Token not found in existing TXT records" } powerdns.delete_txt_record(change_id, account_number, domain, token) mock_current_app.logger.debug.assert_called_with(log_data) From eb138fc96011167138159590178f9584d6972014 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 08:38:40 -0700 Subject: [PATCH 19/54] Add default celery metrics and logging using celery signals --- lemur/common/celery.py | 223 +++++++++++++++++++++++++++++++++-------- 1 file changed, 180 insertions(+), 43 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index ebf85ed7..b0193515 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -10,27 +10,27 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B import copy import sys import time -from datetime import datetime, timezone, timedelta - from celery import Celery +from celery.app.task import Context from celery.exceptions import SoftTimeLimitExceeded +from celery.signals import task_failure, task_received, task_revoked, task_success +from datetime import datetime, timezone, timedelta from flask import current_app from lemur.authorities.service import get as get_authority +from lemur.certificates import cli as cli_certificate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service +from lemur.dns_providers import cli as cli_dns_providers +from lemur.endpoints import cli as cli_endpoints from lemur.extensions import metrics, sentry from lemur.factory import create_app +from lemur.notifications import cli as cli_notification from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.sources.cli import clean, sync, validate_sources from lemur.sources.service import add_aws_destination_to_sources -from lemur.certificates import cli as cli_certificate -from lemur.dns_providers import cli as cli_dns_providers -from lemur.notifications import cli as cli_notification -from lemur.endpoints import cli as cli_endpoints - if current_app: flask_app = current_app @@ -67,7 +67,7 @@ def is_task_active(fun, task_id, args): from celery.task.control import inspect if not args: - args = '()' # empty args + args = "()" # empty args i = inspect() active_tasks = i.active() @@ -80,6 +80,37 @@ def is_task_active(fun, task_id, args): return False +def get_celery_request_tags(**kwargs): + request = kwargs.get("request") + sender_hostname = "unknown" + sender = kwargs.get("sender") + if sender: + try: + sender_hostname = sender.hostname + except AttributeError: + sender_hostname = vars(sender.request).get("origin", "unknown") + if request and not isinstance( + request, Context + ): # unlike others, task_revoked sends a Context for `request` + task_name = request.name + task_id = request.id + receiver_hostname = request.hostname + else: + task_name = sender.name + task_id = sender.request.id + receiver_hostname = sender.request.hostname + + tags = { + "task_name": task_name, + "task_id": task_id, + "sender_hostname": sender_hostname, + "receiver_hostname": receiver_hostname, + } + if kwargs.get("exception"): + tags["error"] = repr(kwargs["exception"]) + return tags + + @celery.task() def report_celery_last_success_metrics(): """ @@ -108,15 +139,115 @@ def report_celery_last_success_metrics(): return current_time = int(time.time()) - schedule = current_app.config.get('CELERYBEAT_SCHEDULE') + schedule = current_app.config.get("CELERYBEAT_SCHEDULE") for _, t in schedule.items(): task = t.get("task") last_success = int(red.get(f"{task}.last_success") or 0) - metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success) + metrics.send( + f"{task}.time_since_last_success", "gauge", current_time - last_success + ) red.set( f"{function}.last_success", int(time.time()) ) # Alert if this metric is not seen - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + + +@task_received.connect +def report_number_pending_tasks(**kwargs): + """ + Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used + for autoscaling workers. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + metrics.send( + "celery.new_pending_task", + "TIMER", + 1, + metric_tags=get_celery_request_tags(**kwargs), + ) + + +@task_success.connect +def report_successful_task(**kwargs): + """ + Report a generic success metric as tasks to our metrics broker every time a task finished correctly. + This metric can be used for autoscaling workers. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + tags = get_celery_request_tags(**kwargs) + red.set(f"{tags['task_name']}.last_success", int(time.time())) + metrics.send("celery.successful_task", "TIMER", 1, metric_tags=tags) + + +@task_failure.connect +def report_failed_task(**kwargs): + """ + Report a generic failure metric as tasks to our metrics broker every time a task fails. + This metric can be used for alerting. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "Message": "Celery Task Failure", + } + + # Add traceback if exception info is in the kwargs + einfo = kwargs.get("einfo") + if einfo: + log_data["traceback"] = einfo.traceback + + error_tags = get_celery_request_tags(**kwargs) + + log_data.update(error_tags) + current_app.logger.error(log_data) + metrics.send("celery.failed_task", "TIMER", 1, metric_tags=error_tags) + + +@task_revoked.connect +def report_revoked_task(**kwargs): + """ + Report a generic failure metric as tasks to our metrics broker every time a task is revoked. + This metric can be used for alerting. + https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked + + :param sender: + :param headers: + :param body: + :param kwargs: + :return: + """ + with flask_app.app_context(): + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "Message": "Celery Task Revoked", + } + + error_tags = get_celery_request_tags(**kwargs) + + log_data.update(error_tags) + current_app.logger.error(log_data) + metrics.send("celery.revoked_task", "TIMER", 1, metric_tags=error_tags) @celery.task(soft_time_limit=600) @@ -217,15 +348,15 @@ def fetch_acme_cert(id): log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) - metrics.send(f"{function}.resolved", 'gauge', new) - metrics.send(f"{function}.failed", 'gauge', failed) - metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer) + metrics.send(f"{function}.resolved", "gauge", new) + metrics.send(f"{function}.failed", "gauge", failed) + metrics.send(f"{function}.wrong_issuer", "gauge", wrong_issuer) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( new=new, failed=failed, wrong_issuer=wrong_issuer ) ) - red.set(f'{function}.last_success', int(time.time())) + return log_data @celery.task() @@ -262,8 +393,8 @@ def fetch_all_pending_acme_certs(): current_app.logger.debug(log_data) fetch_acme_cert.delay(cert.id) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task() @@ -296,8 +427,8 @@ def remove_old_acme_certs(): current_app.logger.debug(log_data) pending_certificate_service.delete(cert) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task() @@ -328,8 +459,8 @@ def clean_all_sources(): current_app.logger.debug(log_data) clean_source.delay(source.label) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -366,6 +497,7 @@ def clean_source(source): current_app.logger.error(log_data) sentry.captureException() metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) + return log_data @celery.task() @@ -395,8 +527,8 @@ def sync_all_sources(): current_app.logger.debug(log_data) sync_source.delay(source.label) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=7200) @@ -428,19 +560,23 @@ def sync_source(source): current_app.logger.debug(log_data) try: sync([source]) - metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) + metrics.send( + f"{function}.success", "counter", 1, metric_tags={"source": source} + ) except SoftTimeLimitExceeded: log_data["message"] = "Error syncing source: Time limit exceeded." current_app.logger.error(log_data) sentry.captureException() - metrics.send("sync_source_timeout", "counter", 1, metric_tags={"source": source}) + metrics.send( + "sync_source_timeout", "counter", 1, metric_tags={"source": source} + ) metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) - metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source}) - red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", "counter", 1, metric_tags={"source": source}) + return log_data @celery.task() @@ -477,8 +613,8 @@ def sync_source_destination(): log_data["message"] = "completed Syncing AWS destinations and sources" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -515,8 +651,8 @@ def certificate_reissue(): log_data["message"] = "reissuance completed" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -534,7 +670,6 @@ def certificate_rotate(): "function": function, "message": "rotating certificates", "task_id": task_id, - } if task_id and is_task_active(function, task_id, None): @@ -554,8 +689,8 @@ def certificate_rotate(): log_data["message"] = "rotation completed" current_app.logger.debug(log_data) - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -590,8 +725,8 @@ def endpoints_expire(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=600) @@ -626,8 +761,8 @@ def get_all_zones(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -662,8 +797,8 @@ def check_revoked(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data @celery.task(soft_time_limit=3600) @@ -690,7 +825,9 @@ def notify_expirations(): current_app.logger.debug(log_data) try: - cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])) + cli_notification.expirations( + current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []) + ) except SoftTimeLimitExceeded: log_data["message"] = "Notify expiring Time limit exceeded." current_app.logger.error(log_data) @@ -698,5 +835,5 @@ def notify_expirations(): metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) return - red.set(f'{function}.last_success', int(time.time())) - metrics.send(f"{function}.success", 'counter', 1) + metrics.send(f"{function}.success", "counter", 1) + return log_data From 11b15e7e234e0d5b7b494e35697d2ced5fbb5972 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 08:41:48 -0700 Subject: [PATCH 20/54] Clean up docstrings --- lemur/common/celery.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index b0193515..7c183dc9 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -120,7 +120,6 @@ def report_celery_last_success_metrics(): report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful. Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted from this function. - """ function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None @@ -158,12 +157,6 @@ def report_number_pending_tasks(**kwargs): Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used for autoscaling workers. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): metrics.send( @@ -180,12 +173,6 @@ def report_successful_task(**kwargs): Report a generic success metric as tasks to our metrics broker every time a task finished correctly. This metric can be used for autoscaling workers. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): tags = get_celery_request_tags(**kwargs) @@ -199,12 +186,6 @@ def report_failed_task(**kwargs): Report a generic failure metric as tasks to our metrics broker every time a task fails. This metric can be used for alerting. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): log_data = { @@ -230,12 +211,6 @@ def report_revoked_task(**kwargs): Report a generic failure metric as tasks to our metrics broker every time a task is revoked. This metric can be used for alerting. https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked - - :param sender: - :param headers: - :param body: - :param kwargs: - :return: """ with flask_app.app_context(): log_data = { From 1360d846fd16d0b375de8ce2b810a94aee312c49 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 8 Apr 2020 11:50:42 -0700 Subject: [PATCH 21/54] Improve error logging for a couple of use cases --- lemur/common/defaults.py | 17 +++++++++++++---- lemur/plugins/lemur_aws/plugin.py | 22 ++++++++++++---------- lemur/sources/service.py | 16 ++++++++++------ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index d563dbd0..b9c88e49 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -2,6 +2,7 @@ import re import unicodedata from cryptography import x509 +from cryptography.hazmat.primitives.serialization import Encoding from flask import current_app from lemur.common.utils import is_selfsigned @@ -71,12 +72,20 @@ def common_name(cert): :return: Common name or None """ try: - return cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[ - 0 - ].value.strip() + subject_oid = cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME) + if len(subject_oid) > 0: + return subject_oid[0].value.strip() + return None except Exception as e: sentry.captureException() - current_app.logger.error("Unable to get common name! {0}".format(e)) + current_app.logger.error( + { + "message": "Unable to get common name", + "error": e, + "public_key": cert.public_bytes(Encoding.PEM).decode("utf-8") + }, + exc_info=True + ) def organization(cert): diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 7bb7a3a2..8692348a 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -216,22 +216,24 @@ class AWSSourcePlugin(SourcePlugin): for region in regions: elbs = elb.get_all_elbs(account_number=account_number, region=region) - current_app.logger.info( - "Describing classic load balancers in {0}-{1}".format( - account_number, region - ) - ) + current_app.logger.info({ + "message": "Describing classic load balancers", + "account_number": account_number, + "region": region, + "number_of_load_balancers": len(elbs) + }) for e in elbs: endpoints.extend(get_elb_endpoints(account_number, region, e)) # fetch advanced ELBs elbs_v2 = elb.get_all_elbs_v2(account_number=account_number, region=region) - current_app.logger.info( - "Describing advanced load balancers in {0}-{1}".format( - account_number, region - ) - ) + current_app.logger.info({ + "message": "Describing advanced load balancers", + "account_number": account_number, + "region": region, + "number_of_load_balancers": len(elbs_v2) + }) for e in elbs_v2: endpoints.extend(get_elb_endpoints_v2(account_number, region, e)) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index f4783313..e0f0aacf 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -123,15 +123,19 @@ def sync_endpoints(source): "acct": s.get_option("accountNumber", source.options)}) if not endpoint["certificate"]: - current_app.logger.error( - "Certificate Not Found. Name: {0} Endpoint: {1}".format( - certificate_name, endpoint["name"] - ) - ) + current_app.logger.error({ + "message": "Certificate Not Found", + "certificate_name": certificate_name, + "endpoint_name": endpoint["name"], + "dns_name": endpoint.get("dnsname"), + "account": s.get_option("accountNumber", source.options), + }) + metrics.send("endpoint.certificate.not.found", "counter", 1, metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], - "acct": s.get_option("accountNumber", source.options)}) + "acct": s.get_option("accountNumber", source.options), + "dnsname": endpoint.get("dnsname")}) continue policy = endpoint.pop("policy") From cee81bd693dcc2df831f9652b727542f49de1d22 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 9 Apr 2020 18:17:05 -0700 Subject: [PATCH 22/54] updated requirements, fixed unittests, pytest, and distinguidedName ordering --- lemur/plugins/lemur_acme/tests/test_acme.py | 59 +++--- .../plugins/lemur_acme/tests/test_powerdns.py | 2 +- .../lemur_digicert/tests/test_digicert.py | 2 +- lemur/tests/test_certificates.py | 13 +- requirements-dev.txt | 46 ++-- requirements-docs.txt | 197 +++++++++--------- requirements-tests.txt | 110 +++++----- requirements.txt | 137 ++++++------ 8 files changed, 284 insertions(+), 282 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index b2c32eec..bec7be2b 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,11 +1,10 @@ import unittest +from unittest.mock import patch, Mock from cryptography.x509 import DNSName -from requests.models import Response - -from mock import MagicMock, Mock, patch - from lemur.plugins.lemur_acme import plugin, ultradns +from mock import MagicMock +from requests.models import Response class TestAcme(unittest.TestCase): @@ -57,7 +56,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") def test_start_dns_challenge( - self, mock_get_dns_challenges, mock_len, mock_app, mock_acme + self, mock_get_dns_challenges, mock_len, mock_app, mock_acme ): assert mock_len mock_order = Mock() @@ -88,7 +87,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") @patch("time.sleep") def test_complete_dns_challenge_success( - self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -112,7 +111,7 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_current_app, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -140,12 +139,12 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( - self, - mock_current_app, - mock_get_dns_challenges, - mock_jose, - mock_crypto, - mock_acme, + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, ): mock_cert_response = Mock() mock_cert_response.body = "123" @@ -182,7 +181,7 @@ class TestAcme(unittest.TestCase): assert result_client assert result_registration - @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch('lemur.plugins.lemur_acme.plugin.current_app') def test_get_domains_single(self, mock_current_app): options = {"common_name": "test.netflix.net"} result = self.acme.get_domains(options) @@ -288,14 +287,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") @@ -319,14 +318,14 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, ): mock_client = Mock() mock_acme.return_value = (mock_client, "") diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 167381f2..714cc938 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -1,5 +1,5 @@ import unittest -from mock import Mock, patch +from unittest.mock import patch, Mock from lemur.plugins.lemur_acme import plugin, powerdns diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 1e9ebca4..8bfd1dcf 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -1,4 +1,5 @@ import json +from unittest.mock import patch, Mock import arrow import pytest @@ -6,7 +7,6 @@ from cryptography import x509 from freezegun import freeze_time from lemur.plugins.lemur_digicert import plugin from lemur.tests.vectors import CSR_STR -from mock import Mock, patch def config_mock(*args): diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index adafa605..41584cb3 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -9,7 +9,8 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from marshmallow import ValidationError from freezegun import freeze_time -from mock import patch +# from mock import patch +from unittest.mock import patch from lemur.certificates.service import create_csr from lemur.certificates.views import * # noqa @@ -906,12 +907,12 @@ def test_certificate_get_body(client): assert response_body["serial"] == "211983098819107449768450703123665283596" assert response_body["serialHex"] == "9F7A75B39DAE4C3F9524C68B06DA6A0C" assert response_body["distinguishedName"] == ( - "CN=LemurTrust Unittests Class 1 CA 2018," - "O=LemurTrust Enterprises Ltd," - "OU=Unittesting Operations Center," - "C=EE," + "L=Earth," "ST=N/A," - "L=Earth" + "C=EE," + "OU=Unittesting Operations Center," + "O=LemurTrust Enterprises Ltd," + "CN=LemurTrust Unittests Class 1 CA 2018" ) diff --git a/requirements-dev.txt b/requirements-dev.txt index b5521d38..cb2edc22 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,40 +4,44 @@ # # pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in # -aspy.yaml==1.3.0 # via pre-commit +appdirs==1.4.3 # via virtualenv bleach==3.1.4 # via readme-renderer -certifi==2019.11.28 # via requests +certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography -cfgv==2.0.1 # via pre-commit +cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests -cryptography==2.8 # via secretstorage -docutils==0.15.2 # via readme-renderer +cryptography==2.9 # via secretstorage +distlib==0.3.0 # via virtualenv +docutils==0.16 # via readme-renderer +filelock==3.0.12 # via virtualenv flake8==3.5.0 # via -r requirements-dev.in -identify==1.4.9 # via pre-commit -idna==2.8 # via requests -invoke==1.3.0 # via -r requirements-dev.in -jeepney==0.4.2 # via secretstorage -keyring==21.0.0 # via twine +identify==1.4.14 # via pre-commit +idna==2.9 # via requests +importlib-metadata==1.6.0 # via keyring, pre-commit, twine, virtualenv +invoke==1.4.1 # via -r requirements-dev.in +jeepney==0.4.3 # via keyring, secretstorage +keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 -nodeenv==1.3.3 # via -r requirements-dev.in, pre-commit +nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==1.21.0 # via -r requirements-dev.in +pre-commit==2.2.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 -pycparser==2.19 # via cffi +pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 -pygments==2.5.2 # via readme-renderer -pyyaml==5.2 # via -r requirements-dev.in, aspy.yaml, pre-commit -readme-renderer==24.0 # via twine +pygments==2.6.1 # via readme-renderer +pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit +readme-renderer==25.0 # via twine requests-toolbelt==0.9.1 # via twine -requests==2.22.0 # via requests-toolbelt, twine +requests==2.23.0 # via requests-toolbelt, twine secretstorage==3.1.2 # via keyring -six==1.13.0 # via bleach, cfgv, cryptography, pre-commit, readme-renderer +six==1.14.0 # via bleach, cryptography, readme-renderer, virtualenv toml==0.10.0 # via pre-commit -tqdm==4.41.1 # via twine +tqdm==4.45.0 # via twine twine==3.1.1 # via -r requirements-dev.in -urllib3==1.25.7 # via requests -virtualenv==16.7.9 # via pre-commit +urllib3==1.25.8 # via requests +virtualenv==20.0.17 # via pre-commit webencodings==0.5.1 # via bleach +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-docs.txt b/requirements-docs.txt index 893965ca..d3eaa4d1 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,111 +4,108 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.0.0 +acme==1.3.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx -alembic-autogenerate-enums==0.0.2 -alembic==1.3.2 -amqp==2.5.2 -aniso8601==8.0.0 -arrow==0.15.5 -asyncpool==1.0 +alembic-autogenerate-enums==0.0.2 # via -r requirements.txt +alembic==1.4.2 # via -r requirements.txt, flask-migrate +amqp==2.5.2 # via -r requirements.txt, kombu +aniso8601==8.0.0 # via -r requirements.txt, flask-restful +arrow==0.15.5 # via -r requirements.txt +asyncpool==1.0 # via -r requirements.txt babel==2.8.0 # via sphinx -bcrypt==3.1.7 -billiard==3.6.1.0 -blinker==1.4 -boto3==1.10.46 -botocore==1.13.46 -celery[redis]==4.4.0 -certifi==2019.11.28 -certsrv==2.1.1 -cffi==1.13.2 -chardet==3.0.4 -click==7.0 -cloudflare==2.3.1 -cryptography==2.8 -dnspython3==1.15.0 -dnspython==1.15.0 -docutils==0.15.2 -dyn==1.8.1 -flask-bcrypt==0.7.1 -flask-cors==3.0.8 -flask-mail==0.9.1 -flask-migrate==2.5.2 -flask-principal==0.4.0 -flask-replicated==1.3 -flask-restful==0.3.7 -flask-script==2.0.6 -flask-sqlalchemy==2.4.1 -flask==1.1.1 -future==0.18.2 -gunicorn==20.0.4 -hvac==0.9.6 -idna==2.8 +bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko +billiard==3.6.3.0 # via -r requirements.txt, celery +blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven +boto3==1.12.39 # via -r requirements.txt +botocore==1.15.39 # via -r requirements.txt, boto3, s3transfer +celery[redis]==4.4.2 # via -r requirements.txt +certifi==2020.4.5.1 # via -r requirements.txt, requests +certsrv==2.1.1 # via -r requirements.txt +cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl +chardet==3.0.4 # via -r requirements.txt, requests +click==7.1.1 # via -r requirements.txt, flask +cloudflare==2.6.5 # via -r requirements.txt +cryptography==2.9 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +dnspython3==1.15.0 # via -r requirements.txt +dnspython==1.15.0 # via -r requirements.txt, dnspython3 +docutils==0.15.2 # via -r requirements.txt, botocore, sphinx +dyn==1.8.1 # via -r requirements.txt +flask-bcrypt==0.7.1 # via -r requirements.txt +flask-cors==3.0.8 # via -r requirements.txt +flask-mail==0.9.1 # via -r requirements.txt +flask-migrate==2.5.3 # via -r requirements.txt +flask-principal==0.4.0 # via -r requirements.txt +flask-replicated==1.3 # via -r requirements.txt +flask-restful==0.3.8 # via -r requirements.txt +flask-script==2.0.6 # via -r requirements.txt +flask-sqlalchemy==2.4.1 # via -r requirements.txt, flask-migrate +flask==1.1.2 # via -r requirements.txt, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven +future==0.18.2 # via -r requirements.txt, cloudflare +gunicorn==20.0.4 # via -r requirements.txt +hvac==0.10.1 # via -r requirements.txt +idna==2.9 # via -r requirements.txt, requests imagesize==1.2.0 # via sphinx -importlib-metadata==1.3.0 -inflection==0.3.1 -itsdangerous==1.1.0 -javaobj-py3==0.4.0.1 -jinja2==2.10.3 -jmespath==0.9.4 -josepy==1.2.0 -jsonlines==1.2.0 -kombu==4.6.7 -lockfile==0.12.2 -logmatic-python==0.1.7 -mako==1.1.0 -markupsafe==1.1.1 -marshmallow-sqlalchemy==0.21.0 -marshmallow==2.20.4 -mock==3.0.5 -more-itertools==8.0.2 -ndg-httpsclient==0.5.1 -packaging==19.2 # via sphinx -paramiko==2.7.1 -pem==19.3.0 -psycopg2==2.8.4 -pyasn1-modules==0.2.7 -pyasn1==0.4.8 -pycparser==2.19 -pycryptodomex==3.9.4 -pygments==2.5.2 # via sphinx -pyjks==19.0.0 -pyjwt==1.7.1 -pynacl==1.3.0 -pyopenssl==19.1.0 -pyparsing==2.4.6 # via packaging -pyrfc3339==1.1 -python-dateutil==2.8.1 -python-editor==1.0.4 -python-json-logger==0.1.11 -pytz==2019.3 -pyyaml==5.2 -raven[flask]==6.10.0 -redis==3.3.11 -requests-toolbelt==0.9.1 -requests[security]==2.22.0 -retrying==1.3.3 -s3transfer==0.2.1 -six==1.13.0 +importlib-metadata==1.6.0 # via -r requirements.txt, kombu +inflection==0.4.0 # via -r requirements.txt +itsdangerous==1.1.0 # via -r requirements.txt, flask +javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks +jinja2==2.11.1 # via -r requirements.txt, flask, sphinx +jmespath==0.9.5 # via -r requirements.txt, boto3, botocore +josepy==1.3.0 # via -r requirements.txt, acme +jsonlines==1.2.0 # via -r requirements.txt, cloudflare +kombu==4.6.8 # via -r requirements.txt, celery +lockfile==0.12.2 # via -r requirements.txt +logmatic-python==0.1.7 # via -r requirements.txt +mako==1.1.2 # via -r requirements.txt, alembic +markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako +marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt +marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy +mock==4.0.2 # via -r requirements.txt, acme +ndg-httpsclient==0.5.1 # via -r requirements.txt +packaging==20.3 # via sphinx +paramiko==2.7.1 # via -r requirements.txt +pem==20.1.0 # via -r requirements.txt +psycopg2==2.8.5 # via -r requirements.txt +pycparser==2.20 # via -r requirements.txt, cffi +pycryptodomex==3.9.7 # via -r requirements.txt, pyjks +pygments==2.6.1 # via sphinx +pyjks==19.0.0 # via -r requirements.txt +pyjwt==1.7.1 # via -r requirements.txt +pynacl==1.3.0 # via -r requirements.txt, paramiko +pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests +pyparsing==2.4.7 # via packaging +pyrfc3339==1.1 # via -r requirements.txt, acme +python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore +python-editor==1.0.4 # via -r requirements.txt, alembic +python-json-logger==0.1.11 # via -r requirements.txt, logmatic-python +pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 +pyyaml==5.3.1 # via -r requirements.txt, cloudflare +raven[flask]==6.10.0 # via -r requirements.txt +redis==3.4.1 # via -r requirements.txt, celery +requests-toolbelt==0.9.1 # via -r requirements.txt, acme +requests[security]==2.23.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx +retrying==1.3.3 # via -r requirements.txt +s3transfer==0.3.3 # via -r requirements.txt, boto3 +six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx -sphinx-rtd-theme==0.4.3 -sphinx==2.3.1 -sphinxcontrib-applehelp==1.0.1 # via sphinx -sphinxcontrib-devhelp==1.0.1 # via sphinx -sphinxcontrib-htmlhelp==1.0.2 # via sphinx -sphinxcontrib-httpdomain==1.7.0 +sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in +sphinx==3.0.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx +sphinxcontrib-httpdomain==1.7.0 # via -r requirements-docs.in sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.2 # via sphinx -sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.12 -tabulate==0.8.6 -twofish==0.3.0 -urllib3==1.25.7 -vine==1.3.0 -werkzeug==0.16.0 -xmltodict==0.12.0 -zipp==0.6.0 +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy-utils==0.36.3 # via -r requirements.txt +sqlalchemy==1.3.16 # via -r requirements.txt, alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.7 # via -r requirements.txt +twofish==0.3.0 # via -r requirements.txt, pyjks +urllib3==1.25.8 # via -r requirements.txt, botocore, requests +vine==1.3.0 # via -r requirements.txt, amqp, celery +werkzeug==1.0.1 # via -r requirements.txt, flask +xmltodict==0.12.0 # via -r requirements.txt +zipp==3.1.0 # via -r requirements.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index 293bd350..874fbb5c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -6,85 +6,87 @@ # appdirs==1.4.3 # via black attrs==19.3.0 # via black, jsonschema, pytest -aws-sam-translator==1.19.1 # via cfn-lint +aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.4.3 # via moto -bandit==1.6.2 -black==19.10b0 -boto3==1.10.46 # via aws-sam-translator, moto +bandit==1.6.2 # via -r requirements-tests.in +black==19.10b0 # via -r requirements-tests.in +boto3==1.12.39 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.13.46 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2019.11.28 # via requests -cffi==1.13.2 # via cryptography -cfn-lint==0.26.2 # via moto +botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2020.4.5.1 # via requests +cffi==1.14.0 # via cryptography +cfn-lint==0.29.4 # via moto chardet==3.0.4 # via requests -click==7.0 # via black, flask -coverage==5.0.1 -cryptography==2.8 # via moto, sshpubkeys -docker==4.1.0 # via moto +click==7.1.1 # via black, flask +coverage==5.0.4 # via -r requirements-tests.in +cryptography==2.9 # via moto, sshpubkeys +decorator==4.4.2 # via networkx +docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys -factory-boy==2.12.0 -faker==3.0.0 -fakeredis==1.1.0 -flask==1.1.1 # via pytest-flask -freezegun==0.3.12 +factory-boy==2.12.0 # via -r requirements-tests.in +faker==4.0.2 # via -r requirements-tests.in, factory-boy +fakeredis==1.4.0 # via -r requirements-tests.in +flask==1.1.2 # via pytest-flask +freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk -gitdb2==2.0.6 # via gitpython -gitpython==3.0.5 # via bandit +gitdb==4.0.2 # via gitpython +gitpython==3.1.0 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.3.0 # via jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.10.3 # via flask, moto -jmespath==0.9.4 # via boto3, botocore +jinja2==2.11.1 # via flask, moto +jmespath==0.9.5 # via boto3, botocore jsondiff==1.1.2 # via moto -jsonpatch==1.24 # via cfn-lint -jsonpickle==1.2 # via aws-xray-sdk +jsonpatch==1.25 # via cfn-lint +jsonpickle==1.3 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==3.2.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 -mock==3.0.5 # via moto -more-itertools==8.0.2 # via pytest, zipp -moto==1.3.14 -nose==1.3.7 -packaging==19.2 # via pytest -pathspec==0.7.0 # via black -pbr==5.4.4 # via stevedore +mock==4.0.2 # via moto +more-itertools==8.2.0 # via pytest +moto==1.3.14 # via -r requirements-tests.in +networkx==2.4 # via cfn-lint +nose==1.3.7 # via -r requirements-tests.in +packaging==20.3 # via pytest +pathspec==0.8.0 # via black +pbr==5.4.5 # via stevedore pluggy==0.13.1 # via pytest py==1.8.1 # via pytest pyasn1==0.4.8 # via python-jose, rsa -pycparser==2.19 # via cffi -pyflakes==2.1.1 -pyparsing==2.4.6 # via packaging -pyrsistent==0.15.6 # via jsonschema -pytest-flask==0.15.0 -pytest-mock==1.13.0 -pytest==5.3.2 +pycparser==2.20 # via cffi +pyflakes==2.1.1 # via -r requirements-tests.in +pyparsing==2.4.7 # via packaging +pyrsistent==0.16.0 # via jsonschema +pytest-flask==1.0.0 # via -r requirements-tests.in +pytest-mock==3.0.0 # via -r requirements-tests.in +pytest==5.4.1 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto -pyyaml==5.2 -redis==3.3.11 # via fakeredis -regex==2019.12.20 # via black -requests-mock==1.7.0 -requests==2.22.0 # via docker, moto, requests-mock, responses -responses==0.10.9 # via moto +pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto +redis==3.4.1 # via fakeredis +regex==2020.4.4 # via black +requests-mock==1.7.0 # via -r requirements-tests.in +requests==2.23.0 # via docker, moto, requests-mock, responses +responses==0.10.12 # via moto rsa==4.0 # via python-jose -s3transfer==0.2.1 # via boto3 -six==1.13.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, faker, fakeredis, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client -smmap2==2.0.5 # via gitdb2 +s3transfer==0.3.3 # via boto3 +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +smmap==3.0.1 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto -stevedore==1.31.0 # via bandit +stevedore==1.32.0 # via bandit text-unidecode==1.3 # via faker toml==0.10.0 # via black -typed-ast==1.4.0 # via black -urllib3==1.25.7 # via botocore, requests -wcwidth==0.1.8 # via pytest +typed-ast==1.4.1 # via black +urllib3==1.25.8 # via botocore, requests +wcwidth==0.1.9 # via pytest websocket-client==0.57.0 # via docker -werkzeug==0.16.0 # via flask, moto, pytest-flask -wrapt==1.11.2 # via aws-xray-sdk +werkzeug==1.0.1 # via flask, moto, pytest-flask +wrapt==1.12.1 # via aws-xray-sdk xmltodict==0.12.0 # via moto -zipp==0.6.0 # via importlib-metadata +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.txt b/requirements.txt index 639c9377..964576d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,96 +4,95 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.0.0 -alembic-autogenerate-enums==0.0.2 -alembic==1.3.2 # via flask-migrate +acme==1.3.0 # via -r requirements.in +alembic-autogenerate-enums==0.0.2 # via -r requirements.in +alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.15.5 -asyncpool==1.0 +arrow==0.15.5 # via -r requirements.in +asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko -billiard==3.6.1.0 # via celery +billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.10.46 -botocore==1.13.46 -celery[redis]==4.4.0 -certifi==2019.11.28 -certsrv==2.1.1 -cffi==1.13.2 # via bcrypt, cryptography, pynacl +boto3==1.12.39 # via -r requirements.in +botocore==1.15.39 # via -r requirements.in, boto3, s3transfer +celery[redis]==4.4.2 # via -r requirements.in +certifi==2020.4.5.1 # via -r requirements.in, requests +certsrv==2.1.1 # via -r requirements.in +cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.0 # via flask -cloudflare==2.3.1 -cryptography==2.8 -dnspython3==1.15.0 +click==7.1.1 # via flask +cloudflare==2.6.5 # via -r requirements.in +cryptography==2.9 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 docutils==0.15.2 # via botocore -dyn==1.8.1 -flask-bcrypt==0.7.1 -flask-cors==3.0.8 -flask-mail==0.9.1 -flask-migrate==2.5.2 -flask-principal==0.4.0 -flask-replicated==1.3 -flask-restful==0.3.7 -flask-script==2.0.6 -flask-sqlalchemy==2.4.1 -flask==1.1.1 -future==0.18.2 -gunicorn==20.0.4 -hvac==0.9.6 -idna==2.8 # via requests -importlib-metadata==1.3.0 # via kombu -inflection==0.3.1 +dyn==1.8.1 # via -r requirements.in +flask-bcrypt==0.7.1 # via -r requirements.in +flask-cors==3.0.8 # via -r requirements.in +flask-mail==0.9.1 # via -r requirements.in +flask-migrate==2.5.3 # via -r requirements.in +flask-principal==0.4.0 # via -r requirements.in +flask-replicated==1.3 # via -r requirements.in +flask-restful==0.3.8 # via -r requirements.in +flask-script==2.0.6 # via -r requirements.in +flask-sqlalchemy==2.4.1 # via -r requirements.in, flask-migrate +flask==1.1.2 # via -r requirements.in, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven +future==0.18.2 # via -r requirements.in, cloudflare +gunicorn==20.0.4 # via -r requirements.in +hvac==0.10.1 # via -r requirements.in +idna==2.9 # via requests +importlib-metadata==1.6.0 # via kombu +inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks -jinja2==2.10.3 -jmespath==0.9.4 # via boto3, botocore -josepy==1.2.0 # via acme +jinja2==2.11.1 # via -r requirements.in, flask +jmespath==0.9.5 # via boto3, botocore +josepy==1.3.0 # via acme jsonlines==1.2.0 # via cloudflare -kombu==4.6.7 # via celery -lockfile==0.12.2 -logmatic-python==0.1.7 -mako==1.1.0 # via alembic +kombu==4.6.8 # via celery +lockfile==0.12.2 # via -r requirements.in +logmatic-python==0.1.7 # via -r requirements.in +mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.21.0 -marshmallow==2.20.4 -mock==3.0.5 # via acme -more-itertools==8.0.2 # via zipp -ndg-httpsclient==0.5.1 -paramiko==2.7.1 -pem==19.3.0 -psycopg2==2.8.4 -pyasn1-modules==0.2.7 # via pyjks, python-ldap +marshmallow-sqlalchemy==0.22.3 # via -r requirements.in +marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy +mock==4.0.2 # via acme +ndg-httpsclient==0.5.1 # via -r requirements.in +paramiko==2.7.1 # via -r requirements.in +pem==20.1.0 # via -r requirements.in +psycopg2==2.8.5 # via -r requirements.in +pyasn1-modules==0.2.8 # via pyjks, python-ldap pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap -pycparser==2.19 # via cffi -pycryptodomex==3.9.4 # via pyjks -pyjks==19.0.0 -pyjwt==1.7.1 +pycparser==2.20 # via cffi +pycryptodomex==3.9.7 # via pyjks +pyjks==19.0.0 # via -r requirements.in +pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko -pyopenssl==19.1.0 +pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests pyrfc3339==1.1 # via acme python-dateutil==2.8.1 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python -python-ldap==3.2.0 +python-ldap==3.2.0 # via -r requirements.in pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.2 -raven[flask]==6.10.0 -redis==3.3.11 +pyyaml==5.3.1 # via -r requirements.in, cloudflare +raven[flask]==6.10.0 # via -r requirements.in +redis==3.4.1 # via -r requirements.in, celery requests-toolbelt==0.9.1 # via acme -requests[security]==2.22.0 -retrying==1.3.3 -s3transfer==0.2.1 # via boto3 -six==1.13.0 -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.6 +requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt +retrying==1.3.3 # via -r requirements.in +s3transfer==0.3.3 # via boto3 +six==1.14.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils +sqlalchemy-utils==0.36.3 # via -r requirements.in +sqlalchemy==1.3.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.7 # via -r requirements.in twofish==0.3.0 # via pyjks -urllib3==1.25.7 # via botocore, requests +urllib3==1.25.8 # via botocore, requests vine==1.3.0 # via amqp, celery -werkzeug==0.16.0 # via flask -xmltodict==0.12.0 -zipp==0.6.0 # via importlib-metadata +werkzeug==1.0.1 # via flask +xmltodict==0.12.0 # via -r requirements.in +zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From f8657998e64cf3c4cb32406dbaa15296803962bb Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 15 Apr 2020 14:17:34 -0700 Subject: [PATCH 23/54] updated python requirements and still works --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 22 +++++++++++----------- requirements.txt | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d3eaa4d1..54bde14e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -49,7 +49,7 @@ importlib-metadata==1.6.0 # via -r requirements.txt, kombu inflection==0.4.0 # via -r requirements.txt itsdangerous==1.1.0 # via -r requirements.txt, flask javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks -jinja2==2.11.1 # via -r requirements.txt, flask, sphinx +jinja2==2.11.2 # via -r requirements.txt, flask, sphinx jmespath==0.9.5 # via -r requirements.txt, boto3, botocore josepy==1.3.0 # via -r requirements.txt, acme jsonlines==1.2.0 # via -r requirements.txt, cloudflare @@ -89,7 +89,7 @@ s3transfer==0.3.3 # via -r requirements.txt, boto3 six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in -sphinx==3.0.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.0.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 874fbb5c..5f9cafcc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,7 +7,7 @@ appdirs==1.4.3 # via black attrs==19.3.0 # via black, jsonschema, pytest aws-sam-translator==1.22.0 # via cfn-lint -aws-xray-sdk==2.4.3 # via moto +aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in boto3==1.12.39 # via aws-sam-translator, moto @@ -15,31 +15,31 @@ boto==2.49.0 # via moto botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography -cfn-lint==0.29.4 # via moto +cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.1 # via black, flask -coverage==5.0.4 # via -r requirements-tests.in +coverage==5.1 # via -r requirements-tests.in cryptography==2.9 # via moto, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in -faker==4.0.2 # via -r requirements-tests.in, factory-boy +faker==4.0.3 # via -r requirements-tests.in, factory-boy fakeredis==1.4.0 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk -gitdb==4.0.2 # via gitpython -gitpython==3.1.0 # via bandit +gitdb==4.0.4 # via gitpython +gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.11.1 # via flask, moto +jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore jsondiff==1.1.2 # via moto jsonpatch==1.25 # via cfn-lint -jsonpickle==1.3 # via aws-xray-sdk +jsonpickle==1.4 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch jsonschema==3.2.0 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 @@ -55,7 +55,7 @@ pluggy==0.13.1 # via pytest py==1.8.1 # via pytest pyasn1==0.4.8 # via python-jose, rsa pycparser==2.20 # via cffi -pyflakes==2.1.1 # via -r requirements-tests.in +pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in @@ -73,7 +73,7 @@ responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client -smmap==3.0.1 # via gitdb +smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto stevedore==1.32.0 # via bandit diff --git a/requirements.txt b/requirements.txt index 964576d2..bdad75b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ importlib-metadata==1.6.0 # via kombu inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks -jinja2==2.11.1 # via -r requirements.in, flask +jinja2==2.11.2 # via -r requirements.in, flask jmespath==0.9.5 # via boto3, botocore josepy==1.3.0 # via acme jsonlines==1.2.0 # via cloudflare From e33b767d12f9850d8fbfbf698e32ecdc3a50fbb7 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 15 Apr 2020 14:31:41 -0700 Subject: [PATCH 24/54] updating package.jso based on `npm update` --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9b899176..84e5289d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ }, "devDependencies": { "gulp": "^3.9.1", - "jshint": "^2.8.0", + "jshint": "^2.11.0", "karma-chrome-launcher": "^2.0.0" } } From 4fcb050fa81b4080ff92e3665679d870fc112f57 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 16 Apr 2020 01:08:39 -0700 Subject: [PATCH 25/54] fixing bootstrap, bootswatch, and updating bower --- bower.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bower.json b/bower.json index f7d5500d..44cce994 100644 --- a/bower.json +++ b/bower.json @@ -11,12 +11,12 @@ "angular": "1.4.9", "json3": "~3.3", "es5-shim": "~4.5.0", - "bootstrap": "~3.3.6", "angular-bootstrap": "~1.1.1", "angular-animate": "~1.4.9", "restangular": "~1.5.1", "ng-table": "~0.8.3", "moment": "~2.11.1", + "bootstrap": "~3.4.1", "angular-loading-bar": "~0.8.0", "angular-moment": "~0.10.3", "moment-range": "~2.1.0", @@ -24,7 +24,7 @@ "angularjs-toaster": "~1.0.0", "angular-chart.js": "~0.8.8", "ngletteravatar": "~4.0.0", - "bootswatch": "~3.3.6", + "bootswatch": "~3.4.1", "fontawesome": "~4.5.0", "satellizer": "~0.13.4", "angular-ui-router": "~0.2.15", diff --git a/package.json b/package.json index 771a4386..1a54eccc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "url": "git://github.com/netflix/lemur.git" }, "dependencies": { - "bower": "^1.8.2", + "bower": "^1.8.8", "browser-sync": "^2.26.7", "del": "^2.2.2", "gulp-autoprefixer": "^3.1.1", From 8d0007b9c06f3e40305eab1083dcf7bb3fdda14f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 24 Apr 2020 15:48:06 -0700 Subject: [PATCH 26/54] fixing the private DNS zone issue. Private hosted zones will never be visible to third-parties like LetsEncrypt, and Lemur should not consider them as authoritative zones. This fix, make sure they are not added to the dns_provider table. --- lemur/plugins/lemur_acme/route53.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index 55da5161..aaccb57e 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -35,9 +35,10 @@ def get_zones(client=None): zones = [] for page in paginator.paginate(): for zone in page["HostedZones"]: - zones.append( - zone["Name"][:-1] - ) # We need [:-1] to strip out the trailing dot. + if not zone["Config"]["PrivateZone"]: + zones.append( + zone["Name"][:-1] + ) # We need [:-1] to strip out the trailing dot. return zones From 273c3e2793647389e00e2509b0e0fd047aa540fe Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 11:52:43 -0700 Subject: [PATCH 27/54] Celery task to enable autorotate for all certificates attached to endpoints without it enabled --- lemur/certificates/service.py | 25 ++++++++++++++++++++++--- lemur/common/celery.py | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index a6bbba30..b031d86b 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -118,6 +118,21 @@ def get_all_pending_cleaning_expired(source): ) +def get_all_certs_attached_to_endpoint_without_rotate(): + """ + Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled. + + :return: list of certificates attached to an endpoint without autorotate + """ + return ( + Certificate.query.filter(Certificate.endpoints.any()) + .filter(Certificate.rotation == False) + .filter(Certificate.not_after >= arrow.now()) + .filter(not_(Certificate.replaced.any())) + .all() # noqa + ) + + def get_all_pending_cleaning_expiring_in_days(source, days_to_expire): """ Retrieves all certificates that are available for cleaning, not attached to endpoint, @@ -144,7 +159,9 @@ def get_all_pending_cleaning_issued_since_days(source, days_since_issuance): :param source: the source to search for certificates :return: list of pending certificates """ - not_in_use_window = arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + not_in_use_window = ( + arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD") + ) return ( Certificate.query.filter(Certificate.sources.any(id=source.id)) .filter(not_(Certificate.endpoints.any())) @@ -367,9 +384,11 @@ def render(args): show_expired = args.pop("showExpired") if show_expired != 1: - one_month_old = arrow.now()\ - .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\ + one_month_old = ( + arrow.now() + .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1)) .format("YYYY-MM-DD") + ) query = query.filter(Certificate.not_after > one_month_old) time_range = args.pop("time_range") diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 7c183dc9..a5f608b2 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -17,8 +17,10 @@ from celery.signals import task_failure, task_received, task_revoked, task_succe from datetime import datetime, timezone, timedelta from flask import current_app +from lemur import database from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate +from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_rotate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.dns_providers import cli as cli_dns_providers @@ -812,3 +814,25 @@ def notify_expirations(): metrics.send(f"{function}.success", "counter", 1) return log_data + + +@celery.task(soft_time_limit=3600) +def enable_autorotate_for_certs_attached_to_endpoint(): + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "task_id": task_id, + } + + eligible_certs = get_all_certs_attached_to_endpoint_without_rotate() + for cert in eligible_certs: + log_data["certificate"] = cert.name + log_data["certificate_id"] = cert.id + log_data["message"] = "Enabling auto-rotate for certificate" + current_app.logger.info(log_data) + cert.rotation = True + database.update(cert) From 863af7a3e5f509b984ff2adba31515ebb7187624 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 12:16:46 -0700 Subject: [PATCH 28/54] Making CLI command ; Running black --- lemur/certificates/cli.py | 50 +++++++++++++++++++++++------------ lemur/certificates/service.py | 2 +- lemur/common/celery.py | 21 +++++++-------- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index b57ff175..ca6b0248 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -5,29 +5,18 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import sys import multiprocessing -from tabulate import tabulate -from sqlalchemy import or_ - +import sys from flask import current_app - -from flask_script import Manager from flask_principal import Identity, identity_changed - +from flask_script import Manager +from sqlalchemy import or_ +from tabulate import tabulate from lemur import database -from lemur.extensions import sentry -from lemur.extensions import metrics -from lemur.plugins.base import plugins -from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS -from lemur.deployment import service as deployment_service -from lemur.endpoints import service as endpoint_service -from lemur.notifications.messaging import send_rotation_notification -from lemur.domains.models import Domain from lemur.authorities.models import Authority -from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.models import Certificate +from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.service import ( reissue_certificate, get_certificate_primitives, @@ -35,9 +24,16 @@ from lemur.certificates.service import ( get_by_name, get_all_certs, get, + get_all_certs_attached_to_endpoint_without_autorotate, ) - from lemur.certificates.verify import verify_string +from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS +from lemur.deployment import service as deployment_service +from lemur.domains.models import Domain +from lemur.endpoints import service as endpoint_service +from lemur.extensions import sentry, metrics +from lemur.notifications.messaging import send_rotation_notification +from lemur.plugins.base import plugins manager = Manager(usage="Handles all certificate related tasks.") @@ -482,3 +478,23 @@ def check_revoked(): cert.status = "unknown" database.update(cert) + + +@manager.command +def automatically_enable_autorotate(): + """ + This function automatically enables autorotation for unexpired certificates that are + attached to an endpoint but do not have autorotate enabled. + """ + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + } + + eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate() + for cert in eligible_certs: + log_data["certificate"] = cert.name + log_data["certificate_id"] = cert.id + log_data["message"] = "Enabling auto-rotate for certificate" + current_app.logger.info(log_data) + cert.rotation = True + database.update(cert) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index b031d86b..5d1e6e63 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -118,7 +118,7 @@ def get_all_pending_cleaning_expired(source): ) -def get_all_certs_attached_to_endpoint_without_rotate(): +def get_all_certs_attached_to_endpoint_without_autorotate(): """ Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled. diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a5f608b2..7701b82d 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -17,10 +17,8 @@ from celery.signals import task_failure, task_received, task_revoked, task_succe from datetime import datetime, timezone, timedelta from flask import current_app -from lemur import database from lemur.authorities.service import get as get_authority from lemur.certificates import cli as cli_certificate -from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_rotate from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.dns_providers import cli as cli_dns_providers @@ -818,21 +816,20 @@ def notify_expirations(): @celery.task(soft_time_limit=3600) def enable_autorotate_for_certs_attached_to_endpoint(): - function = f"{__name__}.{sys._getframe().f_code.co_name}" + """ + This celery task automatically enables autorotation for unexpired certificates that are + attached to an endpoint but do not have autorotate enabled. + :return: + """ task_id = None if celery.current_task: task_id = celery.current_task.request.id log_data = { - "function": function, + "function": f"{__name__}.{sys._getframe().f_code.co_name}", "task_id": task_id, + "message": "Enabling autorotate to eligible certificates", } + current_app.logger.debug(log_data) - eligible_certs = get_all_certs_attached_to_endpoint_without_rotate() - for cert in eligible_certs: - log_data["certificate"] = cert.name - log_data["certificate_id"] = cert.id - log_data["message"] = "Enabling auto-rotate for certificate" - current_app.logger.info(log_data) - cert.rotation = True - database.update(cert) + cli_certificate.automatically_enable_autorotate() From 7e97d885dfea1b86d7274c472f106a318ddc4738 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 28 Apr 2020 13:16:27 -0700 Subject: [PATCH 29/54] Address comments --- lemur/common/celery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 7701b82d..5df470ab 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -821,15 +821,18 @@ def enable_autorotate_for_certs_attached_to_endpoint(): attached to an endpoint but do not have autorotate enabled. :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None if celery.current_task: task_id = celery.current_task.request.id log_data = { - "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "function": function, "task_id": task_id, "message": "Enabling autorotate to eligible certificates", } current_app.logger.debug(log_data) cli_certificate.automatically_enable_autorotate() + metrics.send(f"{function}.success", "counter", 1) + return log_data From 4c40e806bcb0be7006f20654c39803dd0b4c8ab0 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 29 Apr 2020 08:41:42 -0700 Subject: [PATCH 30/54] Requirements fix for Pip 20.1 --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index fa5a23bc..f73ebb6f 100644 --- a/setup.py +++ b/setup.py @@ -45,16 +45,16 @@ with open(os.path.join(ROOT, 'lemur', '__about__.py')) as f: exec(f.read(), about) # nosec: about file is benign install_requires_g = parse_requirements("requirements.txt", session=PipSession()) -install_requires = [str(ir.req) for ir in install_requires_g] +install_requires = [str(ir.requirement) for ir in install_requires_g] tests_require_g = parse_requirements("requirements-tests.txt", session=PipSession()) -tests_require = [str(ir.req) for ir in tests_require_g] +tests_require = [str(ir.requirement) for ir in tests_require_g] -docs_require_g = parse_requirements("requirements-docs.txt", session=PipSession()) -docs_require = [str(ir.req) for ir in docs_require_g] +docs_require_g = parse_requirements("require20ments-docs.txt", session=PipSession()) +docs_require = [str(ir.requirement) for ir in docs_require_g] dev_requires_g = parse_requirements("requirements-dev.txt", session=PipSession()) -dev_requires = [str(ir.req) for ir in dev_requires_g] +dev_requires = [str(ir.requirement) for ir in dev_requires_g] class SmartInstall(install): From ba8184c87492413dfed09a3405e0ae560ed9b9aa Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Wed, 29 Apr 2020 08:51:09 -0700 Subject: [PATCH 31/54] Fix requirement parsing for pip 20.1 --- setup.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index f73ebb6f..4ce03d70 100644 --- a/setup.py +++ b/setup.py @@ -45,16 +45,20 @@ with open(os.path.join(ROOT, 'lemur', '__about__.py')) as f: exec(f.read(), about) # nosec: about file is benign install_requires_g = parse_requirements("requirements.txt", session=PipSession()) -install_requires = [str(ir.requirement) for ir in install_requires_g] - tests_require_g = parse_requirements("requirements-tests.txt", session=PipSession()) -tests_require = [str(ir.requirement) for ir in tests_require_g] - -docs_require_g = parse_requirements("require20ments-docs.txt", session=PipSession()) -docs_require = [str(ir.requirement) for ir in docs_require_g] - +docs_require_g = parse_requirements("requirements-docs.txt", session=PipSession()) dev_requires_g = parse_requirements("requirements-dev.txt", session=PipSession()) -dev_requires = [str(ir.requirement) for ir in dev_requires_g] + +if tuple(map(int, pip.__version__.split('.'))) >= (20, 1): + install_requires = [str(ir.requirement) for ir in install_requires_g] + tests_require = [str(ir.requirement) for ir in tests_require_g] + docs_require = [str(ir.requirement) for ir in docs_require_g] + dev_requires = [str(ir.requirement) for ir in dev_requires_g] +else: + install_requires = [str(ir.req) for ir in install_requires_g] + tests_require = [str(ir.req) for ir in tests_require_g] + docs_require = [str(ir.req) for ir in docs_require_g] + dev_requires = [str(ir.req) for ir in dev_requires_g] class SmartInstall(install): From d469700cbf46c820f9f73366fb524c6232787c6f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 00:23:52 +0000 Subject: [PATCH 32/54] Bump fakeredis from 1.4.0 to 1.4.1 Bumps [fakeredis](https://github.com/jamesls/fakeredis) from 1.4.0 to 1.4.1. - [Release notes](https://github.com/jamesls/fakeredis/releases) - [Commits](https://github.com/jamesls/fakeredis/compare/1.4.0...1.4.1) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 5f9cafcc..7ffb2d5f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -26,14 +26,14 @@ docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in faker==4.0.3 # via -r requirements-tests.in, factory-boy -fakeredis==1.4.0 # via -r requirements-tests.in +fakeredis==1.4.1 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk gitdb==4.0.4 # via gitpython gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle itsdangerous==1.1.0 # via flask jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore @@ -72,7 +72,7 @@ requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 -six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto From 7f6eae7213db19d62c0963fea6eb68396fa22f54 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 00:25:47 +0000 Subject: [PATCH 33/54] Bump cryptography from 2.9 to 2.9.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.9 to 2.9.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.9...2.9.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 4 +--- requirements-docs.txt | 7 ++++--- requirements-tests.txt | 6 +++--- requirements.txt | 4 +--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cb2edc22..36a0bd7f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,14 +10,13 @@ certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests -cryptography==2.9 # via secretstorage +cryptography==2.9.2 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv flake8==3.5.0 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests -importlib-metadata==1.6.0 # via keyring, pre-commit, twine, virtualenv invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine @@ -41,7 +40,6 @@ twine==3.1.1 # via -r requirements-dev.in urllib3==1.25.8 # via requests virtualenv==20.0.17 # via pre-commit webencodings==0.5.1 # via bleach -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-docs.txt b/requirements-docs.txt index 54bde14e..c1abe7e0 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -25,7 +25,7 @@ cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynac chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask cloudflare==2.6.5 # via -r requirements.txt -cryptography==2.9 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +cryptography==2.9.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 docutils==0.15.2 # via -r requirements.txt, botocore, sphinx @@ -45,7 +45,6 @@ gunicorn==20.0.4 # via -r requirements.txt hvac==0.10.1 # via -r requirements.txt idna==2.9 # via -r requirements.txt, requests imagesize==1.2.0 # via sphinx -importlib-metadata==1.6.0 # via -r requirements.txt, kombu inflection==0.4.0 # via -r requirements.txt itsdangerous==1.1.0 # via -r requirements.txt, flask javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks @@ -66,6 +65,8 @@ packaging==20.3 # via sphinx paramiko==2.7.1 # via -r requirements.txt pem==20.1.0 # via -r requirements.txt psycopg2==2.8.5 # via -r requirements.txt +pyasn1-modules==0.2.8 # via -r requirements.txt, pyjks, python-ldap +pyasn1==0.4.8 # via -r requirements.txt, ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.20 # via -r requirements.txt, cffi pycryptodomex==3.9.7 # via -r requirements.txt, pyjks pygments==2.6.1 # via sphinx @@ -78,6 +79,7 @@ pyrfc3339==1.1 # via -r requirements.txt, acme python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore python-editor==1.0.4 # via -r requirements.txt, alembic python-json-logger==0.1.11 # via -r requirements.txt, logmatic-python +python-ldap==3.2.0 # via -r requirements.txt pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt @@ -105,7 +107,6 @@ urllib3==1.25.8 # via -r requirements.txt, botocore, requests vine==1.3.0 # via -r requirements.txt, amqp, celery werkzeug==1.0.1 # via -r requirements.txt, flask xmltodict==0.12.0 # via -r requirements.txt -zipp==3.1.0 # via -r requirements.txt, importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index 5f9cafcc..494ac1d4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -19,7 +19,7 @@ cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.1 # via black, flask coverage==5.1 # via -r requirements-tests.in -cryptography==2.9 # via moto, sshpubkeys +cryptography==2.9.2 # via moto, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto docutils==0.15.2 # via botocore @@ -33,7 +33,7 @@ future==0.18.2 # via aws-xray-sdk gitdb==4.0.4 # via gitpython gitpython==3.1.1 # via bandit idna==2.8 # via moto, requests -importlib-metadata==1.6.0 # via jsonpickle, jsonschema, pluggy, pytest +importlib-metadata==1.6.0 # via jsonpickle itsdangerous==1.1.0 # via flask jinja2==2.11.2 # via flask, moto jmespath==0.9.5 # via boto3, botocore @@ -72,7 +72,7 @@ requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 -six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +six==1.14.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap==3.0.2 # via gitdb sortedcontainers==2.1.0 # via fakeredis sshpubkeys==3.1.0 # via moto diff --git a/requirements.txt b/requirements.txt index bdad75b0..70a04ee1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask cloudflare==2.6.5 # via -r requirements.in -cryptography==2.9 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +cryptography==2.9.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 docutils==0.15.2 # via botocore @@ -42,7 +42,6 @@ future==0.18.2 # via -r requirements.in, cloudflare gunicorn==20.0.4 # via -r requirements.in hvac==0.10.1 # via -r requirements.in idna==2.9 # via requests -importlib-metadata==1.6.0 # via kombu inflection==0.4.0 # via -r requirements.in itsdangerous==1.1.0 # via flask javaobj-py3==0.4.0.1 # via pyjks @@ -92,7 +91,6 @@ urllib3==1.25.8 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==1.0.1 # via flask xmltodict==0.12.0 # via -r requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From f68900d2b355660f495f9243cdab19cdc5338a30 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 7 May 2020 18:28:01 -0700 Subject: [PATCH 34/54] improving logging and the possibility of defining which Authorities qualify for auto-rotation --- lemur/certificates/cli.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index ca6b0248..3be794e7 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -483,18 +483,35 @@ def check_revoked(): @manager.command def automatically_enable_autorotate(): """ - This function automatically enables autorotation for unexpired certificates that are + This function automatically enables auto-rotation for unexpired certificates that are attached to an endpoint but do not have autorotate enabled. + + WARNING: This will overwrite the Auto-rotate toggle! """ log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", } + permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", []) + eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate() for cert in eligible_certs: + + if cert.authority_id not in permitted_authorities: + continue + log_data["certificate"] = cert.name log_data["certificate_id"] = cert.id log_data["message"] = "Enabling auto-rotate for certificate" current_app.logger.info(log_data) cert.rotation = True database.update(cert) + # TODO: add the cert destination to the logging + metrics.send("automatically_enable_autorotate", + "counter", 1, + metric_tags={"certificate": cert.name, + "certificate_id": cert.id, + "authority_id": cert.authority_id, + "authority_name": Authority.get(cert.authority_id).name}) + cert.rotation = True + database.update(cert) From 529ee04ae748f1b138ca90929bdf1b72a1e9b5f2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 8 May 2020 09:15:18 -0700 Subject: [PATCH 35/54] removing duplicate line --- lemur/certificates/cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 3be794e7..54455eec 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -504,8 +504,6 @@ def automatically_enable_autorotate(): log_data["certificate_id"] = cert.id log_data["message"] = "Enabling auto-rotate for certificate" current_app.logger.info(log_data) - cert.rotation = True - database.update(cert) # TODO: add the cert destination to the logging metrics.send("automatically_enable_autorotate", "counter", 1, From fa13bda99e2ff56355ee5aa6ab4d313282ec019c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 16:45:06 +0000 Subject: [PATCH 36/54] Bump sphinx from 3.0.1 to 3.0.3 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.0.1 to 3.0.3. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.0.1...v3.0.3) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index c1abe7e0..7dc714a9 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -91,7 +91,7 @@ s3transfer==0.3.3 # via -r requirements.txt, boto3 six==1.14.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in -sphinx==3.0.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.0.3 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 8e2226180a38af17e2589cd60fe235374f6c3946 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 18:35:37 +0000 Subject: [PATCH 37/54] Bump pre-commit from 2.2.0 to 2.3.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.2.0 to 2.3.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.2.0...v2.3.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 36a0bd7f..68e28b73 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.2.0 # via -r requirements-dev.in +pre-commit==2.3.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 From 5ad9c11716565bf67c81fdb98d849b44b6dada44 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 18:42:41 +0000 Subject: [PATCH 38/54] Bump arrow from 0.15.5 to 0.15.6 Bumps [arrow](https://github.com/crsmithdev/arrow) from 0.15.5 to 0.15.6. - [Release notes](https://github.com/crsmithdev/arrow/releases) - [Changelog](https://github.com/crsmithdev/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/crsmithdev/arrow/compare/0.15.5...0.15.6) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 7dc714a9..db255de3 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -10,7 +10,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate amqp==2.5.2 # via -r requirements.txt, kombu aniso8601==8.0.0 # via -r requirements.txt, flask-restful -arrow==0.15.5 # via -r requirements.txt +arrow==0.15.6 # via -r requirements.txt asyncpool==1.0 # via -r requirements.txt babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko diff --git a/requirements.txt b/requirements.txt index 70a04ee1..f8d553d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.15.5 # via -r requirements.in +arrow==0.15.6 # via -r requirements.in asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery From dac95313b84c4db66d179531e7e8e2f7f56dc35f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:29:51 +0000 Subject: [PATCH 39/54] Bump pyjks from 19.0.0 to 20.0.0 Bumps [pyjks](https://github.com/kurtbrose/pyjks) from 19.0.0 to 20.0.0. - [Release notes](https://github.com/kurtbrose/pyjks/releases) - [Changelog](https://github.com/kurtbrose/pyjks/blob/master/CHANGELOG.md) - [Commits](https://github.com/kurtbrose/pyjks/compare/v19.0.0...v20.0.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index db255de3..14e54b59 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -70,7 +70,7 @@ pyasn1==0.4.8 # via -r requirements.txt, ndg-httpsclient, pyasn1-mod pycparser==2.20 # via -r requirements.txt, cffi pycryptodomex==3.9.7 # via -r requirements.txt, pyjks pygments==2.6.1 # via sphinx -pyjks==19.0.0 # via -r requirements.txt +pyjks==20.0.0 # via -r requirements.txt pyjwt==1.7.1 # via -r requirements.txt pynacl==1.3.0 # via -r requirements.txt, paramiko pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests diff --git a/requirements.txt b/requirements.txt index f8d553d4..83585828 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,7 +65,7 @@ pyasn1-modules==0.2.8 # via pyjks, python-ldap pyasn1==0.4.8 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.20 # via cffi pycryptodomex==3.9.7 # via pyjks -pyjks==19.0.0 # via -r requirements.in +pyjks==20.0.0 # via -r requirements.in pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests From 6bef8fb9d7f86e7e76b9ef4489367f03a6d72489 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:36:54 +0000 Subject: [PATCH 40/54] Bump acme from 1.3.0 to 1.4.0 Bumps [acme](https://github.com/letsencrypt/letsencrypt) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/letsencrypt/letsencrypt/releases) - [Commits](https://github.com/letsencrypt/letsencrypt/compare/v1.3.0...v1.4.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 3 +-- requirements.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 14e54b59..6c2a37e4 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.3.0 # via -r requirements.txt +acme==1.4.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate @@ -59,7 +59,6 @@ mako==1.1.2 # via -r requirements.txt, alembic markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy -mock==4.0.2 # via -r requirements.txt, acme ndg-httpsclient==0.5.1 # via -r requirements.txt packaging==20.3 # via sphinx paramiko==2.7.1 # via -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 83585828..ad92d95f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.3.0 # via -r requirements.in +acme==1.4.0 # via -r requirements.in alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu @@ -56,7 +56,6 @@ mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako marshmallow-sqlalchemy==0.22.3 # via -r requirements.in marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy -mock==4.0.2 # via acme ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.1 # via -r requirements.in pem==20.1.0 # via -r requirements.in From 5ac65062c93e52bcac58209337018aaae3db3e19 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:49:15 +0000 Subject: [PATCH 41/54] Bump sqlalchemy-utils from 0.36.3 to 0.36.5 Bumps [sqlalchemy-utils](https://github.com/kvesteri/sqlalchemy-utils) from 0.36.3 to 0.36.5. - [Release notes](https://github.com/kvesteri/sqlalchemy-utils/releases) - [Changelog](https://github.com/kvesteri/sqlalchemy-utils/blob/master/CHANGES.rst) - [Commits](https://github.com/kvesteri/sqlalchemy-utils/compare/0.36.3...0.36.5) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 6c2a37e4..d8bc26d7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -98,7 +98,7 @@ sphinxcontrib-httpdomain==1.7.0 # via -r requirements-docs.in sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sqlalchemy-utils==0.36.3 # via -r requirements.txt +sqlalchemy-utils==0.36.5 # via -r requirements.txt sqlalchemy==1.3.16 # via -r requirements.txt, alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.7 # via -r requirements.txt twofish==0.3.0 # via -r requirements.txt, pyjks diff --git a/requirements.txt b/requirements.txt index ad92d95f..60b38553 100644 --- a/requirements.txt +++ b/requirements.txt @@ -82,7 +82,7 @@ requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, retrying==1.3.3 # via -r requirements.in s3transfer==0.3.3 # via boto3 six==1.14.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils -sqlalchemy-utils==0.36.3 # via -r requirements.in +sqlalchemy-utils==0.36.5 # via -r requirements.in sqlalchemy==1.3.16 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.7 # via -r requirements.in twofish==0.3.0 # via pyjks From 5afa32fa226ff972eb4ca2d078058e5258ea6874 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 8 May 2020 22:57:55 +0000 Subject: [PATCH 42/54] Bump boto3 from 1.12.39 to 1.13.6 Bumps [boto3](https://github.com/boto/boto3) from 1.12.39 to 1.13.6. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.12.39...1.13.6) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d8bc26d7..a468419c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,8 +16,8 @@ babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.12.39 # via -r requirements.txt -botocore==1.15.39 # via -r requirements.txt, boto3, s3transfer +boto3==1.13.6 # via -r requirements.txt +botocore==1.16.6 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 9c9bbff0..5bb54cc6 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in -boto3==1.12.39 # via aws-sam-translator, moto +boto3==1.13.6 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.15.39 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.16.6 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 60b38553..c2e269e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,8 @@ asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.12.39 # via -r requirements.in -botocore==1.15.39 # via -r requirements.in, boto3, s3transfer +boto3==1.13.6 # via -r requirements.in +botocore==1.16.6 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 201a96ff72d7c518a37b5497fbb920c61d5a6144 Mon Sep 17 00:00:00 2001 From: Luka Matijevic Date: Mon, 11 May 2020 13:09:23 +0200 Subject: [PATCH 43/54] Improve periodic tasks docs --- docs/production/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/production/index.rst b/docs/production/index.rst index cd044ca4..b91ed6bd 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -390,6 +390,10 @@ Here are the Celery configuration variables that should be set:: CELERY_IMPORTS = ('lemur.common.celery') CELERY_TIMEZONE = 'UTC' +Do not forget to import crontab module in your configuration file:: + + from celery.task.schedules import crontab + You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks. The scheduler can be started with:: From 5d75204a4f81ec30bdebfba277b8def133dc2a1f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 13:45:08 +0000 Subject: [PATCH 44/54] Bump faker from 4.0.3 to 4.1.0 Bumps [faker](https://github.com/joke2k/faker) from 4.0.3 to 4.1.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.0.3...v4.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 5bb54cc6..32b4d72b 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ docker==4.2.0 # via moto docutils==0.15.2 # via botocore ecdsa==0.15 # via python-jose, sshpubkeys factory-boy==2.12.0 # via -r requirements-tests.in -faker==4.0.3 # via -r requirements-tests.in, factory-boy +faker==4.1.0 # via -r requirements-tests.in, factory-boy fakeredis==1.4.1 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==0.3.15 # via -r requirements-tests.in From c0829478fdf62d46b7ba006129ec21b8489c5ba0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:14:27 +0000 Subject: [PATCH 45/54] Bump requests-mock from 1.7.0 to 1.8.0 Bumps [requests-mock](https://github.com/jamielennox/requests-mock) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/jamielennox/requests-mock/releases) - [Commits](https://github.com/jamielennox/requests-mock/compare/1.7.0...1.8.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 32b4d72b..cac0f42a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -67,7 +67,7 @@ pytz==2019.3 # via moto pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto redis==3.4.1 # via fakeredis regex==2020.4.4 # via black -requests-mock==1.7.0 # via -r requirements-tests.in +requests-mock==1.8.0 # via -r requirements-tests.in requests==2.23.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose From 9e1737edc49b307a8526b278ddec8010620ac6d5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:25:47 +0000 Subject: [PATCH 46/54] Bump pytest from 5.4.1 to 5.4.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 5.4.1 to 5.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/5.4.1...5.4.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index cac0f42a..369408dd 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -60,7 +60,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in pytest-mock==3.0.0 # via -r requirements-tests.in -pytest==5.4.1 # via -r requirements-tests.in, pytest-flask, pytest-mock +pytest==5.4.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto From dfd9acb0d7f0b604162c57ebf791c7a060a569a3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:37:21 +0000 Subject: [PATCH 47/54] Bump pytest-mock from 3.0.0 to 3.1.0 Bumps [pytest-mock](https://github.com/pytest-dev/pytest-mock) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/pytest-dev/pytest-mock/releases) - [Changelog](https://github.com/pytest-dev/pytest-mock/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-mock/compare/v3.0.0...v3.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 369408dd..3b5f83db 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -59,7 +59,7 @@ pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in -pytest-mock==3.0.0 # via -r requirements-tests.in +pytest-mock==3.1.0 # via -r requirements-tests.in pytest==5.4.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto From 23cd7d3d625a8919ebeb9d8497ad76e0e30154ba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:47:18 +0000 Subject: [PATCH 48/54] Bump marshmallow-sqlalchemy from 0.22.3 to 0.23.0 Bumps [marshmallow-sqlalchemy](https://github.com/marshmallow-code/marshmallow-sqlalchemy) from 0.22.3 to 0.23.0. - [Release notes](https://github.com/marshmallow-code/marshmallow-sqlalchemy/releases) - [Changelog](https://github.com/marshmallow-code/marshmallow-sqlalchemy/blob/dev/CHANGELOG.rst) - [Commits](https://github.com/marshmallow-code/marshmallow-sqlalchemy/compare/0.22.3...0.23.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index a468419c..8b22bb16 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -57,7 +57,7 @@ lockfile==0.12.2 # via -r requirements.txt logmatic-python==0.1.7 # via -r requirements.txt mako==1.1.2 # via -r requirements.txt, alembic markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako -marshmallow-sqlalchemy==0.22.3 # via -r requirements.txt +marshmallow-sqlalchemy==0.23.0 # via -r requirements.txt marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy ndg-httpsclient==0.5.1 # via -r requirements.txt packaging==20.3 # via sphinx diff --git a/requirements.txt b/requirements.txt index c2e269e5..40fff126 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ lockfile==0.12.2 # via -r requirements.in logmatic-python==0.1.7 # via -r requirements.in mako==1.1.2 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.22.3 # via -r requirements.in +marshmallow-sqlalchemy==0.23.0 # via -r requirements.in marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.1 # via -r requirements.in From d3a0fe74911bbe891bc7cd3b7d4d91b448229454 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 16:58:47 +0000 Subject: [PATCH 49/54] Bump cloudflare from 2.6.5 to 2.7.1 Bumps [cloudflare](https://github.com/cloudflare/python-cloudflare) from 2.6.5 to 2.7.1. - [Release notes](https://github.com/cloudflare/python-cloudflare/releases) - [Changelog](https://github.com/cloudflare/python-cloudflare/blob/master/CHANGELOG.md) - [Commits](https://github.com/cloudflare/python-cloudflare/compare/2.6.5...2.7.1) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 8b22bb16..33b53872 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -24,7 +24,7 @@ certsrv==2.1.1 # via -r requirements.txt cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask -cloudflare==2.6.5 # via -r requirements.txt +cloudflare==2.7.1 # via -r requirements.txt cryptography==2.9.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 diff --git a/requirements.txt b/requirements.txt index 40fff126..d413c566 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask -cloudflare==2.6.5 # via -r requirements.in +cloudflare==2.7.1 # via -r requirements.in cryptography==2.9.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 From 9bc017c9bf4e2d81411750f06ce00a537f703d01 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:11:39 +0000 Subject: [PATCH 50/54] Bump botocore from 1.16.6 to 1.16.11 Bumps [botocore](https://github.com/boto/botocore) from 1.16.6 to 1.16.11. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.16.6...1.16.11) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 33b53872..7d49fe56 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,7 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.13.6 # via -r requirements.txt -botocore==1.16.6 # via -r requirements.txt, boto3, s3transfer +botocore==1.16.11 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 3b5f83db..5ca1f238 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in boto3==1.13.6 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.16.6 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.16.11 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index d413c566..72f3938b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.13.6 # via -r requirements.in -botocore==1.16.6 # via -r requirements.in, boto3, s3transfer +botocore==1.16.11 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From d010553d23b1166122dab68394973f0ffc5c8847 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:23:23 +0000 Subject: [PATCH 51/54] Bump redis from 3.4.1 to 3.5.2 Bumps [redis](https://github.com/andymccurdy/redis-py) from 3.4.1 to 3.5.2. - [Release notes](https://github.com/andymccurdy/redis-py/releases) - [Changelog](https://github.com/andymccurdy/redis-py/blob/master/CHANGES) - [Commits](https://github.com/andymccurdy/redis-py/compare/3.4.1...3.5.2) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 7d49fe56..d3e61970 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -82,7 +82,7 @@ python-ldap==3.2.0 # via -r requirements.txt pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt -redis==3.4.1 # via -r requirements.txt, celery +redis==3.5.2 # via -r requirements.txt, celery requests-toolbelt==0.9.1 # via -r requirements.txt, acme requests[security]==2.23.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx retrying==1.3.3 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 5ca1f238..93d992e5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -65,7 +65,7 @@ python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose==3.1.0 # via moto pytz==2019.3 # via moto pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto -redis==3.4.1 # via fakeredis +redis==3.5.2 # via fakeredis regex==2020.4.4 # via black requests-mock==1.8.0 # via -r requirements-tests.in requests==2.23.0 # via docker, moto, requests-mock, responses diff --git a/requirements.txt b/requirements.txt index 72f3938b..0ad0708c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -76,7 +76,7 @@ python-ldap==3.2.0 # via -r requirements.in pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.in, cloudflare raven[flask]==6.10.0 # via -r requirements.in -redis==3.4.1 # via -r requirements.in, celery +redis==3.5.2 # via -r requirements.in, celery requests-toolbelt==0.9.1 # via acme requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt retrying==1.3.3 # via -r requirements.in From 2bf03a0bc245c20524034b72477104bbd3f0a4b2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:35:43 +0000 Subject: [PATCH 52/54] Bump boto3 from 1.13.6 to 1.13.11 Bumps [boto3](https://github.com/boto/boto3) from 1.13.6 to 1.13.11. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.13.6...1.13.11) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d3e61970..16d97413 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,7 +16,7 @@ babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.13.6 # via -r requirements.txt +boto3==1.13.11 # via -r requirements.txt botocore==1.16.11 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.4.5.1 # via -r requirements.txt, requests diff --git a/requirements-tests.txt b/requirements-tests.txt index 93d992e5..79340e51 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==19.10b0 # via -r requirements-tests.in -boto3==1.13.6 # via aws-sam-translator, moto +boto3==1.13.11 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.16.11 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.4.5.1 # via requests diff --git a/requirements.txt b/requirements.txt index 0ad0708c..315f39b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.13.6 # via -r requirements.in +boto3==1.13.11 # via -r requirements.in botocore==1.16.11 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.4.5.1 # via -r requirements.in, requests From 4d4ff8509062dc82369c00b107120a4c5bffb88d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 17:44:22 +0000 Subject: [PATCH 53/54] Bump pre-commit from 2.3.0 to 2.4.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.3.0 to 2.4.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.3.0...v2.4.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 68e28b73..785d3f29 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -23,7 +23,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.3.0 # via -r requirements-dev.in +pre-commit==2.4.0 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 From fd444403bb8d97f7c72fe0c434ac15f8b7b8bcba Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 21 May 2020 15:32:38 -0700 Subject: [PATCH 54/54] improved logging. - adding destination name, fixing broken metric. --- lemur/certificates/cli.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 54455eec..d007e458 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -15,6 +15,7 @@ from tabulate import tabulate from lemur import database from lemur.authorities.models import Authority +from lemur.authorities.service import get as authorities_get_by_id from lemur.certificates.models import Certificate from lemur.certificates.schemas import CertificateOutputSchema from lemur.certificates.service import ( @@ -490,6 +491,7 @@ def automatically_enable_autorotate(): """ log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Enabling auto-rotate for certificate" } permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", []) @@ -502,14 +504,20 @@ def automatically_enable_autorotate(): log_data["certificate"] = cert.name log_data["certificate_id"] = cert.id - log_data["message"] = "Enabling auto-rotate for certificate" + log_data["authority_id"] = cert.authority_id + log_data["authority_name"] = authorities_get_by_id(cert.authority_id).name + if cert.destinations: + log_data["destination_names"] = ', '.join([d.label for d in cert.destinations]) + else: + log_data["destination_names"] = "NONE" current_app.logger.info(log_data) - # TODO: add the cert destination to the logging metrics.send("automatically_enable_autorotate", "counter", 1, - metric_tags={"certificate": cert.name, - "certificate_id": cert.id, - "authority_id": cert.authority_id, - "authority_name": Authority.get(cert.authority_id).name}) + metric_tags={"certificate": log_data["certificate"], + "certificate_id": log_data["certificate_id"], + "authority_id": log_data["authority_id"], + "authority_name": log_data["authority_name"], + "destination_names": log_data["destination_names"] + }) cert.rotation = True database.update(cert)