From 1c59fb575ca813e4a93800e87c30f180951e4fa8 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 29 Oct 2020 14:28:54 -0700 Subject: [PATCH 001/100] Fix the 'more' button to view certificates associated with a notification --- .../app/angular/notifications/notification/notification.js | 6 ++++++ lemur/static/app/angular/notifications/services.js | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lemur/static/app/angular/notifications/notification/notification.js b/lemur/static/app/angular/notifications/notification/notification.js index d3cfac9b..d23299db 100644 --- a/lemur/static/app/angular/notifications/notification/notification.js +++ b/lemur/static/app/angular/notifications/notification/notification.js @@ -49,6 +49,7 @@ angular.module('lemur') }); }); NotificationService.getCertificates(notification); + $scope.page = 1; }); PluginService.getByType('notification').then(function (plugins) { @@ -86,5 +87,10 @@ angular.module('lemur') $uibModalInstance.dismiss('cancel'); }; + $scope.loadMoreCertificates = function () { + $scope.page++; + NotificationService.loadMoreCertificates($scope.notification, $scope.page); + }; + $scope.certificateService = CertificateService; }); diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index 02443701..535c52f8 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -42,7 +42,7 @@ angular.module('lemur') NotificationService.loadMoreCertificates = function (notification, page) { notification.getList('certificates', {page: page}).then(function (certificates) { _.each(certificates, function (certificate) { - notification.roles.push(certificate); + notification.certificates.push(certificate); }); }); }; From 8e8a89bdfb6b5eeea6d5f0c7be4175862088f056 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Mon, 2 Nov 2020 16:17:11 -0800 Subject: [PATCH 002/100] Refactor notification PUT to expect add/remove sets instead of full certificate set --- lemur/notifications/messaging.py | 3 ++- lemur/notifications/schemas.py | 2 ++ lemur/notifications/service.py | 14 +++++++++++-- lemur/notifications/views.py | 3 +++ .../app/angular/notifications/services.js | 20 ++++++++++++++++++- 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 3928689e..35cb1806 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -105,6 +105,7 @@ def send_plugin_notification(event_type, data, recipients, notification): "message": f"Sending expiration notification for to recipients {recipients}", "notification_type": "expiration", "certificate_targets": recipients, + "plugin": notification.plugin.slug, } status = FAILURE_METRIC_STATUS try: @@ -120,7 +121,7 @@ def send_plugin_notification(event_type, data, recipients, notification): "notification", "counter", 1, - metric_tags={"status": status, "event_type": event_type}, + metric_tags={"status": status, "event_type": event_type, "plugin": notification.plugin.slug}, ) if status == SUCCESS_METRIC_STATUS: diff --git a/lemur/notifications/schemas.py b/lemur/notifications/schemas.py index a3ff4c99..6ef5c506 100644 --- a/lemur/notifications/schemas.py +++ b/lemur/notifications/schemas.py @@ -21,6 +21,8 @@ class NotificationInputSchema(LemurInputSchema): active = fields.Boolean() plugin = fields.Nested(PluginInputSchema, required=True) certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) + added_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) + removed_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) class NotificationOutputSchema(LemurOutputSchema): diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 34edccc0..84dbef6b 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -104,7 +104,7 @@ def create(label, plugin_name, options, description, certificates): return database.create(notification) -def update(notification_id, label, plugin_name, options, description, active, certificates): +def update(notification_id, label, plugin_name, options, description, active, certificates, added_certificates, removed_certificates): """ Updates an existing notification. @@ -115,6 +115,8 @@ def update(notification_id, label, plugin_name, options, description, active, ce :param description: :param active: :param certificates: + :param added_certificates: + :param removed_certificates: :rtype : Notification :return: """ @@ -125,7 +127,15 @@ def update(notification_id, label, plugin_name, options, description, active, ce notification.options = options notification.description = description notification.active = active - notification.certificates = certificates + current_app.logger.info(f"Initial: {notification.certificates}") + current_app.logger.info(f"Adding: {added_certificates}") + current_app.logger.info(f"Removing: {removed_certificates}") + if certificates: + notification.certificates = certificates + else: + notification.certificates = notification.certificates + added_certificates + notification.certificates = [c for c in notification.certificates if c not in removed_certificates] + current_app.logger.info(f"Final: {notification.certificates}") return database.update(notification) diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index f6eef655..3f19f5df 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -337,6 +337,7 @@ class Notifications(AuthenticatedResource): :reqheader Authorization: OAuth token to authenticate :statuscode 200: no error """ + print(f"Updating with data: {data}") return service.update( notification_id, data["label"], @@ -345,6 +346,8 @@ class Notifications(AuthenticatedResource): data["description"], data["active"], data["certificates"], + data["added_certificates"], + data["removed_certificates"], ) def delete(self, notification_id): diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index 9e8c9b33..e1a645db 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -8,10 +8,27 @@ angular.module('lemur') if (this.certificates === undefined) { this.certificates = []; } + if (this.addedCertificates === undefined) { + this.addedCertificates = []; + } this.certificates.push(certificate); + this.addedCertificates.push(certificate); + if (this.removedCertificates !== undefined) { + const index = this.removedCertificates.indexOf(certificate); + if (index > -1) { + this.removedCertificates.splice(index, 1); + } + } }, removeCertificate: function (index) { - this.certificates.splice(index, 1); + if (this.removedCertificates === undefined) { + this.removedCertificates = []; + } + const removedCert = this.certificates.splice(index, 1); + this.removedCertificates.push(removedCert); + if (this.addedCertificates !== undefined && this.addedCertificates.indexOf(removedCert) > -1) { + this.addedCertificates.splice(this.addedCertificates.indexOf(removedCert), 1); + } } }); }); @@ -52,6 +69,7 @@ angular.module('lemur') }; NotificationService.update = function (notification) { + this.certificates = []; return notification.put(); }; From 8659504a8ba6e1d57f916b1c27d90348a77eb75e Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Mon, 2 Nov 2020 16:19:30 -0800 Subject: [PATCH 003/100] Remove debug logs --- lemur/notifications/service.py | 4 ---- lemur/notifications/views.py | 1 - 2 files changed, 5 deletions(-) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 84dbef6b..5ec6f045 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -127,15 +127,11 @@ def update(notification_id, label, plugin_name, options, description, active, ce notification.options = options notification.description = description notification.active = active - current_app.logger.info(f"Initial: {notification.certificates}") - current_app.logger.info(f"Adding: {added_certificates}") - current_app.logger.info(f"Removing: {removed_certificates}") if certificates: notification.certificates = certificates else: notification.certificates = notification.certificates + added_certificates notification.certificates = [c for c in notification.certificates if c not in removed_certificates] - current_app.logger.info(f"Final: {notification.certificates}") return database.update(notification) diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index 3f19f5df..c120982c 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -337,7 +337,6 @@ class Notifications(AuthenticatedResource): :reqheader Authorization: OAuth token to authenticate :statuscode 200: no error """ - print(f"Updating with data: {data}") return service.update( notification_id, data["label"], From e9860ee72ae773f316bae6d0161e28ae1b29825a Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 16 Feb 2021 17:54:57 -0800 Subject: [PATCH 004/100] Fix TTL calculation for API keys --- lemur/auth/service.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lemur/auth/service.py b/lemur/auth/service.py index 1705e0c9..6ce9a5b6 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -75,7 +75,7 @@ def create_token(user, aid=None, ttl=None): if ttl == -1: del payload["exp"] else: - payload["exp"] = ttl + payload["exp"] = datetime.utcnow() + timedelta(days=ttl) token = jwt.encode(payload, current_app.config["LEMUR_TOKEN_SECRET"]) return token @@ -116,9 +116,8 @@ def login_required(f): return dict(message="Token has been revoked"), 403 if access_key.ttl != -1: current_time = datetime.utcnow() - expired_time = datetime.fromtimestamp( - access_key.issued_at + access_key.ttl - ) + # API key uses days + expired_time = datetime.fromtimestamp(access_key.issued_at) + timedelta(days=access_key.ttl) if current_time >= expired_time: return dict(message="Token has expired"), 403 From 42044e99aef1a55b5e759e11f29e60670484d7cc Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 16 Feb 2021 18:07:37 -0800 Subject: [PATCH 005/100] Attempt to fix docs build --- requirements-docs.in | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements-docs.in b/requirements-docs.in index f025a85d..e0df9714 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -1,7 +1,11 @@ # Note: python-ldap from requirements breaks due to readthedocs.io not having the correct header files # The `make up-reqs` will update all requirement text files, and forcibly remove python-ldap # from requirements-docs.txt -# However, dependabot doesn't use `make up-reqs`, so `-r requirements.txt` has been removed completely. +# However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here +# Without including these dependencies, the docs are unable to include generated autodocs +Flask + +# docs specific sphinx sphinxcontrib-httpdomain sphinx-rtd-theme From 8cabffcb70d4f9b3553a883a21cc162b1ffd1f81 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 16 Feb 2021 18:13:04 -0800 Subject: [PATCH 006/100] Attempt to fix docs build --- requirements-docs.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-docs.txt b/requirements-docs.txt index 2e76e73f..c33a9f9f 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,6 +18,8 @@ idna==2.9 # via requests imagesize==1.2.0 # via sphinx +flask==1.1.2 + # manual debug jinja2==2.11.3 # via sphinx markupsafe==1.1.1 From e607210fe9a8157ad05b6ac737fd2247e39e1736 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 16 Feb 2021 18:22:07 -0800 Subject: [PATCH 007/100] Add .readthedocs.yml file --- .readthedocs.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..54eb8741 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,22 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Build docs in all formats (html, pdf, epub) +formats: all + +# Set the version of Python and requirements required to build the docs +python: + version: 3.7 + install: + - requirements: requirements-docs.txt + - method: setuptools + path: . + system_packages: true \ No newline at end of file From 160ecd926d9b2af205b758129b0e7a1b2ec1e377 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 09:42:07 -0800 Subject: [PATCH 008/100] Debug docs --- .../lemur_email/templates/revocation.html | 163 ++++++++++++++++++ requirements-docs.in | 1 + requirements-docs.txt | 2 + 3 files changed, 166 insertions(+) create mode 100644 lemur/plugins/lemur_email/templates/revocation.html diff --git a/lemur/plugins/lemur_email/templates/revocation.html b/lemur/plugins/lemur_email/templates/revocation.html new file mode 100644 index 00000000..58625786 --- /dev/null +++ b/lemur/plugins/lemur_email/templates/revocation.html @@ -0,0 +1,163 @@ + + + + + + + + Lemur + + +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ Lemur +
+
+ + + + + + + + + + + + + + +
+ Your certificate has been revoked! +
+
+ + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ Hi, +
This is a Lemur certificate revocation notice. +
+ + + + + + + + +
+ {{ message.certificates.name }} + +
{{ message.certificates.endpoints | length }} Endpoints +
{{ message.certificates.owner }} +
{{ message.certificates.validityEnd | time }} +
{{ message.certificates.status }} +
Details + +
+
+ If this revocation was unexpected, please reach out to {{ ", ".join(message.certificates.security_email) }}. +
+
Best,
Lemur +
+ + + + + + +
*All times are in UTC
+
+
+
+ + + + + + + + + +
You received this mandatory email announcement to update you about + important changes to your TLS certificate. +
+
© 2016 Lemur
+
+
+
+
diff --git a/requirements-docs.in b/requirements-docs.in index e0df9714..f2551059 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -4,6 +4,7 @@ # However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here # Without including these dependencies, the docs are unable to include generated autodocs Flask +flask_replicated # docs specific sphinx diff --git a/requirements-docs.txt b/requirements-docs.txt index c33a9f9f..7d094b77 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -20,6 +20,8 @@ imagesize==1.2.0 # via sphinx flask==1.1.2 # manual debug +flask-replicated==1.4 + # manual debug jinja2==2.11.3 # via sphinx markupsafe==1.1.1 From 058877d76b9d46e728d68b65439222928a4edc5f Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 09:47:52 -0800 Subject: [PATCH 009/100] Remove accidental file --- .../lemur_email/templates/revocation.html | 163 ------------------ 1 file changed, 163 deletions(-) delete mode 100644 lemur/plugins/lemur_email/templates/revocation.html diff --git a/lemur/plugins/lemur_email/templates/revocation.html b/lemur/plugins/lemur_email/templates/revocation.html deleted file mode 100644 index 58625786..00000000 --- a/lemur/plugins/lemur_email/templates/revocation.html +++ /dev/null @@ -1,163 +0,0 @@ - - - - - - - - Lemur - - -
- - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - -
- - - - - - -
- Lemur -
-
- - - - - - - - - - - - - - -
- Your certificate has been revoked! -
-
- - - - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - -
- Hi, -
This is a Lemur certificate revocation notice. -
- - - - - - - - -
- {{ message.certificates.name }} - -
{{ message.certificates.endpoints | length }} Endpoints -
{{ message.certificates.owner }} -
{{ message.certificates.validityEnd | time }} -
{{ message.certificates.status }} -
Details - -
-
- If this revocation was unexpected, please reach out to {{ ", ".join(message.certificates.security_email) }}. -
-
Best,
Lemur -
- - - - - - -
*All times are in UTC
-
-
-
- - - - - - - - - -
You received this mandatory email announcement to update you about - important changes to your TLS certificate. -
-
© 2016 Lemur
-
-
-
-
From 45b84bd08831a392156ca914816dff6e7fdfbdc4 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 12:44:02 -0800 Subject: [PATCH 010/100] Debug docs --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index f2551059..a308f0ad 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -5,6 +5,7 @@ # Without including these dependencies, the docs are unable to include generated autodocs Flask flask_replicated +logmatic-python # docs specific sphinx diff --git a/requirements-docs.txt b/requirements-docs.txt index 7d094b77..0db302f7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -24,6 +24,8 @@ flask-replicated==1.4 # manual debug jinja2==2.11.3 # via sphinx +logmatic-python==0.1.7 + # manual debug markupsafe==1.1.1 # via jinja2 packaging==20.3 From e29ebb4b61d6f14d8ff3e11c8b19d9028d869d84 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 12:44:32 -0800 Subject: [PATCH 011/100] Add arrow --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index a308f0ad..987b5b34 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -3,6 +3,7 @@ # from requirements-docs.txt # However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here # Without including these dependencies, the docs are unable to include generated autodocs +arrow Flask flask_replicated logmatic-python diff --git a/requirements-docs.txt b/requirements-docs.txt index 0db302f7..8c7735ff 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -6,6 +6,8 @@ # alabaster==0.7.12 # via sphinx +arrow==0.17.0 + # manual debug babel==2.8.0 # via sphinx certifi==2020.12.5 From ec9e1c0dd08d9ab8a0ad85bd4987c94b7cd78952 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 12:53:49 -0800 Subject: [PATCH 012/100] Add cryptography --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 987b5b34..bf8c22a2 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -4,6 +4,7 @@ # However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here # Without including these dependencies, the docs are unable to include generated autodocs arrow +cryptography Flask flask_replicated logmatic-python diff --git a/requirements-docs.txt b/requirements-docs.txt index 8c7735ff..71282c7b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -14,6 +14,8 @@ certifi==2020.12.5 # via requests chardet==3.0.4 # via requests +cryptography==3.4.5 + # manual debug docutils==0.15.2 # via sphinx idna==2.9 From b265ecf588067a2ae973486b6a3a78d616bca8e0 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 12:56:27 -0800 Subject: [PATCH 013/100] Make sure it's still broken if we add everything --- requirements-docs.in | 1 + requirements-docs.txt | 262 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index bf8c22a2..a90c30d7 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -3,6 +3,7 @@ # from requirements-docs.txt # However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here # Without including these dependencies, the docs are unable to include generated autodocs +-r requirements.txt arrow cryptography Flask diff --git a/requirements-docs.txt b/requirements-docs.txt index 71282c7b..6f641617 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -71,6 +71,268 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx urllib3==1.25.8 # via requests +acme==1.12.0 + # via -r requirements.in +alembic-autogenerate-enums==0.0.2 + # via -r requirements.in +alembic==1.4.2 + # via flask-migrate +amqp==2.5.2 + # via kombu +aniso8601==8.0.0 + # via flask-restful +arrow==0.17.0 + # via -r requirements.in +asyncpool==1.0 + # via -r requirements.in +bcrypt==3.1.7 + # via + # flask-bcrypt + # paramiko +beautifulsoup4==4.9.1 + # via cloudflare +billiard==3.6.3.0 + # via celery +blinker==1.4 + # via + # flask-mail + # flask-principal + # raven +boto3==1.17.7 + # via -r requirements.in +botocore==1.20.7 + # via + # -r requirements.in + # boto3 + # s3transfer +celery[redis]==4.4.2 + # via -r requirements.in +certifi==2020.12.5 + # via + # -r requirements.in + # requests +certsrv==2.1.1 + # via -r requirements.in +cffi==1.14.0 + # via + # bcrypt + # cryptography + # pynacl +chardet==3.0.4 + # via requests +click==7.1.2 + # via flask +cloudflare==2.8.15 + # via -r requirements.in +cryptography==3.4.5 + # via + # -r requirements.in + # acme + # josepy + # paramiko + # pyopenssl + # requests +dnspython3==1.15.0 + # via -r requirements.in +dnspython==1.15.0 + # via dnspython3 +dyn==1.8.1 + # via -r requirements.in +flask-bcrypt==0.7.1 + # via -r requirements.in +flask-cors==3.0.10 + # via -r requirements.in +flask-mail==0.9.1 + # via -r requirements.in +flask-migrate==2.6.0 + # via -r requirements.in +flask-principal==0.4.0 + # via -r requirements.in +flask-replicated==1.4 + # via -r requirements.in +flask-restful==0.3.8 + # via -r requirements.in +flask-script==2.0.6 + # via -r requirements.in +flask-sqlalchemy==2.4.4 + # via + # -r requirements.in + # flask-migrate +flask==1.1.2 + # via + # -r requirements.in + # flask-bcrypt + # flask-cors + # flask-mail + # flask-migrate + # flask-principal + # flask-restful + # flask-script + # flask-sqlalchemy + # raven +future==0.18.2 + # via -r requirements.in +gunicorn==20.0.4 + # via -r requirements.in +hvac==0.10.8 + # via -r requirements.in +idna==2.9 + # via requests +inflection==0.5.1 + # via -r requirements.in +itsdangerous==1.1.0 + # via flask +javaobj-py3==0.4.0.1 + # via pyjks +jinja2==2.11.3 + # via + # -r requirements.in + # flask +jmespath==0.9.5 + # via + # boto3 + # botocore +josepy==1.3.0 + # via acme +jsonlines==1.2.0 + # via cloudflare +kombu==4.6.8 + # via celery +lockfile==0.12.2 + # via -r requirements.in +logmatic-python==0.1.7 + # via -r requirements.in +mako==1.1.2 + # via alembic +markupsafe==1.1.1 + # via + # jinja2 + # mako +marshmallow-sqlalchemy==0.23.1 + # via -r requirements.in +marshmallow==2.20.4 + # via + # -r requirements.in + # marshmallow-sqlalchemy +ndg-httpsclient==0.5.1 + # via -r requirements.in +paramiko==2.7.2 + # via -r requirements.in +pem==21.1.0 + # via -r requirements.in +psycopg2==2.8.6 + # via -r requirements.in +pyasn1-modules==0.2.8 + # via + # pyjks + # python-ldap +pyasn1==0.4.8 + # via + # ndg-httpsclient + # pyasn1-modules + # pyjks + # python-ldap +pycparser==2.20 + # via cffi +pycryptodomex==3.9.7 + # via pyjks +pyjks==20.0.0 + # via -r requirements.in +pyjwt==2.0.1 + # via -r requirements.in +pynacl==1.3.0 + # via paramiko +pyopenssl==20.0.1 + # via + # -r requirements.in + # acme + # josepy + # ndg-httpsclient + # requests +pyrfc3339==1.1 + # via acme +python-dateutil==2.8.1 + # via + # alembic + # arrow + # botocore +python-editor==1.0.4 + # via alembic +python-json-logger==0.1.11 + # via logmatic-python +python-ldap==3.3.1 + # via -r requirements.in +pytz==2019.3 + # via + # acme + # celery + # flask-restful + # pyrfc3339 +pyyaml==5.4.1 + # via + # -r requirements.in + # cloudflare +raven[flask]==6.10.0 + # via -r requirements.in +redis==3.5.3 + # via + # -r requirements.in + # celery +requests-toolbelt==0.9.1 + # via acme +requests[security]==2.25.1 + # via + # -r requirements.in + # acme + # certsrv + # cloudflare + # hvac + # requests-toolbelt +retrying==1.3.3 + # via -r requirements.in +s3transfer==0.3.3 + # via boto3 +six==1.15.0 + # via + # -r requirements.in + # acme + # bcrypt + # flask-cors + # flask-restful + # hvac + # josepy + # jsonlines + # pynacl + # pyopenssl + # python-dateutil + # retrying + # sqlalchemy-utils +soupsieve==2.0.1 + # via beautifulsoup4 +sqlalchemy-utils==0.36.8 + # via -r requirements.in +sqlalchemy==1.3.16 + # via + # alembic + # flask-sqlalchemy + # marshmallow-sqlalchemy + # sqlalchemy-utils +tabulate==0.8.7 + # via -r requirements.in +twofish==0.3.0 + # via pyjks +urllib3==1.25.8 + # via + # botocore + # requests +vine==1.3.0 + # via + # amqp + # celery +werkzeug==1.0.1 + # via flask +xmltodict==0.12.0 + # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: # setuptools From 938b962a327c3722329065b44f84758b3ccc7a8b Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:05:14 -0800 Subject: [PATCH 014/100] Undo add everything, add just sqlalchemy --- requirements-docs.in | 2 +- requirements-docs.txt | 264 +----------------------------------------- 2 files changed, 3 insertions(+), 263 deletions(-) diff --git a/requirements-docs.in b/requirements-docs.in index a90c30d7..755d0697 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -3,10 +3,10 @@ # from requirements-docs.txt # However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here # Without including these dependencies, the docs are unable to include generated autodocs --r requirements.txt arrow cryptography Flask +Flask-SQLAlchemy flask_replicated logmatic-python diff --git a/requirements-docs.txt b/requirements-docs.txt index 6f641617..2beae50f 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -26,6 +26,8 @@ flask==1.1.2 # manual debug flask-replicated==1.4 # manual debug +flask-sqlalchemy==2.4.4 + # manual debug jinja2==2.11.3 # via sphinx logmatic-python==0.1.7 @@ -71,268 +73,6 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx urllib3==1.25.8 # via requests -acme==1.12.0 - # via -r requirements.in -alembic-autogenerate-enums==0.0.2 - # via -r requirements.in -alembic==1.4.2 - # via flask-migrate -amqp==2.5.2 - # via kombu -aniso8601==8.0.0 - # via flask-restful -arrow==0.17.0 - # via -r requirements.in -asyncpool==1.0 - # via -r requirements.in -bcrypt==3.1.7 - # via - # flask-bcrypt - # paramiko -beautifulsoup4==4.9.1 - # via cloudflare -billiard==3.6.3.0 - # via celery -blinker==1.4 - # via - # flask-mail - # flask-principal - # raven -boto3==1.17.7 - # via -r requirements.in -botocore==1.20.7 - # via - # -r requirements.in - # boto3 - # s3transfer -celery[redis]==4.4.2 - # via -r requirements.in -certifi==2020.12.5 - # via - # -r requirements.in - # requests -certsrv==2.1.1 - # via -r requirements.in -cffi==1.14.0 - # via - # bcrypt - # cryptography - # pynacl -chardet==3.0.4 - # via requests -click==7.1.2 - # via flask -cloudflare==2.8.15 - # via -r requirements.in -cryptography==3.4.5 - # via - # -r requirements.in - # acme - # josepy - # paramiko - # pyopenssl - # requests -dnspython3==1.15.0 - # via -r requirements.in -dnspython==1.15.0 - # via dnspython3 -dyn==1.8.1 - # via -r requirements.in -flask-bcrypt==0.7.1 - # via -r requirements.in -flask-cors==3.0.10 - # via -r requirements.in -flask-mail==0.9.1 - # via -r requirements.in -flask-migrate==2.6.0 - # via -r requirements.in -flask-principal==0.4.0 - # via -r requirements.in -flask-replicated==1.4 - # via -r requirements.in -flask-restful==0.3.8 - # via -r requirements.in -flask-script==2.0.6 - # via -r requirements.in -flask-sqlalchemy==2.4.4 - # via - # -r requirements.in - # flask-migrate -flask==1.1.2 - # via - # -r requirements.in - # flask-bcrypt - # flask-cors - # flask-mail - # flask-migrate - # flask-principal - # flask-restful - # flask-script - # flask-sqlalchemy - # raven -future==0.18.2 - # via -r requirements.in -gunicorn==20.0.4 - # via -r requirements.in -hvac==0.10.8 - # via -r requirements.in -idna==2.9 - # via requests -inflection==0.5.1 - # via -r requirements.in -itsdangerous==1.1.0 - # via flask -javaobj-py3==0.4.0.1 - # via pyjks -jinja2==2.11.3 - # via - # -r requirements.in - # flask -jmespath==0.9.5 - # via - # boto3 - # botocore -josepy==1.3.0 - # via acme -jsonlines==1.2.0 - # via cloudflare -kombu==4.6.8 - # via celery -lockfile==0.12.2 - # via -r requirements.in -logmatic-python==0.1.7 - # via -r requirements.in -mako==1.1.2 - # via alembic -markupsafe==1.1.1 - # via - # jinja2 - # mako -marshmallow-sqlalchemy==0.23.1 - # via -r requirements.in -marshmallow==2.20.4 - # via - # -r requirements.in - # marshmallow-sqlalchemy -ndg-httpsclient==0.5.1 - # via -r requirements.in -paramiko==2.7.2 - # via -r requirements.in -pem==21.1.0 - # via -r requirements.in -psycopg2==2.8.6 - # via -r requirements.in -pyasn1-modules==0.2.8 - # via - # pyjks - # python-ldap -pyasn1==0.4.8 - # via - # ndg-httpsclient - # pyasn1-modules - # pyjks - # python-ldap -pycparser==2.20 - # via cffi -pycryptodomex==3.9.7 - # via pyjks -pyjks==20.0.0 - # via -r requirements.in -pyjwt==2.0.1 - # via -r requirements.in -pynacl==1.3.0 - # via paramiko -pyopenssl==20.0.1 - # via - # -r requirements.in - # acme - # josepy - # ndg-httpsclient - # requests -pyrfc3339==1.1 - # via acme -python-dateutil==2.8.1 - # via - # alembic - # arrow - # botocore -python-editor==1.0.4 - # via alembic -python-json-logger==0.1.11 - # via logmatic-python -python-ldap==3.3.1 - # via -r requirements.in -pytz==2019.3 - # via - # acme - # celery - # flask-restful - # pyrfc3339 -pyyaml==5.4.1 - # via - # -r requirements.in - # cloudflare -raven[flask]==6.10.0 - # via -r requirements.in -redis==3.5.3 - # via - # -r requirements.in - # celery -requests-toolbelt==0.9.1 - # via acme -requests[security]==2.25.1 - # via - # -r requirements.in - # acme - # certsrv - # cloudflare - # hvac - # requests-toolbelt -retrying==1.3.3 - # via -r requirements.in -s3transfer==0.3.3 - # via boto3 -six==1.15.0 - # via - # -r requirements.in - # acme - # bcrypt - # flask-cors - # flask-restful - # hvac - # josepy - # jsonlines - # pynacl - # pyopenssl - # python-dateutil - # retrying - # sqlalchemy-utils -soupsieve==2.0.1 - # via beautifulsoup4 -sqlalchemy-utils==0.36.8 - # via -r requirements.in -sqlalchemy==1.3.16 - # via - # alembic - # flask-sqlalchemy - # marshmallow-sqlalchemy - # sqlalchemy-utils -tabulate==0.8.7 - # via -r requirements.in -twofish==0.3.0 - # via pyjks -urllib3==1.25.8 - # via - # botocore - # requests -vine==1.3.0 - # via - # amqp - # celery -werkzeug==1.0.1 - # via flask -xmltodict==0.12.0 - # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: # setuptools From 91f6f752db94aa3c64df17cb27095cf732ebfb98 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:08:40 -0800 Subject: [PATCH 015/100] Add inflection --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 755d0697..bc58be7e 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -8,6 +8,7 @@ cryptography Flask Flask-SQLAlchemy flask_replicated +inflection logmatic-python # docs specific diff --git a/requirements-docs.txt b/requirements-docs.txt index 2beae50f..89a4ea93 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -28,6 +28,8 @@ flask-replicated==1.4 # manual debug flask-sqlalchemy==2.4.4 # manual debug +inflection==0.5.1 + # manual debug jinja2==2.11.3 # via sphinx logmatic-python==0.1.7 From bfa1c067d97e8b5534c557ec0ae1ade8d4aff90a Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:12:10 -0800 Subject: [PATCH 016/100] Add flask-migrate --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index bc58be7e..8da716ad 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -6,6 +6,7 @@ arrow cryptography Flask +Flask-Migrate Flask-SQLAlchemy flask_replicated inflection diff --git a/requirements-docs.txt b/requirements-docs.txt index 89a4ea93..99e8dd83 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -24,6 +24,8 @@ imagesize==1.2.0 # via sphinx flask==1.1.2 # manual debug +flask-migrate==2.6.0 + # manual debug flask-replicated==1.4 # manual debug flask-sqlalchemy==2.4.4 From abdf544e06ec04a18de6382bffb03cbbd44af3b2 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:16:16 -0800 Subject: [PATCH 017/100] Add flask-restful --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 8da716ad..992755e0 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -7,6 +7,7 @@ arrow cryptography Flask Flask-Migrate +Flask-RESTful Flask-SQLAlchemy flask_replicated inflection diff --git a/requirements-docs.txt b/requirements-docs.txt index 99e8dd83..d749780b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -28,6 +28,8 @@ flask-migrate==2.6.0 # manual debug flask-replicated==1.4 # manual debug +flask-restful==0.3.8 + # manual debug flask-sqlalchemy==2.4.4 # manual debug inflection==0.5.1 From c0c1022a5b9309cd31de6129510b18b495e521ff Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:18:49 -0800 Subject: [PATCH 018/100] Add flask-bcrypt --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 992755e0..fcad63ef 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -6,6 +6,7 @@ arrow cryptography Flask +Flask-Bcrypt Flask-Migrate Flask-RESTful Flask-SQLAlchemy diff --git a/requirements-docs.txt b/requirements-docs.txt index d749780b..b613e052 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -24,6 +24,8 @@ imagesize==1.2.0 # via sphinx flask==1.1.2 # manual debug +flask-bcrypt==0.7.1 + # manual debug flask-migrate==2.6.0 # manual debug flask-replicated==1.4 From bbdacaccf916595afe6b9339144f0047ffa0f288 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:22:25 -0800 Subject: [PATCH 019/100] Add flask-principal --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index fcad63ef..d2b2239c 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -8,6 +8,7 @@ cryptography Flask Flask-Bcrypt Flask-Migrate +Flask-Principal Flask-RESTful Flask-SQLAlchemy flask_replicated diff --git a/requirements-docs.txt b/requirements-docs.txt index b613e052..e596b829 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -28,6 +28,8 @@ flask-bcrypt==0.7.1 # manual debug flask-migrate==2.6.0 # manual debug +flask-principal==0.4.0 + # manual debug flask-replicated==1.4 # manual debug flask-restful==0.3.8 From 6aff89c1dc0c1ab9361787271def25a86c9208bf Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:26:13 -0800 Subject: [PATCH 020/100] Add flask-mail, flask-script --- requirements-docs.in | 2 ++ requirements-docs.txt | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index d2b2239c..88c14dc3 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -7,9 +7,11 @@ arrow cryptography Flask Flask-Bcrypt +Flask-Mail Flask-Migrate Flask-Principal Flask-RESTful +Flask-Script Flask-SQLAlchemy flask_replicated inflection diff --git a/requirements-docs.txt b/requirements-docs.txt index e596b829..ad97eaed 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -26,6 +26,8 @@ flask==1.1.2 # manual debug flask-bcrypt==0.7.1 # manual debug +flask-mail==0.9.1 + # manual debug flask-migrate==2.6.0 # manual debug flask-principal==0.4.0 @@ -34,6 +36,8 @@ flask-replicated==1.4 # manual debug flask-restful==0.3.8 # manual debug +flask-script==2.0.6 + # manual debug flask-sqlalchemy==2.4.4 # manual debug inflection==0.5.1 From 5e46e2adf031162214430bd09a16f8d3c728261a Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:38:15 -0800 Subject: [PATCH 021/100] Add raven --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 88c14dc3..ade68fba 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -16,6 +16,7 @@ Flask-SQLAlchemy flask_replicated inflection logmatic-python +raven[flask] # docs specific sphinx diff --git a/requirements-docs.txt b/requirements-docs.txt index ad97eaed..f8b71f8f 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -56,6 +56,8 @@ pyparsing==2.4.7 # via packaging pytz==2019.3 # via babel +raven[flask]==6.10.0 + # manual debug requests==2.25.1 # via sphinx six==1.15.0 From 1ab4fe278dfa8657bf99f0e2c4fd7b59bddf5e8a Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:41:27 -0800 Subject: [PATCH 022/100] Add flask-cors --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index ade68fba..11c4c1d1 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -7,6 +7,7 @@ arrow cryptography Flask Flask-Bcrypt +Flask-Cors Flask-Mail Flask-Migrate Flask-Principal diff --git a/requirements-docs.txt b/requirements-docs.txt index f8b71f8f..56bfd882 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -26,6 +26,8 @@ flask==1.1.2 # manual debug flask-bcrypt==0.7.1 # manual debug +flask-cors==3.0.10 + # manual debug flask-mail==0.9.1 # manual debug flask-migrate==2.6.0 From e9e79309c55a3faadfbc4dbc3621a7aec6c4cab6 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:43:49 -0800 Subject: [PATCH 023/100] Add sqlalchemy-utils --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 11c4c1d1..9dee5474 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -18,6 +18,7 @@ flask_replicated inflection logmatic-python raven[flask] +SQLAlchemy-Utils # docs specific sphinx diff --git a/requirements-docs.txt b/requirements-docs.txt index 56bfd882..9fdaeb63 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -89,6 +89,8 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy-utils==0.36.8 + # manual debug urllib3==1.25.8 # via requests From 6aa6986a143eabf65ef489191178cc4c264b5586 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:45:55 -0800 Subject: [PATCH 024/100] Add pem --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 9dee5474..b6c8d2f4 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -17,6 +17,7 @@ Flask-SQLAlchemy flask_replicated inflection logmatic-python +pem raven[flask] SQLAlchemy-Utils diff --git a/requirements-docs.txt b/requirements-docs.txt index 9fdaeb63..7d99dd51 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -52,6 +52,8 @@ markupsafe==1.1.1 # via jinja2 packaging==20.3 # via sphinx +pem==21.1.0 + # manual debug pygments==2.6.1 # via sphinx pyparsing==2.4.7 From 8086d7afc068719e887645d79961e588478503f2 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:47:59 -0800 Subject: [PATCH 025/100] Add marshmallow --- requirements-docs.in | 2 ++ requirements-docs.txt | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index b6c8d2f4..46650ccc 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -17,6 +17,8 @@ Flask-SQLAlchemy flask_replicated inflection logmatic-python +marshmallow-sqlalchemy +marshmallow<2.20.5 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 pem raven[flask] SQLAlchemy-Utils diff --git a/requirements-docs.txt b/requirements-docs.txt index 7d99dd51..f175d8d0 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -50,6 +50,10 @@ logmatic-python==0.1.7 # manual debug markupsafe==1.1.1 # via jinja2 +marshmallow-sqlalchemy==0.23.1 + # manual debug +marshmallow==2.20.4 + # manual debug packaging==20.3 # via sphinx pem==21.1.0 From bfe3358b16582977e2eb75dfa6e844d4d85f7cce Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:50:05 -0800 Subject: [PATCH 026/100] Add pyjwt --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 46650ccc..d82ea511 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -20,6 +20,7 @@ logmatic-python marshmallow-sqlalchemy marshmallow<2.20.5 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 pem +pyjwt raven[flask] SQLAlchemy-Utils diff --git a/requirements-docs.txt b/requirements-docs.txt index f175d8d0..d815df7d 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -60,6 +60,8 @@ pem==21.1.0 # manual debug pygments==2.6.1 # via sphinx +pyjwt==2.0.1 + # manual debug pyparsing==2.4.7 # via packaging pytz==2019.3 From 40e5c60c397ab47fce24de9cacb01183a90ccdc3 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:52:20 -0800 Subject: [PATCH 027/100] Fix some doc warnings --- CHANGELOG.rst | 4 ++-- docs/administration.rst | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 67b792f8..8fb4f8ed 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,7 @@ Changelog ========= 0.8.0 - `2020-11-13` -~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~ This release comes after more than two years and contains many interesting new features and improvements. In addition to multiple new plugins, such as ACME-http01, ADCS, PowerDNS, UltraDNS, Entrust, SNS, many of Lemur's existing @@ -84,7 +84,7 @@ Upgrading 0.7 - `2018-05-07` -~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~ This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality. The linux_dst plugin will also be deprecated and removed. diff --git a/docs/administration.rst b/docs/administration.rst index 3623f311..706c4027 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1640,7 +1640,7 @@ Slack AWS (Source) ----- +------------ :Authors: Kevin Glisson , @@ -1653,7 +1653,7 @@ AWS (Source) AWS (Destination) ----- +----------------- :Authors: Kevin Glisson , @@ -1666,7 +1666,7 @@ AWS (Destination) AWS (SNS Notification) ------ +---------------------- :Authors: Jasmine Schladen From 8c666b7f0bd5f2f00be4d4ad92efd07125e70e41 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:53:22 -0800 Subject: [PATCH 028/100] Add gunicorn --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index d82ea511..22161779 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -15,6 +15,7 @@ Flask-RESTful Flask-Script Flask-SQLAlchemy flask_replicated +gunicorn inflection logmatic-python marshmallow-sqlalchemy diff --git a/requirements-docs.txt b/requirements-docs.txt index d815df7d..79e44b94 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -42,6 +42,8 @@ flask-script==2.0.6 # manual debug flask-sqlalchemy==2.4.4 # manual debug +gunicorn==20.0.4 + # manual debug inflection==0.5.1 # manual debug jinja2==2.11.3 From dfad5ae968df2612b891a4a80618f2ffc93649ee Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 13:56:04 -0800 Subject: [PATCH 029/100] Add pyopenssl --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 22161779..99d16b29 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -22,6 +22,7 @@ marshmallow-sqlalchemy marshmallow<2.20.5 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 pem pyjwt +pyOpenSSL raven[flask] SQLAlchemy-Utils diff --git a/requirements-docs.txt b/requirements-docs.txt index 79e44b94..1c53e890 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -64,6 +64,8 @@ pygments==2.6.1 # via sphinx pyjwt==2.0.1 # manual debug +pyopenssl==20.0.1 + # manual debug pyparsing==2.4.7 # via packaging pytz==2019.3 From c4a896ecf214f28832e1d01102864ae645121766 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:00:42 -0800 Subject: [PATCH 030/100] Add josepy --- CHANGELOG.rst | 21 ++++++--------------- docs/administration.rst | 6 +++--- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8fb4f8ed..22a9341f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -121,8 +121,7 @@ Happy Holidays! This is a big release with lots of bug fixes and features. Below Features: -* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates. -is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used. +* Per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates is 30 days. Every certificate will gain a policy regardless of if auto-rotation is used. * Adds per-user API Keys, allows users to issue multiple long-lived API tokens with the same permission as the user creating them. * Adds the ability to revoke certificates from the Lemur UI/API, this is currently only supported for the digicert CIS and cfssl plugins. * Allow destinations to support an export function. Useful for file system destinations e.g. S3 to specify the export plugin you wish to run before being sent to the destination. @@ -166,13 +165,9 @@ Big thanks to neilschelly for quite a lot of improvements to the `lemur-cryptogr Other Highlights: -* Closed `#501 `_ - Endpoint resource as now kept in sync via an -expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never -removed from Lemur. -* Closed `#551 `_ - Added the ability to create a 4096 bit key during certificate -creation. Closed `#528 `_ to ensure that issuer plugins supported the new 4096 bit keys. -* Closed `#566 `_ - Fixed an issue changing the notification status for certificates -without private keys. +* Closed `#501 `_ - Endpoint resource as now kept in sync via an expiration mechanism. Such that non-existant endpoints gracefully fall out of Lemur. Certificates are never removed from Lemur. +* Closed `#551 `_ - Added the ability to create a 4096 bit key during certificate creation. Closed `#528 `_ to ensure that issuer plugins supported the new 4096 bit keys. +* Closed `#566 `_ - Fixed an issue changing the notification status for certificates without private keys. * Closed `#594 `_ - Added `replaced` field indicating if a certificate has been superseded. * Closed `#602 `_ - AWS plugin added support for ALBs for endpoint tracking. @@ -196,12 +191,8 @@ Upgrading There have been quite a few issues closed in this release. Some notables: -* Closed `#284 `_ - Created new models for `Endpoints` created associated -AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for -future enhancements of Lemur's certificate 'deployment' capabilities. - -* Closed `#334 `_ - Lemur not has the ability -to restrict certificate expiration dates to weekdays. +* Closed `#284 `_ - Created new models for `Endpoints` created associated AWS ELB endpoint tracking code. This was the major stated goal of this milestone and should serve as the basis for future enhancements of Lemur's certificate 'deployment' capabilities. +* Closed `#334 `_ - Lemur not has the ability to restrict certificate expiration dates to weekdays. Several fixes/tweaks to Lemurs python3 support (thanks chadhendrie!) diff --git a/docs/administration.rst b/docs/administration.rst index 706c4027..5cf398d5 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -78,13 +78,13 @@ Basic Configuration The default connection pool size is 5 for sqlalchemy managed connections. Depending on the number of Lemur instances, please specify per instance connection pool size. Below is an example to set connection pool size to 10. - :: + :: SQLALCHEMY_POOL_SIZE = 10 .. warning:: -This is an optional setting but important to review and set for optimal database connection usage and for overall database performance. + This is an optional setting but important to review and set for optimal database connection usage and for overall database performance. .. data:: SQLALCHEMY_MAX_OVERFLOW :noindex: @@ -99,7 +99,7 @@ This is an optional setting but important to review and set for optimal database .. note:: -Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size. + Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create connections above specified pool size. .. data:: LEMUR_ALLOW_WEEKEND_EXPIRATION diff --git a/requirements-docs.in b/requirements-docs.in index 99d16b29..2254ea54 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -17,6 +17,7 @@ Flask-SQLAlchemy flask_replicated gunicorn inflection +josepy logmatic-python marshmallow-sqlalchemy marshmallow<2.20.5 #schema duplicate issues https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/121 diff --git a/requirements-docs.txt b/requirements-docs.txt index 1c53e890..b9149ee0 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -48,6 +48,8 @@ inflection==0.5.1 # manual debug jinja2==2.11.3 # via sphinx +josepy==1.3.0 + # manual debug logmatic-python==0.1.7 # manual debug markupsafe==1.1.1 From d4643d760a3e1db6662460295e64df88b12cba04 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:07:06 -0800 Subject: [PATCH 031/100] Add dnspython3 --- docs/administration.rst | 4 ++++ lemur/certificates/views.py | 1 + requirements-docs.in | 1 + requirements-docs.txt | 4 ++++ 4 files changed, 10 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 5cf398d5..4cf8e769 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -174,6 +174,7 @@ Basic Configuration .. data:: PUBLIC_CA_MAX_VALIDITY_DAYS :noindex: + Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities. The authorities with cab_compliant option set to true will use this config. The example below overrides the default validity of 397 days and sets it to 365 days. @@ -185,6 +186,7 @@ Basic Configuration .. data:: DEFAULT_VALIDITY_DAYS :noindex: + Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please note that this config is used for cert issuance only through Lemur UI. The example below overrides the default validity @@ -904,10 +906,12 @@ Active Directory Certificate Services Plugin .. data:: ADCS_START :noindex: + Used in ADCS-Sourceplugin. Minimum id of the first certificate to be returned. ID is increased by one until ADCS_STOP. Missing cert-IDs are ignored .. data:: ADCS_STOP :noindex: + Used for ADCS-Sourceplugin. Maximum id of the certificates returned. diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 8d4e6954..f453ac4f 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -59,6 +59,7 @@ class CertificatesListValid(AuthenticatedResource): **Example request**: .. sourcecode:: http + GET /certificates/valid?filter=cn;*.test.example.net&owner=joe@example.com&page=1&count=20 HTTP/1.1 Host: example.com diff --git a/requirements-docs.in b/requirements-docs.in index 2254ea54..b60359dd 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -5,6 +5,7 @@ # Without including these dependencies, the docs are unable to include generated autodocs arrow cryptography +dnspython3 Flask Flask-Bcrypt Flask-Cors diff --git a/requirements-docs.txt b/requirements-docs.txt index b9149ee0..2cff64db 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,6 +16,10 @@ chardet==3.0.4 # via requests cryptography==3.4.5 # manual debug +dnspython3==1.15.0 + # manual debug +dnspython==1.15.0 + # manual debug docutils==0.15.2 # via sphinx idna==2.9 From 824a4b5910d1ebc1037be3bb2113b2f277bef115 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:17:37 -0800 Subject: [PATCH 032/100] add acme, boto, xmltodict --- docs/developer/plugins/index.rst | 9 +++++---- docs/production/index.rst | 2 +- docs/quickstart/index.rst | 6 ++++-- requirements-docs.in | 4 ++++ requirements-docs.txt | 8 ++++++++ 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index 3834b0b9..8ce50014 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -154,9 +154,10 @@ An issuer may take some time to actually issue a certificate for an order. In t # retrieve an order, and check if there is an issued certificate attached to it `cancel_ordered_certificate()` should be implemented to allow an ordered certificate to be canceled before it is issued:: - def cancel_ordered_certificate(self, pending_cert, **kwargs): - # pending_cert should contain the necessary information to match an order - # kwargs can be given to provide information to the issuer for canceling + + def cancel_ordered_certificate(self, pending_cert, **kwargs): + # pending_cert should contain the necessary information to match an order + # kwargs can be given to provide information to the issuer for canceling Destination ----------- @@ -286,7 +287,7 @@ The `ExportPlugin` object requires the implementation of one function:: Custom TLS Provider ------- +------------------- Managing TLS at the enterprise scale could be hard and often organizations offer custom wrapper implementations. It could be ideal to use those while making calls to internal services. The `TLSPlugin` would help to achieve this. It requires the diff --git a/docs/production/index.rst b/docs/production/index.rst index fa0a7dec..3082ee4a 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -501,7 +501,7 @@ rely on celery to create the DNS record. This will change when we implement mix To create a HTTP compatible Authority, you first need to create a new destination that will be used to deploy the challenge token. Visit `Admin` -> `Destination` and click `Create`. The path you provide for the destination needs to -be the exact path that is called when the ACME providers calls ``http:///.well-known/acme-challenge/`. The +be the exact path that is called when the ACME providers calls `http:///.well-known/acme-challenge/`. The token part will be added dynamically by the acme_upload. Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge. diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 3056029d..cf6d3c32 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -148,7 +148,7 @@ Before Lemur will run you need to fill in a few required variables in the config LEMUR_DEFAULT_ORGANIZATIONAL_UNIT Set Up Postgres --------------- +--------------- For production, a dedicated database is recommended, for this guide we will assume postgres has been installed and is on the same machine that Lemur is installed on. @@ -186,6 +186,7 @@ In addition to creating a new user, Lemur also creates a few default email notif Your database installation requires the pg_trgm extension. If you do not have this installed already, you can allow the script to install this for you by adding the SUPERUSER permission to the lemur database user. .. code-block:: bash + sudo -u postgres -i psql postgres=# ALTER USER lemur WITH SUPERUSER @@ -202,6 +203,7 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. note:: If you added the SUPERUSER permission to the lemur database user above, it is recommended you revoke that permission now. .. code-block:: bash + sudo -u postgres -i psql postgres=# ALTER USER lemur WITH NOSUPERUSER @@ -210,7 +212,7 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. Set Up a Reverse Proxy ---------------------- +---------------------- By default, Lemur runs on port 8000. Even if you change this, under normal conditions you won't be able to bind to port 80. To get around this (and to avoid running Lemur as a privileged user, which you shouldn't), we need to set up a simple web proxy. There are many different web servers you can use for this, we like and recommend Nginx. diff --git a/requirements-docs.in b/requirements-docs.in index b60359dd..b21ada07 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -3,7 +3,10 @@ # from requirements-docs.txt # However, dependabot doesn't use `make up-reqs`, so we have to replicate the necessary dependencies here # Without including these dependencies, the docs are unable to include generated autodocs +acme arrow +boto3 +botocore cryptography dnspython3 Flask @@ -27,6 +30,7 @@ pyjwt pyOpenSSL raven[flask] SQLAlchemy-Utils +xmltodict # docs specific sphinx diff --git a/requirements-docs.txt b/requirements-docs.txt index 2cff64db..cfbeb3e5 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,12 +4,18 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # +acme==1.12.0 + # manual debug alabaster==0.7.12 # via sphinx arrow==0.17.0 # manual debug babel==2.8.0 # via sphinx +boto3==1.17.7 + # manual debug +botocore==1.20.7 + # manual debug certifi==2020.12.5 # via requests chardet==3.0.4 @@ -111,6 +117,8 @@ sqlalchemy-utils==0.36.8 # manual debug urllib3==1.25.8 # via requests +xmltodict==0.12.0 + # manual debug # The following packages are considered to be unsafe in a requirements file: # setuptools From 47121906f521c0d81912404c18b6ea6655d51aa9 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:25:04 -0800 Subject: [PATCH 033/100] Add Cloudflare, retrying --- docs/guide/index.rst | 1 + docs/quickstart/index.rst | 4 ++-- requirements-docs.in | 2 ++ requirements-docs.txt | 4 ++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/guide/index.rst b/docs/guide/index.rst index b06a95e0..f3efcb14 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -65,6 +65,7 @@ Import an Existing Certificate You can add notification options and upload the created certificate to a destination, both of these are editable features and can be changed after the certificate has been created. +.. _CreateANewUser: Create a New User ~~~~~~~~~~~~~~~~~ diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index cf6d3c32..f972c2ef 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -191,7 +191,7 @@ Your database installation requires the pg_trgm extension. If you do not have th psql postgres=# ALTER USER lemur WITH SUPERUSER -Additional notifications can be created through the UI or API. See :ref:`Creating Notifications ` and :ref:`Command Line Interface ` for details. +Additional notifications can be created through the UI or API. See :ref:`Notification Options ` and :ref:`Command Line Interface ` for details. **Make note of the password used as this will be used during first login to the Lemur UI.** @@ -209,7 +209,7 @@ Additional notifications can be created through the UI or API. See :ref:`Creati postgres=# ALTER USER lemur WITH NOSUPERUSER -.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. +.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating a New User ` and :ref:`Command Line Interface ` for details. Set Up a Reverse Proxy ---------------------- diff --git a/requirements-docs.in b/requirements-docs.in index b21ada07..6b8f5a81 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -7,6 +7,7 @@ acme arrow boto3 botocore +CloudFlare cryptography dnspython3 Flask @@ -29,6 +30,7 @@ pem pyjwt pyOpenSSL raven[flask] +retrying SQLAlchemy-Utils xmltodict diff --git a/requirements-docs.txt b/requirements-docs.txt index cfbeb3e5..f3e967c6 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -20,6 +20,8 @@ certifi==2020.12.5 # via requests chardet==3.0.4 # via requests +cloudflare==2.8.15 + # manual debug cryptography==3.4.5 # manual debug dnspython3==1.15.0 @@ -84,6 +86,8 @@ pytz==2019.3 # via babel raven[flask]==6.10.0 # manual debug +retrying==1.3.3 + # manual debug requests==2.25.1 # via sphinx six==1.15.0 From e464e62d01912325df6c33bbaa42fcf94ff4f2a2 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:33:00 -0800 Subject: [PATCH 034/100] Add dyn --- docs/developer/plugins/index.rst | 3 +-- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index 8ce50014..517b5a0d 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -145,8 +145,7 @@ The `IssuerPlugin` doesn't have any options like Destination, Source, and Notifi any fields you might need to submit a request to a third party. If there are additional options you need in your plugin feel free to open an issue, or look into adding additional options to issuers yourself. -Asynchronous Certificates -^^^^^^^^^^^^^^^^^^^^^^^^^ +**Asynchronous Certificates** An issuer may take some time to actually issue a certificate for an order. In this case, a `PendingCertificate` is returned, which holds information to recreate a `Certificate` object at a later time. Then, `get_ordered_certificate()` should be run periodically via `python manage.py pending_certs fetch -i all` to attempt to retrieve an ordered certificate:: def get_ordered_ceriticate(self, order_id): diff --git a/requirements-docs.in b/requirements-docs.in index 6b8f5a81..93dd3968 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -10,6 +10,7 @@ botocore CloudFlare cryptography dnspython3 +dyn Flask Flask-Bcrypt Flask-Cors diff --git a/requirements-docs.txt b/requirements-docs.txt index f3e967c6..b18706ae 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -30,6 +30,8 @@ dnspython==1.15.0 # manual debug docutils==0.15.2 # via sphinx +dyn==1.8.1 + # manual debug idna==2.9 # via requests imagesize==1.2.0 From 40f62a0ad7abb4792ca44cc9755e87a4e1aa5ed5 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:35:55 -0800 Subject: [PATCH 035/100] Add tabulate --- requirements-docs.in | 1 + requirements-docs.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index 93dd3968..07b3e987 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -33,6 +33,7 @@ pyOpenSSL raven[flask] retrying SQLAlchemy-Utils +tabulate xmltodict # docs specific diff --git a/requirements-docs.txt b/requirements-docs.txt index b18706ae..41d5133b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -121,6 +121,8 @@ sphinxcontrib-serializinghtml==1.1.4 # via sphinx sqlalchemy-utils==0.36.8 # manual debug +tabulate==0.8.7 + # manual debug urllib3==1.25.8 # via requests xmltodict==0.12.0 From 24c1415983a75eb29d77d7554e06efb614c33c4b Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:47:53 -0800 Subject: [PATCH 036/100] Fix AuthoritiesList post --- lemur/authorities/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/authorities/views.py b/lemur/authorities/views.py index 094a5a74..aa3fbf6d 100644 --- a/lemur/authorities/views.py +++ b/lemur/authorities/views.py @@ -218,8 +218,7 @@ class AuthoritiesList(AuthenticatedResource): :arg parent: the parent authority if this is to be a subca :arg signingAlgorithm: algorithm used to sign the authority :arg keyType: key type - :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored - in an HSM + :arg sensitivity: the sensitivity of the root key, for CloudCA this determines if the root keys are stored in an HSM :arg keyName: name of the key to store in the HSM (CloudCA) :arg serialNumber: serial number of the authority :arg firstSerial: specifies the starting serial number for certificates issued off of this authority From 5f2e32ff92bf4d8af9407c07ffc28342dd74b635 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 14:52:28 -0800 Subject: [PATCH 037/100] Fix AuthorityVisualizations --- lemur/authorities/views.py | 59 +++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/lemur/authorities/views.py b/lemur/authorities/views.py index aa3fbf6d..800c9975 100644 --- a/lemur/authorities/views.py +++ b/lemur/authorities/views.py @@ -493,23 +493,48 @@ class CertificateAuthority(AuthenticatedResource): class AuthorityVisualizations(AuthenticatedResource): def get(self, authority_id): """ - {"name": "flare", - "children": [ - { - "name": "analytics", - "children": [ - { - "name": "cluster", - "children": [ - {"name": "AgglomerativeCluster", "size": 3938}, - {"name": "CommunityStructure", "size": 3812}, - {"name": "HierarchicalCluster", "size": 6714}, - {"name": "MergeEdge", "size": 743} - ] - } - ] - } - ]} + .. http:get:: /authorities/1/visualize + + Authority visualization + + **Example request**: + + .. sourcecode:: http + + GET /certificates/1/visualize HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + {"name": "flare", + "children": [ + { + "name": "analytics", + "children": [ + { + "name": "cluster", + "children": [ + {"name": "AgglomerativeCluster", "size": 3938}, + {"name": "CommunityStructure", "size": 3812}, + {"name": "HierarchicalCluster", "size": 6714}, + {"name": "MergeEdge", "size": 743} + ] + } + ] + } + ] + } + + :reqheader Authorization: OAuth token to authenticate + :statuscode 200: no error + :statuscode 403: unauthenticated """ authority = service.get(authority_id) return dict( From 00c64ba52faae3de639426d41f4c6508a4563352 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 15:02:49 -0800 Subject: [PATCH 038/100] More doc style fixes --- lemur/authorities/views.py | 50 +++++++++++++++---------------- lemur/plugins/lemur_aws/plugin.py | 3 +- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/lemur/authorities/views.py b/lemur/authorities/views.py index 800c9975..16441719 100644 --- a/lemur/authorities/views.py +++ b/lemur/authorities/views.py @@ -132,31 +132,31 @@ class AuthoritiesList(AuthenticatedResource): Accept: application/json, text/javascript Content-Type: application/json;charset=UTF-8 - { - "country": "US", - "state": "California", - "location": "Los Gatos", - "organization": "Netflix", - "organizationalUnit": "Operations", - "type": "root", - "signingAlgorithm": "sha256WithRSA", - "sensitivity": "medium", - "keyType": "RSA2048", - "plugin": { - "slug": "cloudca-issuer" - }, - "name": "TimeTestAuthority5", - "owner": "secure@example.com", - "description": "test", - "commonName": "AcommonName", - "validityYears": "20", - "extensions": { - "subAltNames": { - "names": [] - }, - "custom": [] - } - } + { + "country": "US", + "state": "California", + "location": "Los Gatos", + "organization": "Netflix", + "organizationalUnit": "Operations", + "type": "root", + "signingAlgorithm": "sha256WithRSA", + "sensitivity": "medium", + "keyType": "RSA2048", + "plugin": { + "slug": "cloudca-issuer" + }, + "name": "TimeTestAuthority5", + "owner": "secure@example.com", + "description": "test", + "commonName": "AcommonName", + "validityYears": "20", + "extensions": { + "subAltNames": { + "names": [] + }, + "custom": [] + } + } **Example response**: diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index efcce4d0..61c64dab 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -450,7 +450,8 @@ class S3DestinationPlugin(ExportDestinationPlugin): def upload_acme_token(self, token_path, token, options, **kwargs): """ - This is called from the acme http challenge + This is called from the acme http challenge + :param self: :param token_path: :param token: From da9e949e89a86b5a1d82af1f62b2f207c0026eb6 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 15:08:36 -0800 Subject: [PATCH 039/100] Remove extra spaces --- lemur/destinations/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 7bae57f0..5e302c6d 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -21,7 +21,7 @@ def create(label, plugin_name, options, description=None): :param label: Destination common name :param description: - :rtype : Destination + :rtype: Destination :return: New destination """ # remove any sub-plugin objects before try to save the json options @@ -50,7 +50,7 @@ def update(destination_id, label, plugin_name, options, description): :param plugin_name: :param options: :param description: - :rtype : Destination + :rtype: Destination :return: """ destination = get(destination_id) @@ -81,7 +81,7 @@ def get(destination_id): Retrieves an destination by its lemur assigned ID. :param destination_id: Lemur assigned ID - :rtype : Destination + :rtype: Destination :return: """ return database.get(Destination, destination_id) From 360e4c61540d0202c044ea35160d7fc0c4d5f461 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 17 Feb 2021 15:10:15 -0800 Subject: [PATCH 040/100] Remove extra spaces --- lemur/notifications/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 5bc5f3e1..fd8ba20f 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -94,7 +94,7 @@ def create(label, plugin_name, options, description, certificates): :param options: :param description: :param certificates: - :rtype : Notification + :rtype: Notification :return: """ notification = Notification( @@ -115,7 +115,7 @@ def update(notification_id, label, plugin_name, options, description, active, ce :param description: :param active: :param certificates: - :rtype : Notification + :rtype: Notification :return: """ notification = get(notification_id) @@ -144,7 +144,7 @@ def get(notification_id): Retrieves an notification by its lemur assigned ID. :param notification_id: Lemur assigned ID - :rtype : Notification + :rtype: Notification :return: """ return database.get(Notification, notification_id) From 1918b911b3e798d0d48cf9902c0eb92b742c7b46 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 18 Feb 2021 14:28:15 -0800 Subject: [PATCH 041/100] Make code more parallel --- lemur/static/app/angular/notifications/services.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index e1a645db..2259d6b3 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -14,9 +14,9 @@ angular.module('lemur') this.certificates.push(certificate); this.addedCertificates.push(certificate); if (this.removedCertificates !== undefined) { - const index = this.removedCertificates.indexOf(certificate); - if (index > -1) { - this.removedCertificates.splice(index, 1); + const removedIndex = this.removedCertificates.indexOf(certificate); + if (removedIndex > -1) { + this.removedCertificates.splice(removedIndex, 1); } } }, @@ -26,8 +26,11 @@ angular.module('lemur') } const removedCert = this.certificates.splice(index, 1); this.removedCertificates.push(removedCert); - if (this.addedCertificates !== undefined && this.addedCertificates.indexOf(removedCert) > -1) { - this.addedCertificates.splice(this.addedCertificates.indexOf(removedCert), 1); + if (this.addedCertificates !== undefined) { + const addedIndex = this.addedCertificates.indexOf(removedCert); + if (addedIndex > -1) { + this.addedCertificates.splice(addedIndex, 1); + } } } }); From 85b053ed9877d68907e693b823d5fd70b65833d1 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 18 Feb 2021 14:35:51 -0800 Subject: [PATCH 042/100] Ignore submitted certificates --- lemur/notifications/schemas.py | 1 - lemur/notifications/service.py | 10 +++---- lemur/notifications/views.py | 48 +++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/lemur/notifications/schemas.py b/lemur/notifications/schemas.py index 6ef5c506..d69da14d 100644 --- a/lemur/notifications/schemas.py +++ b/lemur/notifications/schemas.py @@ -20,7 +20,6 @@ class NotificationInputSchema(LemurInputSchema): description = fields.String() active = fields.Boolean() plugin = fields.Nested(PluginInputSchema, required=True) - certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) added_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) removed_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 2e4566eb..372c1843 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -104,7 +104,7 @@ def create(label, plugin_name, options, description, certificates): return database.create(notification) -def update(notification_id, label, plugin_name, options, description, active, certificates, added_certificates, removed_certificates): +def update(notification_id, label, plugin_name, options, description, active, added_certificates, removed_certificates): """ Updates an existing notification. @@ -114,7 +114,6 @@ def update(notification_id, label, plugin_name, options, description, active, ce :param options: :param description: :param active: - :param certificates: :param added_certificates: :param removed_certificates: :rtype: Notification @@ -127,11 +126,8 @@ def update(notification_id, label, plugin_name, options, description, active, ce notification.options = options notification.description = description notification.active = active - if certificates: - notification.certificates = certificates - else: - notification.certificates = notification.certificates + added_certificates - notification.certificates = [c for c in notification.certificates if c not in removed_certificates] + notification.certificates = notification.certificates + added_certificates + notification.certificates = [c for c in notification.certificates if c not in removed_certificates] return database.update(notification) diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index fc7be4e7..b1200091 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -117,7 +117,7 @@ class NotificationsList(AuthenticatedResource): """ .. http:post:: /notifications - Creates a new account + Creates a new notification **Example request**: @@ -214,9 +214,12 @@ class NotificationsList(AuthenticatedResource): "id": 2 } - :arg accountNumber: aws account number - :arg label: human readable account label - :arg comments: some description about the account + :label label: notification name + :label slug: notification plugin slug + :label plugin_options: notification plugin options + :label description: notification description + :label active: whether or not the notification is active/enabled + :label certificates: certificates to attach to notification :reqheader Authorization: OAuth token to authenticate :statuscode 200: no error """ @@ -306,17 +309,29 @@ class Notifications(AuthenticatedResource): """ .. http:put:: /notifications/1 - Updates an account + Updates a notification **Example request**: .. sourcecode:: http - POST /notifications/1 HTTP/1.1 + PUT /notifications/1 HTTP/1.1 Host: example.com Accept: application/json, text/javascript Content-Type: application/json;charset=UTF-8 + { + "label": "labelChanged", + "plugin": { + "slug": "email-notification", + "plugin_options": "???" + }, + "description": "Sample notification", + "active": "true", + "added_certificates": "???", + "removed_certificates": "???" + } + **Example response**: @@ -328,14 +343,24 @@ class Notifications(AuthenticatedResource): { "id": 1, - "accountNumber": 11111111111, "label": "labelChanged", - "comments": "this is a thing" + "plugin": { + "slug": "email-notification", + "plugin_options": "???" + }, + "description": "Sample notification", + "active": "true", + "added_certificates": "???", + "removed_certificates": "???" } - :arg accountNumber: aws account number - :arg label: human readable account label - :arg comments: some description about the account + :label label: notification name + :label slug: notification plugin slug + :label plugin_options: notification plugin options + :label description: notification description + :label active: whether or not the notification is active/enabled + :label added_certificates: certificates to add + :label removed_certificates: certificates to remove :reqheader Authorization: OAuth token to authenticate :statuscode 200: no error """ @@ -346,7 +371,6 @@ class Notifications(AuthenticatedResource): data["plugin"]["plugin_options"], data["description"], data["active"], - data["certificates"], data["added_certificates"], data["removed_certificates"], ) From 0bc66be418f99b6f3c15a393d7ca80f48154c84b Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 18 Feb 2021 17:23:02 -0800 Subject: [PATCH 043/100] Fix docs and enable fail on warning --- .readthedocs.yml | 1 + CHANGELOG.rst | 2 +- docs/conf.py | 5 +- docs/developer/index.rst | 7 + docs/developer/internals/lemur.defaults.rst | 29 ++++ docs/developer/internals/lemur.deployment.rst | 20 +++ docs/developer/internals/lemur.endpoints.rst | 56 ++++++++ docs/developer/internals/lemur.logs.rst | 47 +++++++ .../internals/lemur.plugins.lemur_acme.rst | 83 ++++++++++++ .../internals/lemur.plugins.lemur_atlas.rst | 20 +++ .../lemur.plugins.lemur_cryptography.rst | 20 +++ .../lemur.plugins.lemur_digicert.rst | 20 +++ .../internals/lemur.plugins.lemur_jks.rst | 20 +++ .../lemur.plugins.lemur_kubernetes.rst | 20 +++ .../internals/lemur.plugins.lemur_openssl.rst | 20 +++ .../internals/lemur.plugins.lemur_slack.rst | 20 +++ docs/developer/internals/lemur.reporting.rst | 38 ++++++ docs/developer/internals/lemur.rst | 11 +- docs/developer/internals/lemur.sources.rst | 56 ++++++++ docs/developer/internals/lemur.tests.rst | 11 ++ docs/license/index.rst | 3 +- lemur/certificates/views.py | 3 +- lemur/dns_providers/service.py | 2 +- lemur/dns_providers/views.py | 125 ++++++++++-------- lemur/domains/views.py | 2 +- lemur/pending_certificates/service.py | 15 +-- lemur/pending_certificates/views.py | 4 +- lemur/plugins/bases/destination.py | 5 + lemur/plugins/bases/notification.py | 5 + lemur/plugins/bases/source.py | 5 + lemur/plugins/lemur_acme/acme_handlers.py | 4 +- lemur/roles/views.py | 4 +- lemur/sources/service.py | 6 +- .../notification/notification.js | 6 - .../app/angular/notifications/services.js | 2 +- lemur/users/views.py | 46 ++++++- requirements-docs.in | 8 ++ requirements-docs.txt | 36 ++++- 38 files changed, 683 insertions(+), 104 deletions(-) create mode 100644 docs/developer/internals/lemur.defaults.rst create mode 100644 docs/developer/internals/lemur.deployment.rst create mode 100644 docs/developer/internals/lemur.endpoints.rst create mode 100644 docs/developer/internals/lemur.logs.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_acme.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_atlas.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_cryptography.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_digicert.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_jks.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_kubernetes.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_openssl.rst create mode 100644 docs/developer/internals/lemur.plugins.lemur_slack.rst create mode 100644 docs/developer/internals/lemur.reporting.rst create mode 100644 docs/developer/internals/lemur.sources.rst create mode 100644 docs/developer/internals/lemur.tests.rst diff --git a/.readthedocs.yml b/.readthedocs.yml index 54eb8741..d41769a8 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,6 +8,7 @@ version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py + fail_on_warning: true # Build docs in all formats (html, pdf, epub) formats: all diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 22a9341f..24db16d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -247,7 +247,7 @@ these keys should be fairly trivial, additionally pull requests have been submit should be easier to determine what authorities are available and when an authority has actually been selected. * Closed `#254 `_ - Forces certificate names to be generally unique. If a certificate name (generated or otherwise) is found to be a duplicate we increment by appending a counter. -* Closed `#254 `_ - Switched to using Fernet generated passphrases for exported items. +* Closed `#275 `_ - Switched to using Fernet generated passphrases for exported items. These are more sounds that pseudo random passphrases generated before and have the nice property of being in base64. * Closed `#278 `_ - Added ability to specify a custom name to certificate creation, previously this was only available in the certificate import wizard. diff --git a/docs/conf.py b/docs/conf.py index 55bd20d2..077b66f4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,6 +32,9 @@ if on_rtd: MOCK_MODULES = ["ldap"] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) +autodoc_mock_imports = ["python-ldap", "acme", "certsrv", "dnspython3", "dyn", "factory-boy", "flask_replicated", + "josepy", "logmatic", "pem"] + # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -146,7 +149,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +# html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied diff --git a/docs/developer/index.rst b/docs/developer/index.rst index 8569dda5..1a08ffb5 100644 --- a/docs/developer/index.rst +++ b/docs/developer/index.rst @@ -43,6 +43,13 @@ Building Documentation Inside the ``docs`` directory, you can run ``make`` to build the documentation. See ``make help`` for available options and the `Sphinx Documentation `_ for more information. +Adding New Modules to Documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When a new module is added, it will need to be added to the documentation. +Ideally, we might rely on `sphinx-apidoc `_ to autogenerate our documentation. +Unfortunately, this causes some build problems. +Instead, you'll need to add new modules by hand. Developing Against HEAD ----------------------- diff --git a/docs/developer/internals/lemur.defaults.rst b/docs/developer/internals/lemur.defaults.rst new file mode 100644 index 00000000..0b1767ed --- /dev/null +++ b/docs/developer/internals/lemur.defaults.rst @@ -0,0 +1,29 @@ +defaults Package +================ + +:mod:`defaults` Module +---------------------------------------- + +.. automodule:: lemur.defaults + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +----------------------------- + +.. automodule:: lemur.defaults.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +--------------------------- + +.. automodule:: lemur.defaults.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.deployment.rst b/docs/developer/internals/lemur.deployment.rst new file mode 100644 index 00000000..047ec251 --- /dev/null +++ b/docs/developer/internals/lemur.deployment.rst @@ -0,0 +1,20 @@ +deployment Package +=================== + +:mod:`deployment` Module +---------------------------------------- + +.. automodule:: lemur.deployment + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.deployment.service + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.endpoints.rst b/docs/developer/internals/lemur.endpoints.rst new file mode 100644 index 00000000..14cb67d6 --- /dev/null +++ b/docs/developer/internals/lemur.endpoints.rst @@ -0,0 +1,56 @@ +endpoints Package +=================== + +:mod:`endpoints` Module +---------------------------------------- + +.. automodule:: lemur.endpoints + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cli` Module +-------------------------- + +.. automodule:: lemur.endpoints.cli + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`models` Module +----------------------------- + +.. automodule:: lemur.endpoints.models + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +------------------------------ + +.. automodule:: lemur.endpoints.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.endpoints.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +---------------------------- + +.. automodule:: lemur.endpoints.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.logs.rst b/docs/developer/internals/lemur.logs.rst new file mode 100644 index 00000000..4e01484a --- /dev/null +++ b/docs/developer/internals/lemur.logs.rst @@ -0,0 +1,47 @@ +logs Package +=================== + +:mod:`logs` Module +-------------------- + +.. automodule:: lemur.logs + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`models` Module +------------------------------ + +.. automodule:: lemur.logs.models + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +------------------------------ + +.. automodule:: lemur.logs.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.logs.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +------------------------------ + +.. automodule:: lemur.logs.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_acme.rst b/docs/developer/internals/lemur.plugins.lemur_acme.rst new file mode 100644 index 00000000..57f5209d --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_acme.rst @@ -0,0 +1,83 @@ +lemur_acme package +================================= + +:mod:`lemur_acme` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`acme_handlers` Module +----------------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.acme_handlers + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`challenge_types` Module +------------------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.challenge_types + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cloudflare` Module +------------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.cloudflare + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`dyn` Module +------------------------------------ + +.. automodule:: lemur.plugins.lemur_acme.dyn + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +--------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`powerdns` Module +----------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.powerdns + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`route53` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.route53 + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`ultradns` Module +----------------------------------------- + +.. automodule:: lemur.plugins.lemur_acme.ultradns + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_atlas.rst b/docs/developer/internals/lemur.plugins.lemur_atlas.rst new file mode 100644 index 00000000..01eaaa79 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_atlas.rst @@ -0,0 +1,20 @@ +lemur_atlas package +================================== + +:mod:`lemur_atlas` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_atlas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_atlas.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_cryptography.rst b/docs/developer/internals/lemur.plugins.lemur_cryptography.rst new file mode 100644 index 00000000..ee9f6c93 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_cryptography.rst @@ -0,0 +1,20 @@ +lemur_cryptography package +================================== + +:mod:`lemur_cryptography` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_cryptography + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_cryptography.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_digicert.rst b/docs/developer/internals/lemur.plugins.lemur_digicert.rst new file mode 100644 index 00000000..232d658b --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_digicert.rst @@ -0,0 +1,20 @@ +lemur_digicert package +================================== + +:mod:`lemur_digicert` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_digicert + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_digicert.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_jks.rst b/docs/developer/internals/lemur.plugins.lemur_jks.rst new file mode 100644 index 00000000..a47f653f --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_jks.rst @@ -0,0 +1,20 @@ +lemur_jks package +================================== + +:mod:`lemur_jks` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_jks + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_jks.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_kubernetes.rst b/docs/developer/internals/lemur.plugins.lemur_kubernetes.rst new file mode 100644 index 00000000..7173befb --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_kubernetes.rst @@ -0,0 +1,20 @@ +lemur_kubernetes package +================================== + +:mod:`lemur_kubernetes` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_kubernetes + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_kubernetes.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_openssl.rst b/docs/developer/internals/lemur.plugins.lemur_openssl.rst new file mode 100644 index 00000000..b94b56b4 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_openssl.rst @@ -0,0 +1,20 @@ +lemur_openssl package +================================== + +:mod:`lemur_openssl` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_openssl + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_openssl.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.plugins.lemur_slack.rst b/docs/developer/internals/lemur.plugins.lemur_slack.rst new file mode 100644 index 00000000..371a8880 --- /dev/null +++ b/docs/developer/internals/lemur.plugins.lemur_slack.rst @@ -0,0 +1,20 @@ +lemur_slack package +================================== + +:mod:`lemur_slack` Module +---------------------------------------- + +.. automodule:: lemur.plugins.lemur_slack + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`plugin` Module +-------------------- + +.. automodule:: lemur.plugins.lemur_slack.plugin + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.reporting.rst b/docs/developer/internals/lemur.reporting.rst new file mode 100644 index 00000000..9056e8b6 --- /dev/null +++ b/docs/developer/internals/lemur.reporting.rst @@ -0,0 +1,38 @@ +reporting Package +=================== + +:mod:`reporting` Module +---------------------------------------- + +.. automodule:: lemur.reporting + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cli` Module +------------------------------ + +.. automodule:: lemur.reporting.cli + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.reporting.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +------------------------------ + +.. automodule:: lemur.reporting.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.rst b/docs/developer/internals/lemur.rst index b6517e4b..86b4d000 100644 --- a/docs/developer/internals/lemur.rst +++ b/docs/developer/internals/lemur.rst @@ -28,15 +28,6 @@ lemur Package :undoc-members: :show-inheritance: -:mod:`decorators` Module ------------------------- - -.. automodule:: lemur.decorators - :noindex: - :members: - :undoc-members: - :show-inheritance: - :mod:`exceptions` Module ------------------------ @@ -108,7 +99,7 @@ Subpackages lemur.plugins.lemur_atlas lemur.plugins.lemur_cryptography lemur.plugins.lemur_digicert - lemur.plugins.lemur_java + lemur.plugins.lemur_jks lemur.plugins.lemur_kubernetes lemur.plugins.lemur_openssl lemur.plugins.lemur_slack diff --git a/docs/developer/internals/lemur.sources.rst b/docs/developer/internals/lemur.sources.rst new file mode 100644 index 00000000..6a5c0c42 --- /dev/null +++ b/docs/developer/internals/lemur.sources.rst @@ -0,0 +1,56 @@ +sources Package +=================== + +:mod:`sources` Module +---------------------- + +.. automodule:: lemur.sources + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`cli` Module +------------------------------ + +.. automodule:: lemur.sources.cli + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`models` Module +------------------------------ + +.. automodule:: lemur.sources.models + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`schemas` Module +------------------------------ + +.. automodule:: lemur.sources.schemas + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`service` Module +------------------------------ + +.. automodule:: lemur.sources.service + :noindex: + :members: + :undoc-members: + :show-inheritance: + +:mod:`views` Module +------------------------------ + +.. automodule:: lemur.sources.views + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/developer/internals/lemur.tests.rst b/docs/developer/internals/lemur.tests.rst new file mode 100644 index 00000000..0c385694 --- /dev/null +++ b/docs/developer/internals/lemur.tests.rst @@ -0,0 +1,11 @@ +tests Package +============= + +:mod:`tests` Module +-------------------- + +.. automodule:: lemur.tests + :noindex: + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/license/index.rst b/docs/license/index.rst index 4df00576..3afc9bc8 100644 --- a/docs/license/index.rst +++ b/docs/license/index.rst @@ -17,4 +17,5 @@ A list of additional contributors can be seen on `GitHub = 19 # pyjks < 19 depends on pycryptodome, which conflicts with dyn's usage of pycrypto pyjwt pyOpenSSL raven[flask] +redis retrying SQLAlchemy-Utils tabulate +vine xmltodict +# Test requirements are needed to allow test docs to build +-r requirements-tests.txt # docs specific sphinx diff --git a/requirements-docs.txt b/requirements-docs.txt index 41d5133b..92962269 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,6 +18,8 @@ botocore==1.20.7 # manual debug certifi==2020.12.5 # via requests +certsrv==2.1.1 + # manual debug chardet==3.0.4 # via requests cloudflare==2.8.15 @@ -32,10 +34,10 @@ docutils==0.15.2 # via sphinx dyn==1.8.1 # manual debug -idna==2.9 - # via requests -imagesize==1.2.0 - # via sphinx +factory-boy==3.2.0 + # manual debug +fakeredis==1.4.5 + # manual debug flask==1.1.2 # manual debug flask-bcrypt==0.7.1 @@ -56,8 +58,16 @@ flask-script==2.0.6 # manual debug flask-sqlalchemy==2.4.4 # manual debug +freezegun==1.1.0 + # manual debug gunicorn==20.0.4 # manual debug +hvac==0.10.8 + # manual debug +idna==2.9 + # via requests +imagesize==1.2.0 + # via sphinx inflection==0.5.1 # manual debug jinja2==2.11.3 @@ -72,32 +82,46 @@ marshmallow-sqlalchemy==0.23.1 # manual debug marshmallow==2.20.4 # manual debug +moto==1.3.16 + # manual debug packaging==20.3 # via sphinx +paramiko==2.7.2 + # manual debug pem==21.1.0 # manual debug pygments==2.6.1 # via sphinx +pyjks==20.0.0 + # manual debug pyjwt==2.0.1 # manual debug pyopenssl==20.0.1 # manual debug pyparsing==2.4.7 # via packaging +pytest==6.2.2 + # manual debug pytz==2019.3 # via babel raven[flask]==6.10.0 # manual debug +redis==3.5.3 + # manual debug retrying==1.3.3 # manual debug requests==2.25.1 # via sphinx +s3transfer==0.3.3 + # manual debug six==1.15.0 # via # packaging # sphinxcontrib-httpdomain snowballstemmer==2.0.0 # via sphinx +soupsieve==2.0.1 + # manual debug sphinx-rtd-theme==0.5.1 # via -r requirements-docs.in sphinx==3.5.0 @@ -119,12 +143,16 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx +sqlalchemy==1.3.16 + # manual debug sqlalchemy-utils==0.36.8 # manual debug tabulate==0.8.7 # manual debug urllib3==1.25.8 # via requests +vine==1.3.0 + # manual debug xmltodict==0.12.0 # manual debug From bc260fabb18f698e0a729e8fb26ba81a9a4cfac1 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 18 Feb 2021 17:45:59 -0800 Subject: [PATCH 044/100] Fix style --- lemur/plugins/lemur_acme/acme_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index 3aab83eb..375fd1d6 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -71,7 +71,7 @@ class AcmeHandler(object): return False def strip_wildcard(self, host): - """Removes the leading \\*. and returns Host and whether it was removed or not (True/False)""" + """Removes the leading wildcard and returns Host and whether it was removed or not (True/False)""" prefix = "*." if host.startswith(prefix): return host[len(prefix):], True From 931dd26585000b8634806c4f4a7517a563e167ab Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 18 Feb 2021 17:55:41 -0800 Subject: [PATCH 045/100] Fix style --- lemur/plugins/lemur_acme/acme_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index 375fd1d6..4de6c3f0 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -129,7 +129,7 @@ class AcmeHandler(object): if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() From b3d0b7ce1b5d192ab61930d2c7aa157f2a612a38 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 19 Feb 2021 18:25:05 -0800 Subject: [PATCH 046/100] Fix issue with repeatedly adding and removing --- lemur/notifications/schemas.py | 1 + lemur/notifications/views.py | 2 +- .../app/angular/notifications/services.js | 27 ++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/lemur/notifications/schemas.py b/lemur/notifications/schemas.py index d69da14d..6ef5c506 100644 --- a/lemur/notifications/schemas.py +++ b/lemur/notifications/schemas.py @@ -20,6 +20,7 @@ class NotificationInputSchema(LemurInputSchema): description = fields.String() active = fields.Boolean() plugin = fields.Nested(PluginInputSchema, required=True) + certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) added_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) removed_certificates = fields.Nested(AssociatedCertificateSchema, many=True, missing=[]) diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index b1200091..8ac8e06f 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -242,7 +242,7 @@ class Notifications(AuthenticatedResource): """ .. http:get:: /notifications/1 - Get a specific account + Get a specific notification **Example request**: diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index 2259d6b3..6bb36e65 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -11,26 +11,31 @@ angular.module('lemur') if (this.addedCertificates === undefined) { this.addedCertificates = []; } + if (_.some(this.addedCertificates, function (cert) { + return cert.id === certificate.id; + })) { + return; + } this.certificates.push(certificate); this.addedCertificates.push(certificate); if (this.removedCertificates !== undefined) { - const removedIndex = this.removedCertificates.indexOf(certificate); - if (removedIndex > -1) { - this.removedCertificates.splice(removedIndex, 1); - } + const indexInRemovedList = _.findIndex(this.removedCertificates, function (cert) { + return cert.id === certificate.id; + }); + this.removedCertificates.splice(indexInRemovedList, 1); } }, removeCertificate: function (index) { if (this.removedCertificates === undefined) { this.removedCertificates = []; } - const removedCert = this.certificates.splice(index, 1); + const removedCert = this.certificates.splice(index, 1)[0]; this.removedCertificates.push(removedCert); if (this.addedCertificates !== undefined) { - const addedIndex = this.addedCertificates.indexOf(removedCert); - if (addedIndex > -1) { - this.addedCertificates.splice(addedIndex, 1); - } + const indexInAddedList = _.findIndex(this.addedCertificates, function (cert) { + return cert.id === removedCert.id; + }); + this.addedCertificates.splice(indexInAddedList, 1); } } }); @@ -72,7 +77,9 @@ angular.module('lemur') }; NotificationService.update = function (notification) { - this.certificates = []; + // this.certificates = []; + // this.removedCertificates = []; + // this.addedCertificates = []; return notification.put(); }; From 811ac1a970e8d09932ee6e3ccb3800bc36664506 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 19 Feb 2021 18:27:29 -0800 Subject: [PATCH 047/100] Remove stray comments --- lemur/static/app/angular/notifications/services.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lemur/static/app/angular/notifications/services.js b/lemur/static/app/angular/notifications/services.js index 6bb36e65..9c484277 100644 --- a/lemur/static/app/angular/notifications/services.js +++ b/lemur/static/app/angular/notifications/services.js @@ -77,9 +77,6 @@ angular.module('lemur') }; NotificationService.update = function (notification) { - // this.certificates = []; - // this.removedCertificates = []; - // this.addedCertificates = []; return notification.put(); }; From d22bec1ec9e16f11c540dcdeb044d15603c66a11 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 23 Feb 2021 01:21:27 +0000 Subject: [PATCH 048/100] Bump flask-migrate from 2.6.0 to 2.7.0 Bumps [flask-migrate](https://github.com/miguelgrinberg/flask-migrate) from 2.6.0 to 2.7.0. - [Release notes](https://github.com/miguelgrinberg/flask-migrate/releases) - [Changelog](https://github.com/miguelgrinberg/Flask-Migrate/blob/master/CHANGES.md) - [Commits](https://github.com/miguelgrinberg/flask-migrate/compare/v2.6.0...v2.7.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 533 +++++++++++++++++++++++++++++++++++++----- requirements.txt | 2 +- 2 files changed, 473 insertions(+), 62 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 92962269..d989ebe6 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -5,123 +5,478 @@ # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # acme==1.12.0 - # manual debug + # via -r requirements-docs.in alabaster==0.7.12 # via sphinx +alembic==1.5.5 + # via flask-migrate +aniso8601==9.0.0 + # via flask-restful +appdirs==1.4.3 + # via + # -r requirements-tests.txt + # black arrow==0.17.0 - # manual debug + # via -r requirements-docs.in +attrs==19.3.0 + # via + # -r requirements-tests.txt + # jsonschema + # pytest +aws-sam-translator==1.22.0 + # via + # -r requirements-tests.txt + # cfn-lint +aws-xray-sdk==2.5.0 + # via + # -r requirements-tests.txt + # moto babel==2.8.0 # via sphinx +bandit==1.7.0 + # via -r requirements-tests.txt +bcrypt==3.2.0 + # via + # flask-bcrypt + # paramiko +beautifulsoup4==4.9.3 + # via cloudflare +black==20.8b1 + # via -r requirements-tests.txt +blinker==1.4 + # via + # flask-mail + # flask-principal + # raven boto3==1.17.7 - # manual debug + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # aws-sam-translator + # moto +boto==2.49.0 + # via + # -r requirements-tests.txt + # moto botocore==1.20.7 - # manual debug + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # aws-xray-sdk + # boto3 + # moto + # s3transfer certifi==2020.12.5 - # via requests + # via + # -r requirements-tests.txt + # requests certsrv==2.1.1 - # manual debug + # via -r requirements-docs.in +cffi==1.14.0 + # via + # -r requirements-tests.txt + # bcrypt + # cryptography + # pynacl +cfn-lint==0.29.5 + # via + # -r requirements-tests.txt + # moto chardet==3.0.4 - # via requests + # via + # -r requirements-tests.txt + # requests +click==7.1.2 + # via + # -r requirements-tests.txt + # black + # flask cloudflare==2.8.15 - # manual debug + # via -r requirements-docs.in +coverage==5.4 + # via -r requirements-tests.txt cryptography==3.4.5 - # manual debug + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # acme + # josepy + # moto + # paramiko + # pyopenssl + # python-jose + # requests + # sshpubkeys +decorator==4.4.2 + # via + # -r requirements-tests.txt + # networkx dnspython3==1.15.0 - # manual debug + # via -r requirements-docs.in dnspython==1.15.0 - # manual debug + # via dnspython3 +docker==4.2.0 + # via + # -r requirements-tests.txt + # moto docutils==0.15.2 # via sphinx dyn==1.8.1 - # manual debug + # via -r requirements-docs.in +ecdsa==0.14.1 + # via + # -r requirements-tests.txt + # moto + # python-jose + # sshpubkeys factory-boy==3.2.0 - # manual debug + # via -r requirements-tests.txt +faker==6.1.1 + # via + # -r requirements-tests.txt + # factory-boy fakeredis==1.4.5 - # manual debug -flask==1.1.2 - # manual debug + # via -r requirements-tests.txt flask-bcrypt==0.7.1 - # manual debug + # via -r requirements-docs.in flask-cors==3.0.10 - # manual debug + # via -r requirements-docs.in flask-mail==0.9.1 - # manual debug -flask-migrate==2.6.0 - # manual debug + # via -r requirements-docs.in +flask-migrate==2.7.0 + # via -r requirements-docs.in flask-principal==0.4.0 - # manual debug + # via -r requirements-docs.in flask-replicated==1.4 - # manual debug + # via -r requirements-docs.in flask-restful==0.3.8 - # manual debug + # via -r requirements-docs.in flask-script==2.0.6 - # manual debug + # via -r requirements-docs.in flask-sqlalchemy==2.4.4 - # manual debug + # via + # -r requirements-docs.in + # flask-migrate +flask==1.1.2 + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # flask-bcrypt + # flask-cors + # flask-mail + # flask-migrate + # flask-principal + # flask-restful + # flask-script + # flask-sqlalchemy + # pytest-flask + # raven freezegun==1.1.0 - # manual debug + # via -r requirements-tests.txt +future==0.18.2 + # via + # -r requirements-tests.txt + # aws-xray-sdk +gitdb==4.0.4 + # via + # -r requirements-tests.txt + # gitpython +gitpython==3.1.1 + # via + # -r requirements-tests.txt + # bandit gunicorn==20.0.4 - # manual debug + # via -r requirements-docs.in hvac==0.10.8 - # manual debug + # via -r requirements-docs.in idna==2.9 - # via requests + # via + # -r requirements-tests.txt + # moto + # requests imagesize==1.2.0 # via sphinx +importlib-metadata==1.6.0 + # via + # -r requirements-tests.txt + # jsonpickle inflection==0.5.1 - # manual debug + # via -r requirements-docs.in +iniconfig==1.0.1 + # via + # -r requirements-tests.txt + # pytest +itsdangerous==1.1.0 + # via + # -r requirements-tests.txt + # flask +javaobj-py3==0.4.2 + # via pyjks jinja2==2.11.3 - # via sphinx + # via + # -r requirements-tests.txt + # flask + # moto + # sphinx +jmespath==0.9.5 + # via + # -r requirements-tests.txt + # boto3 + # botocore josepy==1.3.0 - # manual debug + # via + # -r requirements-docs.in + # acme +jsondiff==1.1.2 + # via + # -r requirements-tests.txt + # moto +jsonlines==2.0.0 + # via cloudflare +jsonpatch==1.25 + # via + # -r requirements-tests.txt + # cfn-lint +jsonpickle==1.4 + # via + # -r requirements-tests.txt + # aws-xray-sdk +jsonpointer==2.0 + # via + # -r requirements-tests.txt + # jsonpatch +jsonschema==3.2.0 + # via + # -r requirements-tests.txt + # aws-sam-translator + # cfn-lint logmatic-python==0.1.7 - # manual debug + # via -r requirements-docs.in +mako==1.1.4 + # via alembic markupsafe==1.1.1 - # via jinja2 + # via + # -r requirements-tests.txt + # jinja2 + # mako + # moto marshmallow-sqlalchemy==0.23.1 - # manual debug + # via -r requirements-docs.in marshmallow==2.20.4 - # manual debug + # via + # -r requirements-docs.in + # marshmallow-sqlalchemy +mock==4.0.2 + # via + # -r requirements-tests.txt + # moto +more-itertools==8.2.0 + # via + # -r requirements-tests.txt + # moto moto==1.3.16 - # manual debug + # via -r requirements-tests.txt +mypy-extensions==0.4.3 + # via + # -r requirements-tests.txt + # black +networkx==2.4 + # via + # -r requirements-tests.txt + # cfn-lint +nose==1.3.7 + # via -r requirements-tests.txt packaging==20.3 - # via sphinx + # via + # -r requirements-tests.txt + # pytest + # sphinx paramiko==2.7.2 - # manual debug + # via -r requirements-docs.in +pathspec==0.8.0 + # via + # -r requirements-tests.txt + # black +pbr==5.4.5 + # via + # -r requirements-tests.txt + # stevedore pem==21.1.0 - # manual debug + # via -r requirements-docs.in +pluggy==0.13.1 + # via + # -r requirements-tests.txt + # pytest +py==1.9.0 + # via + # -r requirements-tests.txt + # pytest +pyasn1-modules==0.2.8 + # via pyjks +pyasn1==0.4.8 + # via + # -r requirements-tests.txt + # pyasn1-modules + # pyjks + # python-jose + # rsa +pycparser==2.20 + # via + # -r requirements-tests.txt + # cffi +pycryptodomex==3.10.1 + # via pyjks +pyflakes==2.2.0 + # via -r requirements-tests.txt pygments==2.6.1 # via sphinx pyjks==20.0.0 - # manual debug + # via -r requirements-docs.in pyjwt==2.0.1 - # manual debug + # via -r requirements-docs.in +pynacl==1.4.0 + # via paramiko pyopenssl==20.0.1 - # manual debug + # via + # -r requirements-docs.in + # acme + # josepy + # requests pyparsing==2.4.7 - # via packaging + # via + # -r requirements-tests.txt + # packaging +pyrfc3339==1.1 + # via acme +pyrsistent==0.16.0 + # via + # -r requirements-tests.txt + # jsonschema +pytest-flask==1.1.0 + # via -r requirements-tests.txt +pytest-mock==3.5.1 + # via -r requirements-tests.txt pytest==6.2.2 - # manual debug + # via + # -r requirements-tests.txt + # pytest-flask + # pytest-mock +python-dateutil==2.8.1 + # via + # -r requirements-tests.txt + # alembic + # arrow + # botocore + # faker + # freezegun + # moto +python-editor==1.0.4 + # via alembic +python-jose[cryptography]==3.1.0 + # via + # -r requirements-tests.txt + # moto +python-json-logger==2.0.1 + # via logmatic-python pytz==2019.3 - # via babel + # via + # -r requirements-tests.txt + # acme + # babel + # flask-restful + # moto + # pyrfc3339 +pyyaml==5.4.1 + # via + # -r requirements-tests.txt + # bandit + # cfn-lint + # cloudflare + # moto raven[flask]==6.10.0 - # manual debug + # via -r requirements-docs.in redis==3.5.3 - # manual debug + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # fakeredis +regex==2020.4.4 + # via + # -r requirements-tests.txt + # black +requests-mock==1.8.0 + # via -r requirements-tests.txt +requests-toolbelt==0.9.1 + # via acme +requests[security]==2.25.1 + # via + # -r requirements-tests.txt + # acme + # certsrv + # cloudflare + # docker + # hvac + # moto + # requests-mock + # requests-toolbelt + # responses + # sphinx +responses==0.10.12 + # via + # -r requirements-tests.txt + # moto retrying==1.3.3 - # manual debug -requests==2.25.1 - # via sphinx + # via -r requirements-docs.in +rsa==4.0 + # via + # -r requirements-tests.txt + # python-jose s3transfer==0.3.3 - # manual debug + # via + # -r requirements-tests.txt + # boto3 six==1.15.0 # via + # -r requirements-tests.txt + # acme + # aws-sam-translator + # bandit + # bcrypt + # cfn-lint + # docker + # ecdsa + # fakeredis + # flask-cors + # flask-restful + # hvac + # josepy + # jsonschema + # moto # packaging + # pynacl + # pyopenssl + # pyrsistent + # python-dateutil + # python-jose + # requests-mock + # responses + # retrying # sphinxcontrib-httpdomain + # sqlalchemy-utils + # stevedore + # websocket-client +smmap==3.0.2 + # via + # -r requirements-tests.txt + # gitdb snowballstemmer==2.0.0 # via sphinx +sortedcontainers==2.1.0 + # via + # -r requirements-tests.txt + # fakeredis soupsieve==2.0.1 - # manual debug + # via beautifulsoup4 sphinx-rtd-theme==0.5.1 # via -r requirements-docs.in sphinx==3.5.0 @@ -143,18 +498,74 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sqlalchemy==1.3.16 - # manual debug sqlalchemy-utils==0.36.8 - # manual debug + # via -r requirements-docs.in +sqlalchemy==1.3.16 + # via + # alembic + # flask-sqlalchemy + # marshmallow-sqlalchemy + # sqlalchemy-utils +sshpubkeys==3.1.0 + # via + # -r requirements-tests.txt + # moto +stevedore==1.32.0 + # via + # -r requirements-tests.txt + # bandit tabulate==0.8.7 - # manual debug + # via -r requirements-docs.in +text-unidecode==1.3 + # via + # -r requirements-tests.txt + # faker +toml==0.10.1 + # via + # -r requirements-tests.txt + # black + # pytest +twofish==0.3.0 + # via pyjks +typed-ast==1.4.1 + # via + # -r requirements-tests.txt + # black +typing-extensions==3.7.4.3 + # via + # -r requirements-tests.txt + # black urllib3==1.25.8 - # via requests + # via + # -r requirements-tests.txt + # botocore + # requests vine==1.3.0 - # manual debug + # via -r requirements-docs.in +websocket-client==0.57.0 + # via + # -r requirements-tests.txt + # docker +werkzeug==1.0.1 + # via + # -r requirements-tests.txt + # flask + # moto + # pytest-flask +wrapt==1.12.1 + # via + # -r requirements-tests.txt + # aws-xray-sdk xmltodict==0.12.0 - # manual debug + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # moto +zipp==3.1.0 + # via + # -r requirements-tests.txt + # importlib-metadata + # moto # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.txt b/requirements.txt index c6a21ef7..103194c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,7 +77,7 @@ flask-cors==3.0.10 # via -r requirements.in flask-mail==0.9.1 # via -r requirements.in -flask-migrate==2.6.0 +flask-migrate==2.7.0 # via -r requirements.in flask-principal==0.4.0 # via -r requirements.in From d278c6e13204500c328f2b52ebce016ffab97e9d Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 24 Feb 2021 11:43:35 -0800 Subject: [PATCH 049/100] Include notification interval (converted to days) in SNS notifications --- lemur/notifications/messaging.py | 1 + lemur/plugins/lemur_aws/plugin.py | 2 +- lemur/plugins/lemur_aws/sns.py | 32 ++++++++++++++++---- lemur/plugins/lemur_aws/tests/test_sns.py | 36 +++++++++++++++++++---- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 4b33656d..2e2f37ad 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -201,6 +201,7 @@ def send_plugin_notification(event_type, data, recipients, notification): "notification_plugin": notification.plugin.slug, "certificate_targets": recipients, "plugin": notification.plugin.slug, + "notification_id": notification.id, } status = FAILURE_METRIC_STATUS try: diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 61c64dab..bf7d89bb 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -564,4 +564,4 @@ class SNSNotificationPlugin(ExpirationNotificationPlugin): f"{self.get_option('topicName', options)}" current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}") - sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) + sns.publish(topic_arn, message, notification_type, options, region_name=self.get_option("region", options)) diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index fab45b82..cde9ed9c 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -12,20 +12,21 @@ import boto3 from flask import current_app -def publish(topic_arn, certificates, notification_type, **kwargs): +def publish(topic_arn, certificates, notification_type, options, **kwargs): sns_client = boto3.client("sns", **kwargs) message_ids = {} subject = "Lemur: {0} Notification".format(notification_type.capitalize()) for certificate in certificates: - message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, subject) + message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type, + subject, options) return message_ids -def publish_single(sns_client, topic_arn, certificate, notification_type, subject): +def publish_single(sns_client, topic_arn, certificate, notification_type, subject, options): response = sns_client.publish( TopicArn=topic_arn, - Message=format_message(certificate, notification_type), + Message=format_message(certificate, notification_type, options), Subject=subject, ) @@ -46,7 +47,7 @@ def create_certificate_url(name): ) -def format_message(certificate, notification_type): +def format_message(certificate, notification_type, options): json_message = { "notification_type": notification_type, "certificate_name": certificate["name"], @@ -57,4 +58,25 @@ def format_message(certificate, notification_type): "owner": certificate["owner"], "details": create_certificate_url(certificate["name"]) } + if notification_type == "expiration": + json_message["notification_interval_days"] = calculate_expiration_days(options) return json.dumps(json_message) + + +def calculate_expiration_days(options): + unit = get_option(options, "unit") + interval = get_option(options, "interval") + if unit == "weeks": + return interval * 7 + + elif unit == "months": + return interval * 30 + + elif unit == "days": + return interval + + +def get_option(options, option_name): + for o in options: + if o.get("name") == option_name: + return o.get("value", o.get("default")) diff --git a/lemur/plugins/lemur_aws/tests/test_sns.py b/lemur/plugins/lemur_aws/tests/test_sns.py index c8688194..66ad3e96 100644 --- a/lemur/plugins/lemur_aws/tests/test_sns.py +++ b/lemur/plugins/lemur_aws/tests/test_sns.py @@ -13,9 +13,31 @@ from lemur.tests.test_messaging import verify_sender_email @mock_sns() -def test_format(certificate, endpoint): +def test_format_nonexpiration(certificate, endpoint): data = [certificate_notification_output_schema.dump(certificate).data] + for certificate in data: + expected_message = { + "notification_type": "not-expiration", + "certificate_name": certificate["name"], + "expires": arrow.get(certificate["validityEnd"]).format("YYYY-MM-DDTHH:mm:ss"), + "issuer": certificate["issuer"], + "id": certificate["id"], + "endpoints_detected": 0, + "owner": certificate["owner"], + "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) + } + # We don't currently support any SNS notifications besides expiration; + # when we do, this test will probably need to be refactored. + # For now, this is a placeholder proving empty options works as long as it's not "expiration" type + assert expected_message == json.loads(format_message(certificate, "not-expiration", None)) + + +@mock_sns() +def test_format_expiration(certificate, endpoint): + data = [certificate_notification_output_schema.dump(certificate).data] + options = get_options() + for certificate in data: expected_message = { "notification_type": "expiration", @@ -25,9 +47,10 @@ def test_format(certificate, endpoint): "id": certificate["id"], "endpoints_detected": 0, "owner": certificate["owner"], - "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) + "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]), + "notification_interval_days": 10 # 10 days specified in options } - assert expected_message == json.loads(format_message(certificate, "expiration")) + assert expected_message == json.loads(format_message(certificate, "expiration", options)) @mock_sns() @@ -52,7 +75,7 @@ def test_publish(certificate, endpoint): topic_arn, sqs_client, queue_url = create_and_subscribe_to_topic() - message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1") + message_ids = publish(topic_arn, data, "expiration", get_options(), region_name="us-east-1") assert len(message_ids) == len(data) received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"] @@ -61,7 +84,7 @@ def test_publish(certificate, endpoint): actual_message = next( (m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None) actual_json = json.loads(actual_message["Body"]) - assert actual_json["Message"] == format_message(certificate, "expiration") + assert actual_json["Message"] == format_message(certificate, "expiration", get_options()) assert actual_json["Subject"] == "Lemur: Expiration Notification" @@ -98,7 +121,8 @@ def test_send_expiration_notification(): received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"] assert len(received_messages) == 1 - expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration") + expected_message = format_message(certificate_notification_output_schema.dump(certificate).data, "expiration", + notification.options) actual_message = json.loads(received_messages[0]["Body"])["Message"] assert actual_message == expected_message From 4c13d1a5a2d4b47db670c3e019b404935ab6ec14 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 24 Feb 2021 21:29:50 +0000 Subject: [PATCH 050/100] Bump sphinx from 3.5.0 to 3.5.1 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.5.0 to 3.5.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.5.0...v3.5.1) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d989ebe6..68bc0dd3 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -479,7 +479,7 @@ soupsieve==2.0.1 # via beautifulsoup4 sphinx-rtd-theme==0.5.1 # via -r requirements-docs.in -sphinx==3.5.0 +sphinx==3.5.1 # via # -r requirements-docs.in # sphinx-rtd-theme From 1d6023e6d8563851370c2446a08682ac21c70e05 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 24 Feb 2021 13:42:43 -0800 Subject: [PATCH 051/100] Reuse get_option --- lemur/plugins/lemur_aws/sns.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index cde9ed9c..f44f8702 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -11,6 +11,8 @@ import arrow import boto3 from flask import current_app +from lemur.plugins.lemur_aws.plugin import SNSNotificationPlugin + def publish(topic_arn, certificates, notification_type, options, **kwargs): sns_client = boto3.client("sns", **kwargs) @@ -64,8 +66,8 @@ def format_message(certificate, notification_type, options): def calculate_expiration_days(options): - unit = get_option(options, "unit") - interval = get_option(options, "interval") + unit = SNSNotificationPlugin.get_option("unit", options) + interval = SNSNotificationPlugin.get_option("interval", options) if unit == "weeks": return interval * 7 @@ -74,9 +76,3 @@ def calculate_expiration_days(options): elif unit == "days": return interval - - -def get_option(options, option_name): - for o in options: - if o.get("name") == option_name: - return o.get("value", o.get("default")) From 6579630ae7d936e647484162c8fc47761525c979 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 24 Feb 2021 21:55:58 +0000 Subject: [PATCH 052/100] Bump tabulate from 0.8.7 to 0.8.9 Bumps [tabulate](https://github.com/astanin/python-tabulate) from 0.8.7 to 0.8.9. - [Release notes](https://github.com/astanin/python-tabulate/releases) - [Changelog](https://github.com/astanin/python-tabulate/blob/master/CHANGELOG) - [Commits](https://github.com/astanin/python-tabulate/compare/v0.8.7...v0.8.9) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 68bc0dd3..1da81d2e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -514,7 +514,7 @@ stevedore==1.32.0 # via # -r requirements-tests.txt # bandit -tabulate==0.8.7 +tabulate==0.8.9 # via -r requirements-docs.in text-unidecode==1.3 # via diff --git a/requirements.txt b/requirements.txt index 103194c3..23488a95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -250,7 +250,7 @@ sqlalchemy==1.3.16 # flask-sqlalchemy # marshmallow-sqlalchemy # sqlalchemy-utils -tabulate==0.8.7 +tabulate==0.8.9 # via -r requirements.in twofish==0.3.0 # via pyjks From 6d9dca510e8b2d348390902fba229dfa3d536815 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 24 Feb 2021 22:36:07 +0000 Subject: [PATCH 053/100] Bump josepy from 1.3.0 to 1.7.0 Bumps [josepy](https://github.com/certbot/josepy) from 1.3.0 to 1.7.0. - [Release notes](https://github.com/certbot/josepy/releases) - [Changelog](https://github.com/certbot/josepy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/certbot/josepy/compare/v1.3.0...v1.7.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 1da81d2e..af5ab5ce 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -223,7 +223,7 @@ jmespath==0.9.5 # -r requirements-tests.txt # boto3 # botocore -josepy==1.3.0 +josepy==1.7.0 # via # -r requirements-docs.in # acme diff --git a/requirements.txt b/requirements.txt index 23488a95..35603bef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -125,7 +125,7 @@ jmespath==0.9.5 # via # boto3 # botocore -josepy==1.3.0 +josepy==1.7.0 # via acme jsonlines==1.2.0 # via cloudflare From 86dcac26a2edfc8dc22a5c0e472e10bfad5a4e05 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 24 Feb 2021 23:05:15 +0000 Subject: [PATCH 054/100] Bump faker from 6.1.1 to 6.5.0 Bumps [faker](https://github.com/joke2k/faker) from 6.1.1 to 6.5.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/joke2k/faker/compare/v6.1.1...v6.5.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index af5ab5ce..ee3bec8a 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -131,7 +131,7 @@ ecdsa==0.14.1 # sshpubkeys factory-boy==3.2.0 # via -r requirements-tests.txt -faker==6.1.1 +faker==6.5.0 # via # -r requirements-tests.txt # factory-boy diff --git a/requirements-tests.txt b/requirements-tests.txt index e7bc7e0c..4a887937 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -60,7 +60,7 @@ ecdsa==0.14.1 # sshpubkeys factory-boy==3.2.0 # via -r requirements-tests.in -faker==6.1.1 +faker==6.5.0 # via # -r requirements-tests.in # factory-boy From 775a7b7da844ecf770ad7a1f44b536f36dc4d1c9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 24 Feb 2021 23:51:43 +0000 Subject: [PATCH 055/100] Bump botocore from 1.20.7 to 1.20.15 Bumps [botocore](https://github.com/boto/botocore) from 1.20.7 to 1.20.15. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.20.7...1.20.15) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index ee3bec8a..d2476d29 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -58,7 +58,7 @@ boto==2.49.0 # via # -r requirements-tests.txt # moto -botocore==1.20.7 +botocore==1.20.15 # via # -r requirements-docs.in # -r requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 4a887937..b2fe6505 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ boto3==1.17.7 # moto boto==2.49.0 # via moto -botocore==1.20.7 +botocore==1.20.15 # via # aws-xray-sdk # boto3 diff --git a/requirements.txt b/requirements.txt index 35603bef..1467b3b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ blinker==1.4 # raven boto3==1.17.7 # via -r requirements.in -botocore==1.20.7 +botocore==1.20.15 # via # -r requirements.in # boto3 From 358de8e25e774a6cee3f0048e132a7cf774913a9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 25 Feb 2021 00:21:38 +0000 Subject: [PATCH 056/100] Bump boto3 from 1.17.7 to 1.17.15 Bumps [boto3](https://github.com/boto/boto3) from 1.17.7 to 1.17.15. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.17.7...1.17.15) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index d2476d29..edeb5adf 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -48,7 +48,7 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.7 +boto3==1.17.15 # via # -r requirements-docs.in # -r requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index b2fe6505..71014a8c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -18,7 +18,7 @@ bandit==1.7.0 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.17.7 +boto3==1.17.15 # via # aws-sam-translator # moto diff --git a/requirements.txt b/requirements.txt index 1467b3b7..085b91ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.7 +boto3==1.17.15 # via -r requirements.in botocore==1.20.15 # via From 72b4e93e06714ec10084ccb9368cb23038c436c2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 25 Feb 2021 00:49:18 +0000 Subject: [PATCH 057/100] Bump cryptography from 3.4.5 to 3.4.6 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.5 to 3.4.6. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.5...3.4.6) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 576ccd48..fd9da0d3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,7 +18,7 @@ chardet==3.0.4 # via requests colorama==0.4.3 # via twine -cryptography==3.4.5 +cryptography==3.4.6 # via secretstorage distlib==0.3.0 # via virtualenv diff --git a/requirements-docs.txt b/requirements-docs.txt index edeb5adf..045b1898 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -95,7 +95,7 @@ cloudflare==2.8.15 # via -r requirements-docs.in coverage==5.4 # via -r requirements-tests.txt -cryptography==3.4.5 +cryptography==3.4.6 # via # -r requirements-docs.in # -r requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 71014a8c..9ab0e5f7 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -44,7 +44,7 @@ click==7.1.2 # flask coverage==5.4 # via -r requirements-tests.in -cryptography==3.4.5 +cryptography==3.4.6 # via # moto # python-jose diff --git a/requirements.txt b/requirements.txt index 085b91ef..7cfd6068 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ click==7.1.2 # via flask cloudflare==2.8.15 # via -r requirements.in -cryptography==3.4.5 +cryptography==3.4.6 # via # -r requirements.in # acme From a7f8da91b894e4e6c051ccd08c639cbf37175d88 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Wed, 24 Feb 2021 17:42:28 -0800 Subject: [PATCH 058/100] Fix docs --- celerybeat-schedule | Bin 0 -> 16384 bytes lemur/plugins/lemur_aws/sns.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 celerybeat-schedule diff --git a/celerybeat-schedule b/celerybeat-schedule new file mode 100644 index 0000000000000000000000000000000000000000..57dfab375242ffb5787c40e4a5167d864d1b4c9f GIT binary patch literal 16384 zcmeI$zi!h&90%}AoHU8kv}pnYgoIe2X$Ol|g&C>pLN-%`CoIRgDv=VWKl_53NRcvC z@CtnZW`x+8nRoyu1~$YKAeFn^1&kDsTA}hsUt9Lic6{f{$9F&3es}V?&Z>;f78z75 znfOA*`)bA57ag#M00bZa0SG_<0uX=z1Rwwb2teRp6u4;`ji@2u00$+2et>>}et>a+ zJb*laJb*laJb*laJb*m#pXC8dk7`yik=@Z7`HYTxX8**TaIif;@$4}ti=4`7QAr%v zDsX^9dVHWiTugs>|MtVW??2@glkdv%va51||Hb&j8O~Y&4sb{d=5v?u7iQE#PoIhv50=I*&yBU`v-T$Z%P{1H$uso^M|6Uln7<)=j?3%4*ar^xx*G z?s=CdQ)@!ZQ4hdz(j7#;KefEPZ0?S;<7~@c#8@6s-wGb>@5)`3iFp*Xdb~Zgo;m4C zyqY{K8i?+}$md6g-8c}k!h|wm$a_2N)n^f+YSp{l1w#_W&rgw~p`6G1hX z>dEgw_?ew`7Mt86o4mHkG#JZFX(??}kT%6*D!F7gul&NMJhxfym|0N8r4`AdY^oDl l`@yC*v#GmuDRCqjCI~ Date: Wed, 24 Feb 2021 18:09:23 -0800 Subject: [PATCH 059/100] Remove celerybear-schedule --- .gitignore | 1 + celerybeat-schedule | Bin 16384 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 celerybeat-schedule diff --git a/.gitignore b/.gitignore index 72e85f26..02a24ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ lemur/tests/tmp /lemur/plugins/lemur_email/tests/expiration-rendered.html /lemur/plugins/lemur_email/tests/rotation-rendered.html +.celerybeat-schedule diff --git a/celerybeat-schedule b/celerybeat-schedule deleted file mode 100644 index 57dfab375242ffb5787c40e4a5167d864d1b4c9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI$zi!h&90%}AoHU8kv}pnYgoIe2X$Ol|g&C>pLN-%`CoIRgDv=VWKl_53NRcvC z@CtnZW`x+8nRoyu1~$YKAeFn^1&kDsTA}hsUt9Lic6{f{$9F&3es}V?&Z>;f78z75 znfOA*`)bA57ag#M00bZa0SG_<0uX=z1Rwwb2teRp6u4;`ji@2u00$+2et>>}et>a+ zJb*laJb*laJb*laJb*m#pXC8dk7`yik=@Z7`HYTxX8**TaIif;@$4}ti=4`7QAr%v zDsX^9dVHWiTugs>|MtVW??2@glkdv%va51||Hb&j8O~Y&4sb{d=5v?u7iQE#PoIhv50=I*&yBU`v-T$Z%P{1H$uso^M|6Uln7<)=j?3%4*ar^xx*G z?s=CdQ)@!ZQ4hdz(j7#;KefEPZ0?S;<7~@c#8@6s-wGb>@5)`3iFp*Xdb~Zgo;m4C zyqY{K8i?+}$md6g-8c}k!h|wm$a_2N)n^f+YSp{l1w#_W&rgw~p`6G1hX z>dEgw_?ew`7Mt86o4mHkG#JZFX(??}kT%6*D!F7gul&NMJhxfym|0N8r4`AdY^oDl l`@yC*v#GmuDRCqjCI~ Date: Mon, 1 Mar 2021 18:44:01 +0000 Subject: [PATCH 060/100] Bump botocore from 1.20.15 to 1.20.17 Bumps [botocore](https://github.com/boto/botocore) from 1.20.15 to 1.20.17. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.20.15...1.20.17) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 045b1898..6745c102 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -58,7 +58,7 @@ boto==2.49.0 # via # -r requirements-tests.txt # moto -botocore==1.20.15 +botocore==1.20.17 # via # -r requirements-docs.in # -r requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 9ab0e5f7..5551390e 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ boto3==1.17.15 # moto boto==2.49.0 # via moto -botocore==1.20.15 +botocore==1.20.17 # via # aws-xray-sdk # boto3 diff --git a/requirements.txt b/requirements.txt index 7cfd6068..cf8791c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ blinker==1.4 # raven boto3==1.17.15 # via -r requirements.in -botocore==1.20.15 +botocore==1.20.17 # via # -r requirements.in # boto3 From 1b4d511db2a210540b6943ec238118f5d387aa73 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Mar 2021 19:22:35 +0000 Subject: [PATCH 061/100] Bump coverage from 5.4 to 5.5 Bumps [coverage](https://github.com/nedbat/coveragepy) from 5.4 to 5.5. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/coverage-5.4...coverage-5.5) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 6745c102..9b973004 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -93,7 +93,7 @@ click==7.1.2 # flask cloudflare==2.8.15 # via -r requirements-docs.in -coverage==5.4 +coverage==5.5 # via -r requirements-tests.txt cryptography==3.4.6 # via diff --git a/requirements-tests.txt b/requirements-tests.txt index 5551390e..fe97bc09 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -42,7 +42,7 @@ click==7.1.2 # via # black # flask -coverage==5.4 +coverage==5.5 # via -r requirements-tests.in cryptography==3.4.6 # via From 735e7b10f954186e16af70e095faa37ce5340afd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Mar 2021 21:02:28 +0000 Subject: [PATCH 062/100] Bump pytest-flask from 1.1.0 to 1.2.0 Bumps [pytest-flask](https://github.com/pytest-dev/pytest-flask) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/pytest-dev/pytest-flask/releases) - [Changelog](https://github.com/pytest-dev/pytest-flask/blob/master/docs/changelog.rst) - [Commits](https://github.com/pytest-dev/pytest-flask/compare/1.1.0...1.2.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 9b973004..3c579e09 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -352,7 +352,7 @@ pyrsistent==0.16.0 # via # -r requirements-tests.txt # jsonschema -pytest-flask==1.1.0 +pytest-flask==1.2.0 # via -r requirements-tests.txt pytest-mock==3.5.1 # via -r requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index fe97bc09..9abf4f9d 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -144,7 +144,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema -pytest-flask==1.1.0 +pytest-flask==1.2.0 # via -r requirements-tests.in pytest-mock==3.5.1 # via -r requirements-tests.in From d0d888a7f1e8395226333f451fa6dd5d7f042186 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Mar 2021 21:34:19 +0000 Subject: [PATCH 063/100] Bump boto3 from 1.17.15 to 1.17.17 Bumps [boto3](https://github.com/boto/boto3) from 1.17.15 to 1.17.17. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.17.15...1.17.17) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 3c579e09..18c010c7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -48,7 +48,7 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.15 +boto3==1.17.17 # via # -r requirements-docs.in # -r requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 9abf4f9d..0047c5d2 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -18,7 +18,7 @@ bandit==1.7.0 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.17.15 +boto3==1.17.17 # via # aws-sam-translator # moto diff --git a/requirements.txt b/requirements.txt index cf8791c5..231d1995 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.15 +boto3==1.17.17 # via -r requirements.in botocore==1.20.17 # via From aabc7ea319b36db3c337333c51bbbb03489d86e1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Mar 2021 22:18:28 +0000 Subject: [PATCH 064/100] Bump arrow from 0.17.0 to 1.0.2 Bumps [arrow](https://github.com/arrow-py/arrow) from 0.17.0 to 1.0.2. - [Release notes](https://github.com/arrow-py/arrow/releases) - [Changelog](https://github.com/arrow-py/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/arrow-py/arrow/compare/0.17.0...1.0.2) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 18c010c7..1eea2865 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,7 +16,7 @@ appdirs==1.4.3 # via # -r requirements-tests.txt # black -arrow==0.17.0 +arrow==1.0.2 # via -r requirements-docs.in attrs==19.3.0 # via diff --git a/requirements.txt b/requirements.txt index 231d1995..ebb48e91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.17.0 +arrow==1.0.2 # via -r requirements.in asyncpool==1.0 # via -r requirements.in From a49570e5f9755aad0ba9c51819a49dc4316c06b8 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 3 Mar 2021 14:24:22 -0800 Subject: [PATCH 065/100] sts seems to require the region where Lemur is deployed --- lemur/plugins/lemur_aws/sts.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/sts.py b/lemur/plugins/lemur_aws/sts.py index c1bd562c..722b5a2c 100644 --- a/lemur/plugins/lemur_aws/sts.py +++ b/lemur/plugins/lemur_aws/sts.py @@ -20,7 +20,13 @@ def sts_client(service, service_type="client"): def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): - sts = boto3.client("sts", config=config) + if current_app.config.get("LEMUR_AWS_REGION"): + deployment_region = current_app.config.get("LEMUR_AWS_REGION") + sts = boto3.client('sts', region_name=deployment_region, + endpoint_url=f"https://sts.{deployment_region}.amazonaws.com/", + config=config) + else: + sts = boto3.client("sts", config=config) arn = "arn:aws:iam::{0}:role/{1}".format( kwargs.pop("account_number"), current_app.config.get("LEMUR_INSTANCE_PROFILE", "Lemur"), From c3eb463c42be3235620b170b69b1b6fc5266afdb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 3 Mar 2021 14:50:22 -0800 Subject: [PATCH 066/100] documentation --- docs/administration.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index 4cf8e769..c2ca7b04 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -209,6 +209,11 @@ Basic Configuration in the UI. When set to False (the default), the certificate delete API will always return "405 method not allowed" and deleted certificates will always be visible in the UI. (default: `False`) +.. data:: LEMUR_AWS_REGION + :noindex: + + This is an optional config applicable for settings where Lemur is deployed in AWS. For accessing regionalized + STS endpoints, LEMUR_AWS_REGION defines the region where Lemur is deployed. Certificate Default Options --------------------------- From 6f197b255f8e8061102ae256644c6162284872e0 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 3 Mar 2021 15:31:20 -0800 Subject: [PATCH 067/100] enumerating the scope of moto package as required by 2.0.0 --- requirements-tests.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.in b/requirements-tests.in index 610f26f9..9f4ce427 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -7,7 +7,7 @@ factory-boy Faker fakeredis freezegun -moto +moto[sts,ec2,elb,elbv2,iam,s3,sns,sqs,ses] nose pyflakes pytest From a4e12f0d75eaabc108b1baeea84c3a69b9cae01b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 3 Mar 2021 16:02:04 -0800 Subject: [PATCH 068/100] (node:73186) [DEP0005] DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead. --- gulp/build.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gulp/build.js b/gulp/build.js index 405cacc8..13915051 100644 --- a/gulp/build.js +++ b/gulp/build.js @@ -40,7 +40,7 @@ function replaceAll(string, find, replace) { function stringSrc(filename, string) { let src = require('stream').Readable({objectMode: true}); src._read = function () { - this.push(new gutil.File({cwd: '', base: '', path: filename, contents: new Buffer(string)})); + this.push(new gutil.File({cwd: '', base: '', path: filename, contents: Buffer.from(string)})); this.push(null); }; return src; From e530664da6d889c77fb6a670cac115ff6fa52fe0 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 4 Mar 2021 19:11:20 -0800 Subject: [PATCH 069/100] exclude revoked certs from default to auto-rotate --- lemur/certificates/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index b9bc16f0..b4f88923 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -153,6 +153,7 @@ def get_all_certs_attached_to_endpoint_without_autorotate(): return ( Certificate.query.filter(Certificate.endpoints.any()) .filter(Certificate.rotation == false()) + .filter(Certificate.revoked == false()) .filter(Certificate.not_after >= arrow.now()) .filter(not_(Certificate.replaced.any())) .all() # noqa From 8e5e8fdd030162e261a72f1d446a8ee56314dc08 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 4 Mar 2021 19:12:57 -0800 Subject: [PATCH 070/100] tests --- lemur/tests/test_certificates.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 962c40b4..fa90280c 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -84,6 +84,27 @@ def test_get_by_serial(session, certificate): assert found +def test_get_all_certs_attached_to_endpoint_without_autorotate(session): + from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_autorotate, \ + cleanup_after_revoke + from lemur.tests.factories import EndpointFactory, CertificateFactory + + # add a certificate with endpoint + s = EndpointFactory() + CertificateFactory(endpoint=s) + session.commit() + + list_before = get_all_certs_attached_to_endpoint_without_autorotate() + len_list_before = len(list_before) + assert len_list_before > 0 + # revoked the first certificate + first_cert_with_endpoitn = list_before[0] + cleanup_after_revoke(first_cert_with_endpoitn) + + list_after = get_all_certs_attached_to_endpoint_without_autorotate() + assert len(list_after) + 1 == len_list_before + + def test_delete_cert(session): from lemur.certificates.service import delete, get from lemur.tests.factories import CertificateFactory From c579405805ee2929d458217b6f6dc6f633b83195 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 4 Mar 2021 19:13:40 -0800 Subject: [PATCH 071/100] since we have created an endpoint, need to iterate on this endpoint_id here --- lemur/tests/test_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/tests/test_endpoints.py b/lemur/tests/test_endpoints.py index af073e53..895ab5b8 100644 --- a/lemur/tests/test_endpoints.py +++ b/lemur/tests/test_endpoints.py @@ -32,7 +32,7 @@ def test_rotate_certificate(client, source_plugin): ) def test_endpoint_get(client, token, status): assert ( - client.get(api.url_for(Endpoints, endpoint_id=1), headers=token).status_code + client.get(api.url_for(Endpoints, endpoint_id=2), headers=token).status_code == status ) From fdd6140995f390e57a6189e28e99e4dd425698b5 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 4 Mar 2021 19:16:06 -0800 Subject: [PATCH 072/100] typo and removing unused session commit --- lemur/tests/test_certificates.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index fa90280c..87214289 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -90,16 +90,14 @@ def test_get_all_certs_attached_to_endpoint_without_autorotate(session): from lemur.tests.factories import EndpointFactory, CertificateFactory # add a certificate with endpoint - s = EndpointFactory() - CertificateFactory(endpoint=s) - session.commit() + EndpointFactory() list_before = get_all_certs_attached_to_endpoint_without_autorotate() len_list_before = len(list_before) assert len_list_before > 0 # revoked the first certificate - first_cert_with_endpoitn = list_before[0] - cleanup_after_revoke(first_cert_with_endpoitn) + first_cert_with_endpoint = list_before[0] + cleanup_after_revoke(first_cert_with_endpoint) list_after = get_all_certs_attached_to_endpoint_without_autorotate() assert len(list_after) + 1 == len_list_before From 580506f60508500febc4d53308e7512225ef78ef Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 4 Mar 2021 19:21:26 -0800 Subject: [PATCH 073/100] lint --- lemur/tests/test_certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 87214289..06a04397 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -87,7 +87,7 @@ def test_get_by_serial(session, certificate): def test_get_all_certs_attached_to_endpoint_without_autorotate(session): from lemur.certificates.service import get_all_certs_attached_to_endpoint_without_autorotate, \ cleanup_after_revoke - from lemur.tests.factories import EndpointFactory, CertificateFactory + from lemur.tests.factories import EndpointFactory # add a certificate with endpoint EndpointFactory() From 48aeb26b1a0170b0f4ae8f3b39284fbca4468cd6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Mar 2021 13:37:03 +0000 Subject: [PATCH 074/100] Bump arrow from 1.0.2 to 1.0.3 Bumps [arrow](https://github.com/arrow-py/arrow) from 1.0.2 to 1.0.3. - [Release notes](https://github.com/arrow-py/arrow/releases) - [Changelog](https://github.com/arrow-py/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/arrow-py/arrow/compare/1.0.2...1.0.3) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 1eea2865..718a56f1 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -16,7 +16,7 @@ appdirs==1.4.3 # via # -r requirements-tests.txt # black -arrow==1.0.2 +arrow==1.0.3 # via -r requirements-docs.in attrs==19.3.0 # via diff --git a/requirements.txt b/requirements.txt index ebb48e91..d6d4df31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==1.0.2 +arrow==1.0.3 # via -r requirements.in asyncpool==1.0 # via -r requirements.in From 396cc5db4021184278e9e864998931a04356f8b2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Mar 2021 17:26:35 +0000 Subject: [PATCH 075/100] Bump sphinx from 3.5.1 to 3.5.2 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.5.1 to 3.5.2. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.5.1...v3.5.2) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 718a56f1..94e20ff7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -479,7 +479,7 @@ soupsieve==2.0.1 # via beautifulsoup4 sphinx-rtd-theme==0.5.1 # via -r requirements-docs.in -sphinx==3.5.1 +sphinx==3.5.2 # via # -r requirements-docs.in # sphinx-rtd-theme From 31180fdca876840e65bb7aa4c9708a9fc207a5b5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Mar 2021 17:58:02 +0000 Subject: [PATCH 076/100] Bump pre-commit from 2.10.1 to 2.11.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.10.1 to 2.11.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.10.1...v2.11.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index fd9da0d3..8bd7fe60 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -50,7 +50,7 @@ packaging==20.9 # via bleach pkginfo==1.5.0.1 # via twine -pre-commit==2.10.1 +pre-commit==2.11.0 # via -r requirements-dev.in pycodestyle==2.6.0 # via flake8 From b319b335a754ec280a9b8d41cbed6c8b65de4853 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Mar 2021 18:30:02 +0000 Subject: [PATCH 077/100] Bump acme from 1.12.0 to 1.13.0 Bumps [acme](https://github.com/letsencrypt/letsencrypt) from 1.12.0 to 1.13.0. - [Release notes](https://github.com/letsencrypt/letsencrypt/releases) - [Commits](https://github.com/letsencrypt/letsencrypt/compare/v1.12.0...v1.13.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 7 ++----- requirements.txt | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 94e20ff7..6474db02 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.12.0 +acme==1.13.0 # via -r requirements-docs.in alabaster==0.7.12 # via sphinx @@ -105,7 +105,6 @@ cryptography==3.4.6 # paramiko # pyopenssl # python-jose - # requests # sshpubkeys decorator==4.4.2 # via @@ -341,7 +340,6 @@ pyopenssl==20.0.1 # -r requirements-docs.in # acme # josepy - # requests pyparsing==2.4.7 # via # -r requirements-tests.txt @@ -408,7 +406,7 @@ requests-mock==1.8.0 # via -r requirements-tests.txt requests-toolbelt==0.9.1 # via acme -requests[security]==2.25.1 +requests==2.25.1 # via # -r requirements-tests.txt # acme @@ -438,7 +436,6 @@ s3transfer==0.3.3 six==1.15.0 # via # -r requirements-tests.txt - # acme # aws-sam-translator # bandit # bcrypt diff --git a/requirements.txt b/requirements.txt index d6d4df31..dbd867d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.12.0 +acme==1.13.0 # via -r requirements.in alembic-autogenerate-enums==0.0.2 # via -r requirements.in @@ -64,7 +64,6 @@ cryptography==3.4.6 # josepy # paramiko # pyopenssl - # requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 @@ -181,7 +180,6 @@ pyopenssl==20.0.1 # acme # josepy # ndg-httpsclient - # requests pyrfc3339==1.1 # via acme python-dateutil==2.8.1 @@ -213,7 +211,7 @@ redis==3.5.3 # celery requests-toolbelt==0.9.1 # via acme -requests[security]==2.25.1 +requests==2.25.1 # via # -r requirements.in # acme @@ -228,7 +226,6 @@ s3transfer==0.3.3 six==1.15.0 # via # -r requirements.in - # acme # bcrypt # flask-cors # flask-restful From d779c74e84cdbacc87f55a9b9d6276bc107f8a03 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Mar 2021 19:40:20 +0000 Subject: [PATCH 078/100] Bump botocore from 1.20.17 to 1.20.22 Bumps [botocore](https://github.com/boto/botocore) from 1.20.17 to 1.20.22. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.20.17...1.20.22) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 6474db02..aa4ba797 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -58,7 +58,7 @@ boto==2.49.0 # via # -r requirements-tests.txt # moto -botocore==1.20.17 +botocore==1.20.22 # via # -r requirements-docs.in # -r requirements-tests.txt @@ -273,7 +273,7 @@ more-itertools==8.2.0 # via # -r requirements-tests.txt # moto -moto==1.3.16 +moto[ec2,elb,elbv2,iam,s3,ses,sns,sqs,sts]==1.3.16 # via -r requirements-tests.txt mypy-extensions==0.4.3 # via diff --git a/requirements-tests.txt b/requirements-tests.txt index 0047c5d2..a7b03f1d 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ boto3==1.17.17 # moto boto==2.49.0 # via moto -botocore==1.20.17 +botocore==1.20.22 # via # aws-xray-sdk # boto3 @@ -114,7 +114,7 @@ mock==4.0.2 # via moto more-itertools==8.2.0 # via moto -moto==1.3.16 +moto[ec2,elb,elbv2,iam,s3,ses,sns,sqs,sts]==1.3.16 # via -r requirements-tests.in mypy-extensions==0.4.3 # via black diff --git a/requirements.txt b/requirements.txt index dbd867d4..43ed49fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ blinker==1.4 # raven boto3==1.17.17 # via -r requirements.in -botocore==1.20.17 +botocore==1.20.22 # via # -r requirements.in # boto3 From 03014ac194156524ed10ac6bf11b90ba8693dcbf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 8 Mar 2021 20:02:10 +0000 Subject: [PATCH 079/100] Bump boto3 from 1.17.17 to 1.17.22 Bumps [boto3](https://github.com/boto/boto3) from 1.17.17 to 1.17.22. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.17.17...1.17.22) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index aa4ba797..e3d80774 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -48,7 +48,7 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.17 +boto3==1.17.22 # via # -r requirements-docs.in # -r requirements-tests.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index a7b03f1d..65ddcf57 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -18,7 +18,7 @@ bandit==1.7.0 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.17.17 +boto3==1.17.22 # via # aws-sam-translator # moto diff --git a/requirements.txt b/requirements.txt index 43ed49fa..8cd446bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.17 +boto3==1.17.22 # via -r requirements.in botocore==1.20.22 # via From b2bfff341f78971f61cb2fb0e48561e492c8de4b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Mar 2021 19:50:52 -0800 Subject: [PATCH 080/100] adding cert bot --- requirements-tests.in | 1 + requirements-tests.txt | 1 + requirements.in | 1 + requirements.txt | 1 + 4 files changed, 4 insertions(+) diff --git a/requirements-tests.in b/requirements-tests.in index 9f4ce427..9b2b6988 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -3,6 +3,7 @@ bandit black coverage +certbot factory-boy Faker fakeredis diff --git a/requirements-tests.txt b/requirements-tests.txt index 0047c5d2..79f36f73 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -30,6 +30,7 @@ botocore==1.20.17 # boto3 # moto # s3transfer +certbot==1.13.0 certifi==2020.12.5 # via requests cffi==1.14.0 diff --git a/requirements.in b/requirements.in index 1eb96f97..91e6309d 100644 --- a/requirements.in +++ b/requirements.in @@ -7,6 +7,7 @@ asyncpool boto3 botocore celery[redis]==4.4.2 # need to first resolve the module not found error https://github.com/celery/celery/issues/6406 +certbot certifi certsrv CloudFlare diff --git a/requirements.txt b/requirements.txt index ebb48e91..5537a81c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,6 +40,7 @@ botocore==1.20.17 # s3transfer celery[redis]==4.4.2 # via -r requirements.in +certbot==1.13.0 certifi==2020.12.5 # via # -r requirements.in From f2205b6025c734ebf9cc06954e25f9aa9af72c57 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 9 Mar 2021 19:51:55 -0800 Subject: [PATCH 081/100] new test vectors --- lemur/tests/vectors.py | 99 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/lemur/tests/vectors.py b/lemur/tests/vectors.py index 7a78818c..c47017ae 100644 --- a/lemur/tests/vectors.py +++ b/lemur/tests/vectors.py @@ -587,3 +587,102 @@ zwKDoqAD+L4wEg8d890Zy2mbzJnDu2HQiMIROaBldKEAMQA= """ CERT_CHAIN_PKCS7_PEM = CERT_CHAIN_PKCS7_STR.encode('utf-8') + +ACME_CHAIN_LONG_STR = SAN_CERT_STR + """ +-----BEGIN CERTIFICATE----- +MIIFWzCCA0OgAwIBAgIQTfQrldHumzpMLrM7jRBd1jANBgkqhkiG9w0BAQsFADBm +MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy +aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ +ZWFyIFgxMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowWTELMAkGA1UE +BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSgwJgYDVQQD +Ex8oU1RBR0lORykgQXJ0aWZpY2lhbCBBcHJpY290IFIzMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAu6TR8+74b46mOE1FUwBrvxzEYLck3iasmKrcQkb+ +gy/z9Jy7QNIAl0B9pVKp4YU76JwxF5DOZZhi7vK7SbCkK6FbHlyU5BiDYIxbbfvO +L/jVGqdsSjNaJQTg3C3XrJja/HA4WCFEMVoT2wDZm8ABC1N+IQe7Q6FEqc8NwmTS +nmmRQm4TQvr06DP+zgFK/MNubxWWDSbSKKTH5im5j2fZfg+j/tM1bGaczFWw8/lS +nukyn5J2L+NJYnclzkXoh9nMFnyPmVbfyDPOc4Y25aTzVoeBKXa/cZ5MM+WddjdL +biWvm19f1sYn1aRaAIrkppv7kkn83vcth8XCG39qC2ZvaQIDAQABo4IBEDCCAQww +DgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAS +BgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTecnpI3zHDplDfn4Uj31c3S10u +ZTAfBgNVHSMEGDAWgBS182Xy/rAKkh/7PH3zRKCsYyXDFDA2BggrBgEFBQcBAQQq +MCgwJgYIKwYBBQUHMAKGGmh0dHA6Ly9zdGcteDEuaS5sZW5jci5vcmcvMCsGA1Ud +HwQkMCIwIKAeoByGGmh0dHA6Ly9zdGcteDEuYy5sZW5jci5vcmcvMCIGA1UdIAQb +MBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCN +DLam9yN0EFxxn/3p+ruWO6n/9goCAM5PT6cC6fkjMs4uas6UGXJjr5j7PoTQf3C1 +vuxiIGRJC6qxV7yc6U0X+w0Mj85sHI5DnQVWN5+D1er7mp13JJA0xbAbHa3Rlczn +y2Q82XKui8WHuWra0gb2KLpfboYj1Ghgkhr3gau83pC/WQ8HfkwcvSwhIYqTqxoZ +Uq8HIf3M82qS9aKOZE0CEmSyR1zZqQxJUT7emOUapkUN9poJ9zGc+FgRZvdro0XB +yphWXDaqMYph0DxW/10ig5j4xmmNDjCRmqIKsKoWA52wBTKKXK1na2ty/lW5dhtA +xkz5rVZFd4sgS4J0O+zm6d5GRkWsNJ4knotGXl8vtS3X40KXeb3A5+/3p0qaD215 +Xq8oSNORfB2oI1kQuyEAJ5xvPTdfwRlyRG3lFYodrRg6poUBD/8fNTXMtzydpRgy +zUQZh/18F6B/iW6cbiRN9r2Hkh05Om+q0/6w0DdZe+8YrNpfhSObr/1eVZbKGMIY +qKmyZbBNu5ysENIK5MPc14mUeKmFjpN840VR5zunoU52lqpLDua/qIM8idk86xGW +xx2ml43DO/Ya/tVZVok0mO0TUjzJIfPqyvr455IsIut4RlCR9Iq0EDTve2/ZwCuG +hSjpTUFGSiQrR2JK2Evp+o6AETUkBCO1aw0PpQBPDQ== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFVDCCBDygAwIBAgIRAO1dW8lt+99NPs1qSY3Rs8cwDQYJKoZIhvcNAQELBQAw +cTELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1 +cml0eSBSZXNlYXJjaCBHcm91cDEtMCsGA1UEAxMkKFNUQUdJTkcpIERvY3RvcmVk +IER1cmlhbiBSb290IENBIFgzMB4XDTIxMDEyMDE5MTQwM1oXDTI0MDkzMDE4MTQw +M1owZjELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBT +ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEiMCAGA1UEAxMZKFNUQUdJTkcpIFByZXRl +bmQgUGVhciBYMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALbagEdD +Ta1QgGBWSYkyMhscZXENOBaVRTMX1hceJENgsL0Ma49D3MilI4KS38mtkmdF6cPW +nL++fgehT0FbRHZgjOEr8UAN4jH6omjrbTD++VZneTsMVaGamQmDdFl5g1gYaigk +kmx8OiCO68a4QXg4wSyn6iDipKP8utsE+x1E28SA75HOYqpdrk4HGxuULvlr03wZ +GTIf/oRt2/c+dYmDoaJhge+GOrLAEQByO7+8+vzOwpNAPEx6LW+crEEZ7eBXih6V +P19sTGy3yfqK5tPtTdXXCOQMKAp+gCj/VByhmIr+0iNDC540gtvV303WpcbwnkkL +YC0Ft2cYUyHtkstOfRcRO+K2cZozoSwVPyB8/J9RpcRK3jgnX9lujfwA/pAbP0J2 +UPQFxmWFRQnFjaq6rkqbNEBgLy+kFL1NEsRbvFbKrRi5bYy2lNms2NJPZvdNQbT/ +2dBZKmJqxHkxCuOQFjhJQNeO+Njm1Z1iATS/3rts2yZlqXKsxQUzN6vNbD8KnXRM +EeOXUYvbV4lqfCf8mS14WEbSiMy87GB5S9ucSV1XUrlTG5UGcMSZOBcEUpisRPEm +QWUOTWIoDQ5FOia/GI+Ki523r2ruEmbmG37EBSBXdxIdndqrjy+QVAmCebyDx9eV +EGOIpn26bW5LKerumJxa/CFBaKi4bRvmdJRLAgMBAAGjgfEwge4wDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLXzZfL+sAqSH/s8ffNE +oKxjJcMUMB8GA1UdIwQYMBaAFAhX2onHolN5DE/d4JCPdLriJ3NEMDgGCCsGAQUF +BwEBBCwwKjAoBggrBgEFBQcwAoYcaHR0cDovL3N0Zy1kc3QzLmkubGVuY3Iub3Jn +LzAtBgNVHR8EJjAkMCKgIKAehhxodHRwOi8vc3RnLWRzdDMuYy5sZW5jci5vcmcv +MCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEB +CwUAA4IBAQB7tR8B0eIQSS6MhP5kuvGth+dN02DsIhr0yJtk2ehIcPIqSxRRmHGl +4u2c3QlvEpeRDp2w7eQdRTlI/WnNhY4JOofpMf2zwABgBWtAu0VooQcZZTpQruig +F/z6xYkBk3UHkjeqxzMN3d1EqGusxJoqgdTouZ5X5QTTIee9nQ3LEhWnRSXDx7Y0 +ttR1BGfcdqHopO4IBqAhbkKRjF5zj7OD8cG35omywUbZtOJnftiI0nFcRaxbXo0v +oDfLD0S6+AC2R3tKpqjkNX6/91hrRFglUakyMcZU/xleqbv6+Lr3YD8PsBTub6lI +oZ2lS38fL18Aon458fbc0BPHtenfhKj5 +-----END CERTIFICATE----- +""" + +ACME_CHAIN_SHORT_STR = SAN_CERT_STR + """ +-----BEGIN CERTIFICATE----- +MIIFWzCCA0OgAwIBAgIQTfQrldHumzpMLrM7jRBd1jANBgkqhkiG9w0BAQsFADBm +MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy +aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ +ZWFyIFgxMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowWTELMAkGA1UE +BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSgwJgYDVQQD +Ex8oU1RBR0lORykgQXJ0aWZpY2lhbCBBcHJpY290IFIzMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAu6TR8+74b46mOE1FUwBrvxzEYLck3iasmKrcQkb+ +gy/z9Jy7QNIAl0B9pVKp4YU76JwxF5DOZZhi7vK7SbCkK6FbHlyU5BiDYIxbbfvO +L/jVGqdsSjNaJQTg3C3XrJja/HA4WCFEMVoT2wDZm8ABC1N+IQe7Q6FEqc8NwmTS +nmmRQm4TQvr06DP+zgFK/MNubxWWDSbSKKTH5im5j2fZfg+j/tM1bGaczFWw8/lS +nukyn5J2L+NJYnclzkXoh9nMFnyPmVbfyDPOc4Y25aTzVoeBKXa/cZ5MM+WddjdL +biWvm19f1sYn1aRaAIrkppv7kkn83vcth8XCG39qC2ZvaQIDAQABo4IBEDCCAQww +DgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAS +BgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTecnpI3zHDplDfn4Uj31c3S10u +ZTAfBgNVHSMEGDAWgBS182Xy/rAKkh/7PH3zRKCsYyXDFDA2BggrBgEFBQcBAQQq +MCgwJgYIKwYBBQUHMAKGGmh0dHA6Ly9zdGcteDEuaS5sZW5jci5vcmcvMCsGA1Ud +HwQkMCIwIKAeoByGGmh0dHA6Ly9zdGcteDEuYy5sZW5jci5vcmcvMCIGA1UdIAQb +MBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCN +DLam9yN0EFxxn/3p+ruWO6n/9goCAM5PT6cC6fkjMs4uas6UGXJjr5j7PoTQf3C1 +vuxiIGRJC6qxV7yc6U0X+w0Mj85sHI5DnQVWN5+D1er7mp13JJA0xbAbHa3Rlczn +y2Q82XKui8WHuWra0gb2KLpfboYj1Ghgkhr3gau83pC/WQ8HfkwcvSwhIYqTqxoZ +Uq8HIf3M82qS9aKOZE0CEmSyR1zZqQxJUT7emOUapkUN9poJ9zGc+FgRZvdro0XB +yphWXDaqMYph0DxW/10ig5j4xmmNDjCRmqIKsKoWA52wBTKKXK1na2ty/lW5dhtA +xkz5rVZFd4sgS4J0O+zm6d5GRkWsNJ4knotGXl8vtS3X40KXeb3A5+/3p0qaD215 +Xq8oSNORfB2oI1kQuyEAJ5xvPTdfwRlyRG3lFYodrRg6poUBD/8fNTXMtzydpRgy +zUQZh/18F6B/iW6cbiRN9r2Hkh05Om+q0/6w0DdZe+8YrNpfhSObr/1eVZbKGMIY +qKmyZbBNu5ysENIK5MPc14mUeKmFjpN840VR5zunoU52lqpLDua/qIM8idk86xGW +xx2ml43DO/Ya/tVZVok0mO0TUjzJIfPqyvr455IsIut4RlCR9Iq0EDTve2/ZwCuG +hSjpTUFGSiQrR2JK2Evp+o6AETUkBCO1aw0PpQBPDQ== +-----END CERTIFICATE----- +""" From caa44c1531bf41a841392d6a6631071d4e7b0faa Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 10 Mar 2021 12:39:50 -0800 Subject: [PATCH 082/100] adding functionality to fetch the desired chain --- lemur/plugins/lemur_acme/acme_handlers.py | 24 +++++++++++++-------- lemur/plugins/lemur_acme/challenge_types.py | 9 +++++--- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index 4de6c3f0..8636d882 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -23,6 +23,7 @@ from acme import challenges, errors, messages from acme.client import BackwardsCompatibleClientV2, ClientNetwork from acme.errors import TimeoutError from acme.messages import Error as AcmeError +from certbot import crypto_util as acme_crypto_util from flask import current_app from lemur.common.utils import generate_private_key @@ -92,7 +93,8 @@ class AcmeHandler(object): deadline = datetime.datetime.now() + datetime.timedelta(seconds=360) try: - orderr = acme_client.poll_and_finalize(order, deadline) + orderr = acme_client.poll_authorizations(order, deadline) + orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True) except (AcmeError, TimeoutError): sentry.captureException(extra={"order_url": str(order.uri)}) @@ -112,14 +114,23 @@ class AcmeHandler(object): f"Successfully resolved Acme order: {order.uri}", exc_info=True ) - pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem) + pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem, + orderr.alternative_fullchains_pem) current_app.logger.debug( "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) ) return pem_certificate, pem_certificate_chain - def extract_cert_and_chain(self, fullchain_pem): + def extract_cert_and_chain(self, fullchain_pem, alternative_fullchains_pem, preferred_issuer=None): + + if not preferred_issuer: + preferred_issuer = current_app.config.get("ACME_PREFERRED_ISSUER", None) + if preferred_issuer: + # returns first chain if not match + fullchain_pem = acme_crypto_util.find_chain_with_issuer([fullchain_pem] + alternative_fullchains_pem, + preferred_issuer) + pem_certificate = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate( @@ -127,12 +138,7 @@ class AcmeHandler(object): ), ).decode() - if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ - and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): - pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") - else: - pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() + pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() return pem_certificate, pem_certificate_chain diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index 538ec236..49ae47a0 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -119,8 +119,10 @@ class AcmeHttpChallenge(AcmeChallenge): current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") try: - finalized_orderr = acme_client.poll_and_finalize(orderr, - datetime.datetime.now() + datetime.timedelta(seconds=90)) + deadline = datetime.datetime.now() + datetime.timedelta(seconds=90) + orderr = acme_client.poll_authorizations(orderr, deadline) + finalized_orderr = acme_client.finalize_order(orderr, deadline, fetch_alternative_chains=True) + except errors.ValidationError as validationError: for authz in validationError.failed_authzrs: for chall in authz.body.challenges: @@ -130,7 +132,8 @@ class AcmeHttpChallenge(AcmeChallenge): ERROR_CODES[chall.error.code])) raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.') - pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem) + pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem, + finalized_orderr.alternative_fullchains_pem) if len(deployed_challenges) != 0: for token_path in deployed_challenges: From 4937c5dc2c7ebccf8e489c78df4fd90fec954885 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 10 Mar 2021 12:42:42 -0800 Subject: [PATCH 083/100] testing test_extract_cert_and_chain --- .../lemur_acme/tests/test_acme_handler.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 324af5ac..74211b1b 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -5,6 +5,12 @@ from flask import Flask from cryptography.x509 import DNSName from lemur.plugins.lemur_acme import acme_handlers +from lemur.tests.vectors import ( + ACME_CHAIN_SHORT_STR, + ACME_CHAIN_LONG_STR, + SAN_CERT_STR, +) + class TestAcmeHandler(unittest.TestCase): def setUp(self): @@ -110,3 +116,18 @@ class TestAcmeHandler(unittest.TestCase): self.assertEqual( result, [options["common_name"], "test2.netflix.net"] ) + + def test_extract_cert_and_chain(self): + # expecting the short chain + leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR, + [ACME_CHAIN_LONG_STR], + "(STAGING) Artificial Apricot R3") + self.assertEqual(leaf_pem, SAN_CERT_STR) + self.assertEqual(chain_pem, ACME_CHAIN_SHORT_STR[len(leaf_pem):].lstrip()) + + # expecting the long chain + leaf_pem, chain_pem = self.acme.extract_cert_and_chain(ACME_CHAIN_SHORT_STR, + [ACME_CHAIN_LONG_STR], + "(STAGING) Doctored Durian Root CA X3") + self.assertEqual(leaf_pem, SAN_CERT_STR) + self.assertEqual(chain_pem, ACME_CHAIN_LONG_STR[len(leaf_pem):].lstrip()) From 45712c582bbca3552db907db37c26994dd46c12d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 10 Mar 2021 12:43:36 -0800 Subject: [PATCH 084/100] fixing the mock to include the alternative chain --- lemur/plugins/lemur_acme/tests/test_acme_http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index 0df9e6b2..d81e165d 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -146,7 +146,8 @@ Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4 idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng== -----END CERTIFICATE----- """ - mock_client.poll_and_finalize.return_value = mock_finalized_order + mock_finalized_order.alternative_fullchains_pem = [mock_finalized_order.fullchain_pem] + mock_client.finalize_order.return_value = mock_finalized_order mock_acme.return_value = (mock_client, "") From addaa3ab1301580a2ca75fddc6d984ffbe0448aa Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 10 Mar 2021 12:43:58 -0800 Subject: [PATCH 085/100] adding the config as an example --- lemur/tests/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index 3dfb5621..51b61a3d 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -201,6 +201,7 @@ ACME_EMAIL = "jim@example.com" ACME_TEL = "4088675309" ACME_DIRECTORY_URL = "https://acme-v01.api.letsencrypt.org" ACME_DISABLE_AUTORESOLVE = True +ACME_PREFERRED_ISSUER = "R3" LDAP_AUTH = True LDAP_BIND_URI = "ldap://localhost" From 13539814db5aed2dcb21be6f0c2456dfcd4d34d2 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 10 Mar 2021 13:23:21 -0800 Subject: [PATCH 086/100] adding documentation --- docs/administration.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/administration.rst b/docs/administration.rst index c2ca7b04..9f4d47d5 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -871,6 +871,15 @@ account. See :ref:`Using a pre-existing ACME account ` for mor This is the registration for the ACME account, the most important part is the uri attribute (in JSON) +.. data:: ACME_PREFERRED_ISSUER + :noindex: + + This is an optional parameter to indicate the preferred chain to retrieve from ACME. + This is applicable to Let's Encrypts recent `migration https://letsencrypt.org/certificates/`_ to their own root where they provide two distinct certificate chains; + the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain is rooted in X1 root CA. + Select "X1" to get the shorter chain (currently alternative), leave blank or "DST Root CA X3" for the longer chain. + + Active Directory Certificate Services Plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From e48f4ffe7748bc838e9e0ceb654dec61fdbf49d1 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 10 Mar 2021 13:29:53 -0800 Subject: [PATCH 087/100] improved documentation --- docs/administration.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 9f4d47d5..af18167e 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -874,9 +874,11 @@ account. See :ref:`Using a pre-existing ACME account ` for mor .. data:: ACME_PREFERRED_ISSUER :noindex: - This is an optional parameter to indicate the preferred chain to retrieve from ACME. - This is applicable to Let's Encrypts recent `migration https://letsencrypt.org/certificates/`_ to their own root where they provide two distinct certificate chains; - the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain is rooted in X1 root CA. + This is an optional parameter to indicate the preferred chain to retrieve from ACME when finalizing the order. + This is applicable to Let's Encrypts recent `migration https://letsencrypt.org/certificates/`_ to their + own root, where they provide two distinct certificate chains (fullchain_pem vs. alternative_fullchains_pem); + the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain + is rooted in X1 root CA. Select "X1" to get the shorter chain (currently alternative), leave blank or "DST Root CA X3" for the longer chain. From 8d44ab2124f9e5a7c06f4ddb6fa3efd605b5498a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 10 Mar 2021 16:41:24 -0800 Subject: [PATCH 088/100] Automate Lemur release Lemur's current release publishing is done manual, which comes with overheads and slows down the release cycle. Automating this operation would allow Lemur to make more frequent releases, for instance at least once a week to pick up the latest dependency updates. --- .../workflows/lemur-publish-release-pypi.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/lemur-publish-release-pypi.yml diff --git a/.github/workflows/lemur-publish-release-pypi.yml b/.github/workflows/lemur-publish-release-pypi.yml new file mode 100644 index 00000000..26185489 --- /dev/null +++ b/.github/workflows/lemur-publish-release-pypi.yml @@ -0,0 +1,31 @@ +# This workflow will upload a Python Package using Twine when a Lemur release is created via github +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Publish Lemur's latest package to PyPI + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.LEMUR_PYPI_API_USERNAME }} + TWINE_PASSWORD: ${{ secrets.LEMUR_PYPI_API_TOKEN }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From 04b9df0a34000f0da3ba919d7e93a2880e6cd931 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Wed, 10 Mar 2021 19:58:41 -0800 Subject: [PATCH 089/100] Update Security disclosure process --- docs/security.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/docs/security.rst b/docs/security.rst index e2712e1f..e4a7ccf6 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -22,7 +22,7 @@ Supported Versions ------------------ At any given time, we will provide security support for the `master`_ branch -as well as the 2 most recent releases. +as well as the most recent release. Disclosure Process ------------------ @@ -30,20 +30,15 @@ Disclosure Process Our process for taking a security issue from private discussion to public disclosure involves multiple steps. -Approximately one week before full public disclosure, we will send advance -notification of the issue to a list of people and organizations, primarily -composed of operating-system vendors and other distributors of -``lemur``. This notification will consist of an email message -containing: +Approximately one week before full public disclosure, we will provide advanced notification that a security issue exists. Depending on the severity of the issue, we may choose to either send a targeted email to known Lemur users and contributors or post an issue to the Lemur repository. In either case, the notification should contain the following. -* A full description of the issue and the affected versions of - ``lemur``. +* A description of the potential impact +* The affected versions of ``lemur``. * The steps we will be taking to remedy the issue. -* The patches, if any, that will be applied to ``lemur``. * The date on which the ``lemur`` team will apply these patches, issue new releases, and publicly disclose the issue. -Simultaneously, the reporter of the issue will receive notification of the date +If the issue was disclosed to us, the reporter will receive notification of the date on which we plan to make the issue public. On the day of disclosure, we will take the following steps: @@ -52,7 +47,7 @@ On the day of disclosure, we will take the following steps: messages for these patches will indicate that they are for security issues, but will not describe the issue in any detail; instead, they will warn of upcoming disclosure. -* Issue the relevant releases. +* Issue an updated release. If a reported issue is believed to be particularly time-sensitive – due to a known exploit in the wild, for example – the time between advance notification From 00e06095476700552a00e1200e4551611631b4bf Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 11 Mar 2021 17:04:28 -0800 Subject: [PATCH 090/100] Doc fix --- docs/administration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index af18167e..bad95026 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -875,7 +875,7 @@ account. See :ref:`Using a pre-existing ACME account ` for mor :noindex: This is an optional parameter to indicate the preferred chain to retrieve from ACME when finalizing the order. - This is applicable to Let's Encrypts recent `migration https://letsencrypt.org/certificates/`_ to their + This is applicable to Let's Encrypts recent `migration `_ to their own root, where they provide two distinct certificate chains (fullchain_pem vs. alternative_fullchains_pem); the main chain will be the long chain that is rooted in the expiring DTS root, whereas the alternative chain is rooted in X1 root CA. From 2b7c151426983385306df2134a3f212faf649485 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 11 Mar 2021 17:10:13 -0800 Subject: [PATCH 091/100] Add missing dependency --- requirements-docs.in | 1 + requirements-docs.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-docs.in b/requirements-docs.in index ec12fbea..87663485 100644 --- a/requirements-docs.in +++ b/requirements-docs.in @@ -7,6 +7,7 @@ acme arrow boto3 botocore +certbot certsrv CloudFlare cryptography diff --git a/requirements-docs.txt b/requirements-docs.txt index e3d80774..84357685 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -66,6 +66,7 @@ botocore==1.20.22 # boto3 # moto # s3transfer +certbot==1.13.0 certifi==2020.12.5 # via # -r requirements-tests.txt From 97bdb8a00a2115b58ef6dadccf9d7aa7e5c4a926 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Mar 2021 17:14:51 -0800 Subject: [PATCH 092/100] Change log for 0.8.1 --- CHANGELOG.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 24db16d0..0c850f6a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,36 @@ Changelog ========= +0.8.1 - `2021-03-12` +~~~~~~~~~~~~~~~~~~~~ + +This release includes improvements on many fronts, such as: + +- Notifications: + - Enhanced SNS flow + - Expiration Summary + - CA expiration email +- EC algorithm as the default +- Improved revocation flow +- Localized AWS STS option +- Improved Lemur doc building +- ACME: + - reduced failed attempts to 3x trials + - support for selecting the chain (Let's Encrypt X1 transition) + - revocation + - http01 documentation +- Entrust: + - Support for cross-signed intermediate CA +- Dependency updates and conflict resolutions + +Special thanks to all who contributed to this release, notably: + +- `peschmae `_ +- `atugushev `_ +- `sirferl `_ + + + 0.8.0 - `2020-11-13` ~~~~~~~~~~~~~~~~~~~~ From a630721a3a458b57505dfaf7030a002e1e6ad48a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 11 Mar 2021 17:16:03 -0800 Subject: [PATCH 093/100] Update CHANGELOG.rst --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0c850f6a..a470bdc4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,7 @@ This release includes improvements on many fronts, such as: - http01 documentation - Entrust: - Support for cross-signed intermediate CA +- Revised disclosure process - Dependency updates and conflict resolutions Special thanks to all who contributed to this release, notably: From d1c21fa23ae4eeeb80f67283dba270fd20760a02 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Mar 2021 10:11:47 -0800 Subject: [PATCH 094/100] updating version number --- lemur/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/__about__.py b/lemur/__about__.py index 0926ef33..b0d09167 100644 --- a/lemur/__about__.py +++ b/lemur/__about__.py @@ -15,7 +15,7 @@ __title__ = "lemur" __summary__ = "Certificate management and orchestration service" __uri__ = "https://github.com/Netflix/lemur" -__version__ = "0.8.0" +__version__ = "0.8.1" __author__ = "The Lemur developers" __email__ = "security@netflix.com" From 29b5c554d618a51b9c8a9843251431cd3c13dd32 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Mar 2021 10:42:59 -0800 Subject: [PATCH 095/100] allow automatic version bumping --- .github/workflows/lemur-publish-release-pypi.yml | 10 ++++++++++ lemur/__about__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lemur-publish-release-pypi.yml b/.github/workflows/lemur-publish-release-pypi.yml index 26185489..816146d0 100644 --- a/.github/workflows/lemur-publish-release-pypi.yml +++ b/.github/workflows/lemur-publish-release-pypi.yml @@ -18,6 +18,16 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.x' + - name: Autobump version + run: | + # from refs/tags/v0.8.1 get 0.8.1 + VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + PLACEHOLDER='__version__ = "develop"' + VERSION_FILE='lemur/__about__.py' + # in case placeholder is missing, exists with code 1 and github actions aborts the build + grep "$PLACEHOLDER" "$VERSION_FILE" + sed -i "s/$PLACEHOLDER/__version__ = \"${VERSION}\"/g" "$VERSION_FILE" + shell: bash - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/lemur/__about__.py b/lemur/__about__.py index b0d09167..2a6db3c1 100644 --- a/lemur/__about__.py +++ b/lemur/__about__.py @@ -15,7 +15,7 @@ __title__ = "lemur" __summary__ = "Certificate management and orchestration service" __uri__ = "https://github.com/Netflix/lemur" -__version__ = "0.8.1" +__version__ = "develop" __author__ = "The Lemur developers" __email__ = "security@netflix.com" From 1d486cf1fd69dc7a37d416866004ea9fc4aafb88 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Mar 2021 11:49:17 -0800 Subject: [PATCH 096/100] updated docs for automated release --- docs/doing-a-release.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/doing-a-release.rst b/docs/doing-a-release.rst index 747668fb..02733f89 100644 --- a/docs/doing-a-release.rst +++ b/docs/doing-a-release.rst @@ -1,9 +1,17 @@ Doing a release =============== -Doing a release of ``lemur`` requires a few steps. +Doing a release of ``lemur`` is now mostly automated and consists of the following steps: -Bumping the version number +* Raise a PR to add the release date and summary in the :doc:`/changelog`. +* Merge above PR and create a new `Github release `_: set the tag starting with v, e.g., v0.9.0 + +The `publish workflow `_ uses the git +tag to set the release version. + +The following describes the manual release steps, which is now obsolete: + +Manually Bumping the version number -------------------------- The next step in doing a release is bumping the version number in the @@ -14,7 +22,7 @@ software. * Do a commit indicating this, and raise a pull request with this. * Wait for it to be merged. -Performing the release +Manually Performing the release ---------------------- The commit that merged the version number bump is now the official release From a0a5e66cc3a033ea4da121b651dbb2b152ef7a23 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 12 Mar 2021 12:10:38 -0800 Subject: [PATCH 097/100] fixing broken doc --- docs/doing-a-release.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doing-a-release.rst b/docs/doing-a-release.rst index 02733f89..8e24e7c1 100644 --- a/docs/doing-a-release.rst +++ b/docs/doing-a-release.rst @@ -12,7 +12,7 @@ tag to set the release version. The following describes the manual release steps, which is now obsolete: Manually Bumping the version number --------------------------- +----------------------------------- The next step in doing a release is bumping the version number in the software. @@ -23,7 +23,7 @@ software. * Wait for it to be merged. Manually Performing the release ----------------------- +------------------------------- The commit that merged the version number bump is now the official release commit for this release. You need an `API key `_, From 0d388a85bb2c2350e1dfcf22709b4d40d6eba85b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 15 Mar 2021 13:49:12 +0000 Subject: [PATCH 098/100] Bump boto3 from 1.17.22 to 1.17.27 Bumps [boto3](https://github.com/boto/boto3) from 1.17.22 to 1.17.27. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.17.22...1.17.27) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 73 +++++++++++++++++++++++++++++++++++++++--- requirements-tests.txt | 61 +++++++++++++++++++++++++++++++++-- requirements.txt | 44 ++++++++++++++++++++++--- 3 files changed, 165 insertions(+), 13 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 84357685..9906a3ea 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -5,7 +5,10 @@ # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # acme==1.13.0 - # via -r requirements-docs.in + # via + # -r requirements-docs.in + # -r requirements-tests.txt + # certbot alabaster==0.7.12 # via sphinx alembic==1.5.5 @@ -48,7 +51,7 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.22 +boto3==1.17.27 # via # -r requirements-docs.in # -r requirements-tests.txt @@ -58,7 +61,7 @@ boto==2.49.0 # via # -r requirements-tests.txt # moto -botocore==1.20.22 +botocore==1.20.27 # via # -r requirements-docs.in # -r requirements-tests.txt @@ -67,6 +70,9 @@ botocore==1.20.22 # moto # s3transfer certbot==1.13.0 + # via + # -r requirements-docs.in + # -r requirements-tests.txt certifi==2020.12.5 # via # -r requirements-tests.txt @@ -94,6 +100,14 @@ click==7.1.2 # flask cloudflare==2.8.15 # via -r requirements-docs.in +configargparse==1.4 + # via + # -r requirements-tests.txt + # certbot +configobj==5.0.6 + # via + # -r requirements-tests.txt + # certbot coverage==5.5 # via -r requirements-tests.txt cryptography==3.4.6 @@ -101,6 +115,7 @@ cryptography==3.4.6 # -r requirements-docs.in # -r requirements-tests.txt # acme + # certbot # josepy # moto # paramiko @@ -111,6 +126,10 @@ decorator==4.4.2 # via # -r requirements-tests.txt # networkx +distro==1.5.0 + # via + # -r requirements-tests.txt + # certbot dnspython3==1.15.0 # via -r requirements-docs.in dnspython==1.15.0 @@ -226,7 +245,9 @@ jmespath==0.9.5 josepy==1.7.0 # via # -r requirements-docs.in + # -r requirements-tests.txt # acme + # certbot jsondiff==1.1.2 # via # -r requirements-tests.txt @@ -293,6 +314,10 @@ packaging==20.3 # sphinx paramiko==2.7.2 # via -r requirements-docs.in +parsedatetime==2.6 + # via + # -r requirements-tests.txt + # certbot pathspec==0.8.0 # via # -r requirements-tests.txt @@ -339,6 +364,7 @@ pynacl==1.4.0 pyopenssl==20.0.1 # via # -r requirements-docs.in + # -r requirements-tests.txt # acme # josepy pyparsing==2.4.7 @@ -346,7 +372,10 @@ pyparsing==2.4.7 # -r requirements-tests.txt # packaging pyrfc3339==1.1 - # via acme + # via + # -r requirements-tests.txt + # acme + # certbot pyrsistent==0.16.0 # via # -r requirements-tests.txt @@ -382,6 +411,7 @@ pytz==2019.3 # -r requirements-tests.txt # acme # babel + # certbot # flask-restful # moto # pyrfc3339 @@ -406,7 +436,9 @@ regex==2020.4.4 requests-mock==1.8.0 # via -r requirements-tests.txt requests-toolbelt==0.9.1 - # via acme + # via + # -r requirements-tests.txt + # acme requests==2.25.1 # via # -r requirements-tests.txt @@ -441,6 +473,7 @@ six==1.15.0 # bandit # bcrypt # cfn-lint + # configobj # docker # ecdsa # fakeredis @@ -564,6 +597,36 @@ zipp==3.1.0 # -r requirements-tests.txt # importlib-metadata # moto +zope.component==4.6.2 + # via + # -r requirements-tests.txt + # certbot +zope.deferredimport==4.3.1 + # via + # -r requirements-tests.txt + # zope.component +zope.deprecation==4.4.0 + # via + # -r requirements-tests.txt + # zope.component +zope.event==4.5.0 + # via + # -r requirements-tests.txt + # zope.component +zope.hookable==5.0.1 + # via + # -r requirements-tests.txt + # zope.component +zope.interface==5.2.0 + # via + # -r requirements-tests.txt + # certbot + # zope.component + # zope.proxy +zope.proxy==4.3.5 + # via + # -r requirements-tests.txt + # zope.deferredimport # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements-tests.txt b/requirements-tests.txt index e0591911..49f56952 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -4,6 +4,8 @@ # # pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in # +acme==1.13.0 + # via certbot appdirs==1.4.3 # via black attrs==19.3.0 @@ -18,19 +20,20 @@ bandit==1.7.0 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.17.22 +boto3==1.17.27 # via # aws-sam-translator # moto boto==2.49.0 # via moto -botocore==1.20.22 +botocore==1.20.27 # via # aws-xray-sdk # boto3 # moto # s3transfer certbot==1.13.0 + # via -r requirements-tests.in certifi==2020.12.5 # via requests cffi==1.14.0 @@ -43,15 +46,25 @@ click==7.1.2 # via # black # flask +configargparse==1.4 + # via certbot +configobj==5.0.6 + # via certbot coverage==5.5 # via -r requirements-tests.in cryptography==3.4.6 # via + # acme + # certbot + # josepy # moto + # pyopenssl # python-jose # sshpubkeys decorator==4.4.2 # via networkx +distro==1.5.0 + # via certbot docker==4.2.0 # via moto ecdsa==0.14.1 @@ -95,6 +108,10 @@ jmespath==0.9.5 # via # boto3 # botocore +josepy==1.7.0 + # via + # acme + # certbot jsondiff==1.1.2 # via moto jsonpatch==1.25 @@ -125,6 +142,8 @@ nose==1.3.7 # via -r requirements-tests.in packaging==20.3 # via pytest +parsedatetime==2.6 + # via certbot pathspec==0.8.0 # via black pbr==5.4.5 @@ -141,8 +160,16 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via -r requirements-tests.in +pyopenssl==20.0.1 + # via + # acme + # josepy pyparsing==2.4.7 # via packaging +pyrfc3339==1.1 + # via + # acme + # certbot pyrsistent==0.16.0 # via jsonschema pytest-flask==1.2.0 @@ -163,7 +190,11 @@ python-dateutil==2.8.1 python-jose[cryptography]==3.1.0 # via moto pytz==2019.3 - # via moto + # via + # acme + # certbot + # moto + # pyrfc3339 pyyaml==5.4.1 # via # -r requirements-tests.in @@ -176,11 +207,15 @@ regex==2020.4.4 # via black requests-mock==1.8.0 # via -r requirements-tests.in +requests-toolbelt==0.9.1 + # via acme requests==2.25.1 # via + # acme # docker # moto # requests-mock + # requests-toolbelt # responses responses==0.10.12 # via moto @@ -193,12 +228,15 @@ six==1.15.0 # aws-sam-translator # bandit # cfn-lint + # configobj # docker # ecdsa # fakeredis + # josepy # jsonschema # moto # packaging + # pyopenssl # pyrsistent # python-dateutil # python-jose @@ -243,6 +281,23 @@ zipp==3.1.0 # via # importlib-metadata # moto +zope.component==4.6.2 + # via certbot +zope.deferredimport==4.3.1 + # via zope.component +zope.deprecation==4.4.0 + # via zope.component +zope.event==4.5.0 + # via zope.component +zope.hookable==5.0.1 + # via zope.component +zope.interface==5.2.0 + # via + # certbot + # zope.component + # zope.proxy +zope.proxy==4.3.5 + # via zope.deferredimport # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements.txt b/requirements.txt index 586f6a5c..c70b42c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,9 @@ # pip-compile --no-index --output-file=requirements.txt requirements.in # acme==1.13.0 - # via -r requirements.in + # via + # -r requirements.in + # certbot alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 @@ -31,9 +33,9 @@ blinker==1.4 # flask-mail # flask-principal # raven -boto3==1.17.22 +boto3==1.17.27 # via -r requirements.in -botocore==1.20.22 +botocore==1.20.27 # via # -r requirements.in # boto3 @@ -41,6 +43,7 @@ botocore==1.20.22 celery[redis]==4.4.2 # via -r requirements.in certbot==1.13.0 + # via -r requirements.in certifi==2020.12.5 # via # -r requirements.in @@ -58,13 +61,20 @@ click==7.1.2 # via flask cloudflare==2.8.15 # via -r requirements.in +configargparse==1.4 + # via certbot +configobj==5.0.6 + # via certbot cryptography==3.4.6 # via # -r requirements.in # acme + # certbot # josepy # paramiko # pyopenssl +distro==1.5.0 + # via certbot dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 @@ -126,7 +136,9 @@ jmespath==0.9.5 # boto3 # botocore josepy==1.7.0 - # via acme + # via + # acme + # certbot jsonlines==1.2.0 # via cloudflare kombu==4.6.8 @@ -151,6 +163,8 @@ ndg-httpsclient==0.5.1 # via -r requirements.in paramiko==2.7.2 # via -r requirements.in +parsedatetime==2.6 + # via certbot pem==21.1.0 # via -r requirements.in psycopg2==2.8.6 @@ -182,7 +196,9 @@ pyopenssl==20.0.1 # josepy # ndg-httpsclient pyrfc3339==1.1 - # via acme + # via + # acme + # certbot python-dateutil==2.8.1 # via # alembic @@ -198,6 +214,7 @@ pytz==2019.3 # via # acme # celery + # certbot # flask-restful # pyrfc3339 pyyaml==5.4.1 @@ -228,6 +245,7 @@ six==1.15.0 # via # -r requirements.in # bcrypt + # configobj # flask-cors # flask-restful # hvac @@ -264,6 +282,22 @@ werkzeug==1.0.1 # via flask xmltodict==0.12.0 # via -r requirements.in +zope.component==4.6.2 + # via certbot +zope.deferredimport==4.3.1 + # via zope.component +zope.deprecation==4.4.0 + # via zope.component +zope.event==4.5.0 + # via zope.component +zope.hookable==5.0.1 + # via zope.component +zope.interface==5.2.0 + # via + # certbot + # zope.component +zope.proxy==4.3.5 + # via zope.deferredimport # The following packages are considered to be unsafe in a requirements file: # setuptools From d097da685aba578d896b4c4b765d33eb8f1d9232 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 15 Mar 2021 18:06:22 +0000 Subject: [PATCH 099/100] Bump pre-commit from 2.11.0 to 2.11.1 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.11.0 to 2.11.1. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.11.0...v2.11.1) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8bd7fe60..f6ea8caa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -50,7 +50,7 @@ packaging==20.9 # via bleach pkginfo==1.5.0.1 # via twine -pre-commit==2.11.0 +pre-commit==2.11.1 # via -r requirements-dev.in pycodestyle==2.6.0 # via flake8 From dc1f1c247ae0017e310b919cda04cdb81694de6d Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 16 Mar 2021 15:39:22 -0700 Subject: [PATCH 100/100] Add config to uptake GitHub's native Dependabot with auto-merge action --- .github/dependabot.yml | 15 +++++++++++++++ .github/workflows/dependabot-auto-merge.yml | 14 ++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/dependabot-auto-merge.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..46b1d24d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ + version: 2 + updates: + - directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "08:00" + timezone: "America/Los_Angeles" + package-ecosystem: "pip" + reviewers: + - "hosseinsh" + - "csine-nflx" + - "charhate" + - "jtschladen" + versioning-strategy: lockfile-only \ No newline at end of file diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml new file mode 100644 index 00000000..be012941 --- /dev/null +++ b/.github/workflows/dependabot-auto-merge.yml @@ -0,0 +1,14 @@ +name: dependabot-auto-merge + +on: + pull_request: + +jobs: + auto-merge: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ahmadnassri/action-dependabot-auto-merge@v2 + with: + target: minor + github-token: ${{ secrets.DEPENDABOT_GITHUB_TOKEN }} \ No newline at end of file