From 40057262e1f2f4d49608077e10f2f4a462fd32f0 Mon Sep 17 00:00:00 2001 From: sirferl Date: Sat, 14 Nov 2020 12:19:16 +0100 Subject: [PATCH 01/25] Azure-Dest: add files --- lemur/plugins/lemur_azure_dest/__init__.py | 4 + lemur/plugins/lemur_azure_dest/plugin.py | 188 ++++++++++++++++++ .../lemur_azure_dest/tests/conftest.py | 1 + 3 files changed, 193 insertions(+) create mode 100644 lemur/plugins/lemur_azure_dest/__init__.py create mode 100755 lemur/plugins/lemur_azure_dest/plugin.py create mode 100644 lemur/plugins/lemur_azure_dest/tests/conftest.py diff --git a/lemur/plugins/lemur_azure_dest/__init__.py b/lemur/plugins/lemur_azure_dest/__init__.py new file mode 100644 index 00000000..f8afd7e3 --- /dev/null +++ b/lemur/plugins/lemur_azure_dest/__init__.py @@ -0,0 +1,4 @@ +try: + VERSION = __import__("pkg_resources").get_distribution(__name__).version +except Exception as e: + VERSION = "unknown" diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py new file mode 100755 index 00000000..12d6e27e --- /dev/null +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -0,0 +1,188 @@ +""" +.. module: lemur.plugins.lemur_azure_dest.plugin + :platform: Unix + :copyright: (c) 2019 + :license: Apache, see LICENCE for more details. + + Plugin for uploading certificates and private key as secret to azure key-vault + that can be pulled down by end point nodes. + +.. moduleauthor:: sirferl +""" +import os +import re +from flask import current_app + +from lemur.common.defaults import common_name, country, state, location, organizational_unit, organization +from lemur.common.utils import parse_certificate +from lemur.plugins.bases import DestinationPlugin +from lemur.plugins.bases import SourcePlugin + +import requests +import json +import base64 + + +def base64encode(string): +# Performs Base64 encoding of string to string using the base64.b64encode() function +# which encodes bytes to bytes. + return base64.b64encode(string.encode()).decode() + + +def handle_response(my_response): + """ + Helper function for parsing responses from the Entrust API. + :param my_response: + :return: :raise Exception: + """ + msg = { + 200: "The request was successful.", + 400: "Keyvault Error" + } + + try: + data = json.loads(my_response.content) + except ValueError: + # catch an empty jason object here + data = {'response': 'No detailed message'} + status_code = my_response.status_code + if status_code > 399: + raise Exception(f"AZURE error: {msg.get(status_code, status_code)}\n{data}") + + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Response", + "status": status_code, + "response": data + } + current_app.logger.info(log_data) + if data == {'response': 'No detailed message'}: + # status if no data + return status_code + else: + # return data from the response + return data + + +def get_access_token(tenant, appID, password, self): + """ + Gets the access token with the appid and the password and returns it + + Improvment option: we can try to save it and renew it only when necessary + + :param tenant: Tenant used + :param appID: Application ID from Azure + :param password: password for Application ID + :return: Access token to post to the keyvault + """ + # prepare the call for the access_token + auth_url = f"https://login.microsoftonline.com/{tenant}/oauth2/token" + post_data = { + 'grant_type' : 'client_credentials', + 'client_id' : appID, + 'client_secret' : password, + 'resource' : 'https://vault.azure.net' + } + try: + response = self.session.post(auth_url, data = post_data) + except requests.exceptions.RequestException as e: + current_app.logger.exception(f"AZURE: Error for POST {e}") + + access_token = json.loads(response.content)["access_token"] + return access_token + + +class AzureDestinationPlugin(DestinationPlugin): + """Azure Keyvault Destination plugin for Lemur""" + + title = "Azure" + slug = "azure-keyvault-destination" + description = "Allow the uploading of certificates to Azure key vault" + + author = "Sirferl" + author_url = "https://github.com/sirferl/lemur" + + options = [ + { + "name": "vaultUrl", + "type": "str", + "required": True, + "validation": "^https?://[a-zA-Z0-9.:-]+$", + "helpMessage": "Valid URL to Azure key vault instance", + }, + { + "name": "azureTenant", + "type": "str", + "required": True, + "validation": "^([a-zA-Z0-9/-/?)+$", + "helpMessage": "Tenant for the Azure Key Vault", + }, + { + "name": "appID", + "type": "str", + "required": True, + "validation": "^([a-zA-Z0-9/-/?)+$", + "helpMessage": "AppID for the Azure Key Vault", + }, + { + "name": "azurePassword", + "type": "str", + "required": True, + "validation": "[0-9a-zA-Z.:_-~]+", + "helpMessage": "Tenant password for the Azure Key Vault", + } + ] + + def __init__(self, *args, **kwargs): + self.session = requests.Session() + super(AzureDestinationPlugin, self).__init__(*args, **kwargs) + + + def upload(self, name, body, private_key, cert_chain, options, **kwargs): + """ + Upload certificate and private key + + :param private_key: + :param cert_chain: + :return: + """ + + # we use the common name to identify the certificate + # Azure does not allow "." in the certificate name we replace them with "-" + cert = parse_certificate(body) + certificate_name = common_name(cert).replace(".","-") + + vault_URI = self.get_option("vaultUrl", options) + tenant = self.get_option("azureTenant", options) + app_id = self.get_option("appID", options) + password = self.get_option("azurePassword", options) + + access_token = get_access_token(tenant, app_id, password, self) + + cert_url = f"{vault_URI}/certificates/{certificate_name}/import?api-version=7.1" + post_header = { + "Authorization" : f"Bearer {access_token}" + } + cert_package = f"{body}\n{private_key}" + current_app.logger.debug(f"AZURE: encoded certificate: {cert_package}") + + post_body = { + "value" : cert_package, + "policy" : { + "key_props": { + "exportable" : True, + "kty" : "RSA", + "key_size" : 2048, + "reuse_key" : True + }, + "secret_props": { + "contentType": "application/x-pem-file" + } + } + } + + try: + response = self.session.post(cert_url, headers = post_header, json = post_body) + except requests.exceptions.RequestException as e: + current_app.logger.exception(f"AZURE: Error for POST {e}") + treturn_value = handle_response(response) diff --git a/lemur/plugins/lemur_azure_dest/tests/conftest.py b/lemur/plugins/lemur_azure_dest/tests/conftest.py new file mode 100644 index 00000000..0e1cd89f --- /dev/null +++ b/lemur/plugins/lemur_azure_dest/tests/conftest.py @@ -0,0 +1 @@ +from lemur.tests.conftest import * # noqa From 62230228a7fe379df658464b8162cf276412caec Mon Sep 17 00:00:00 2001 From: sirferl Date: Sat, 14 Nov 2020 12:49:14 +0100 Subject: [PATCH 02/25] Azure-Dest: Working Plugin --- lemur/plugins/lemur_azure_dest/plugin.py | 12 ++++++++++-- setup.py | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index 12d6e27e..ef4ffd42 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -14,12 +14,14 @@ import re from flask import current_app from lemur.common.defaults import common_name, country, state, location, organizational_unit, organization -from lemur.common.utils import parse_certificate +from lemur.common.utils import parse_certificate, parse_private_key from lemur.plugins.bases import DestinationPlugin from lemur.plugins.bases import SourcePlugin +from cryptography.hazmat.primitives import serialization import requests import json +import sys import base64 @@ -163,7 +165,13 @@ class AzureDestinationPlugin(DestinationPlugin): post_header = { "Authorization" : f"Bearer {access_token}" } - cert_package = f"{body}\n{private_key}" + key_pkcs8 = parse_private_key(private_key).private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + key_pkcs8 = key_pkcs8.decode("utf-8").replace('\\n', '\n') + cert_package = f"{body}\n{key_pkcs8}" current_app.logger.debug(f"AZURE: encoded certificate: {cert_package}") post_body = { diff --git a/setup.py b/setup.py index 59e35b53..b817cc63 100644 --- a/setup.py +++ b/setup.py @@ -157,7 +157,8 @@ setup( 'adcs_issuer = lemur.plugins.lemur_adcs.plugin:ADCSIssuerPlugin', 'adcs_source = lemur.plugins.lemur_adcs.plugin:ADCSSourcePlugin', 'entrust_issuer = lemur.plugins.lemur_entrust.plugin:EntrustIssuerPlugin', - 'entrust_source = lemur.plugins.lemur_entrust.plugin:EntrustSourcePlugin' + 'entrust_source = lemur.plugins.lemur_entrust.plugin:EntrustSourcePlugin', + 'azure_destination = lemur.plugins.lemur_azure_dest.plugin:AzureDestinationPlugin' ], }, classifiers=[ From 48302b6acc8ebe0e696a9ea251abe6c4a5a1d44d Mon Sep 17 00:00:00 2001 From: sirferl Date: Sat, 14 Nov 2020 13:03:27 +0100 Subject: [PATCH 03/25] Azure-Dest: Linted --- lemur/plugins/lemur_azure_dest/plugin.py | 50 +++++++++++------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index ef4ffd42..ecab3a03 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -9,14 +9,11 @@ .. moduleauthor:: sirferl """ -import os -import re from flask import current_app -from lemur.common.defaults import common_name, country, state, location, organizational_unit, organization +from lemur.common.defaults import common_name from lemur.common.utils import parse_certificate, parse_private_key from lemur.plugins.bases import DestinationPlugin -from lemur.plugins.bases import SourcePlugin from cryptography.hazmat.primitives import serialization import requests @@ -26,8 +23,8 @@ import base64 def base64encode(string): -# Performs Base64 encoding of string to string using the base64.b64encode() function -# which encodes bytes to bytes. + # Performs Base64 encoding of string to string using the base64.b64encode() function + # which encodes bytes to bytes. return base64.b64encode(string.encode()).decode() @@ -80,13 +77,13 @@ def get_access_token(tenant, appID, password, self): # prepare the call for the access_token auth_url = f"https://login.microsoftonline.com/{tenant}/oauth2/token" post_data = { - 'grant_type' : 'client_credentials', - 'client_id' : appID, - 'client_secret' : password, - 'resource' : 'https://vault.azure.net' + 'grant_type': 'client_credentials', + 'client_id': appID, + 'client_secret': password, + 'resource': 'https://vault.azure.net' } try: - response = self.session.post(auth_url, data = post_data) + response = self.session.post(auth_url, data=post_data) except requests.exceptions.RequestException as e: current_app.logger.exception(f"AZURE: Error for POST {e}") @@ -139,7 +136,6 @@ class AzureDestinationPlugin(DestinationPlugin): self.session = requests.Session() super(AzureDestinationPlugin, self).__init__(*args, **kwargs) - def upload(self, name, body, private_key, cert_chain, options, **kwargs): """ Upload certificate and private key @@ -152,18 +148,18 @@ class AzureDestinationPlugin(DestinationPlugin): # we use the common name to identify the certificate # Azure does not allow "." in the certificate name we replace them with "-" cert = parse_certificate(body) - certificate_name = common_name(cert).replace(".","-") + certificate_name = common_name(cert).replace(".", "-") vault_URI = self.get_option("vaultUrl", options) tenant = self.get_option("azureTenant", options) app_id = self.get_option("appID", options) password = self.get_option("azurePassword", options) - + access_token = get_access_token(tenant, app_id, password, self) cert_url = f"{vault_URI}/certificates/{certificate_name}/import?api-version=7.1" post_header = { - "Authorization" : f"Bearer {access_token}" + "Authorization": f"Bearer {access_token}" } key_pkcs8 = parse_private_key(private_key).private_bytes( encoding=serialization.Encoding.PEM, @@ -171,26 +167,26 @@ class AzureDestinationPlugin(DestinationPlugin): encryption_algorithm=serialization.NoEncryption(), ) key_pkcs8 = key_pkcs8.decode("utf-8").replace('\\n', '\n') - cert_package = f"{body}\n{key_pkcs8}" + cert_package = f"{body}\n{key_pkcs8}" current_app.logger.debug(f"AZURE: encoded certificate: {cert_package}") post_body = { - "value" : cert_package, - "policy" : { + "value": cert_package, + "policy": { "key_props": { - "exportable" : True, - "kty" : "RSA", - "key_size" : 2048, - "reuse_key" : True - }, - "secret_props": { + "exportable": True, + "kty": "RSA", + "key_size": 2048, + "reuse_key": True + }, + "secret_props":{ "contentType": "application/x-pem-file" - } + } } } try: - response = self.session.post(cert_url, headers = post_header, json = post_body) + response = self.session.post(cert_url, headers=post_header, json=post_body) except requests.exceptions.RequestException as e: current_app.logger.exception(f"AZURE: Error for POST {e}") - treturn_value = handle_response(response) + treturn_value = handle_response(response) From 1b5f17d8b83aef32de57fb695c564dfa10e1379d Mon Sep 17 00:00:00 2001 From: sirferl Date: Sun, 15 Nov 2020 10:28:21 +0100 Subject: [PATCH 04/25] Azure-Dest: More Lint, derive keysize from cert, remove debug output --- lemur/plugins/lemur_azure_dest/plugin.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index ecab3a03..a338d629 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -11,7 +11,7 @@ """ from flask import current_app -from lemur.common.defaults import common_name +from lemur.common.defaults import common_name, bitstrength from lemur.common.utils import parse_certificate, parse_private_key from lemur.plugins.bases import DestinationPlugin @@ -168,20 +168,19 @@ class AzureDestinationPlugin(DestinationPlugin): ) key_pkcs8 = key_pkcs8.decode("utf-8").replace('\\n', '\n') cert_package = f"{body}\n{key_pkcs8}" - current_app.logger.debug(f"AZURE: encoded certificate: {cert_package}") post_body = { "value": cert_package, "policy": { "key_props": { - "exportable": True, - "kty": "RSA", - "key_size": 2048, - "reuse_key": True - }, - "secret_props":{ - "contentType": "application/x-pem-file" - } + "exportable": True, + "kty": "RSA", + "key_size": bitstrength(cert), + "reuse_key": True + }, + "secret_props": { + "contentType": "application/x-pem-file" + } } } @@ -189,4 +188,4 @@ class AzureDestinationPlugin(DestinationPlugin): response = self.session.post(cert_url, headers=post_header, json=post_body) except requests.exceptions.RequestException as e: current_app.logger.exception(f"AZURE: Error for POST {e}") - treturn_value = handle_response(response) + return_value = handle_response(response) From 0521624ccc3784e357704749de29259984c2fe2a Mon Sep 17 00:00:00 2001 From: sirferl Date: Sun, 15 Nov 2020 10:33:36 +0100 Subject: [PATCH 05/25] Azure-Dest: Lint always finds something --- lemur/plugins/lemur_azure_dest/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index a338d629..53860942 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -177,7 +177,7 @@ class AzureDestinationPlugin(DestinationPlugin): "kty": "RSA", "key_size": bitstrength(cert), "reuse_key": True - }, + }, "secret_props": { "contentType": "application/x-pem-file" } From c342fb894d94a88cde3ca7dd2efbf88cc9c687c6 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Mon, 23 Nov 2020 14:55:21 +0100 Subject: [PATCH 06/25] Add documentation for ACME http-01 challenge --- docs/production/index.rst | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/production/index.rst b/docs/production/index.rst index c6f561ca..6b01e951 100644 --- a/docs/production/index.rst +++ b/docs/production/index.rst @@ -415,8 +415,8 @@ And the worker can be started with desired options such as the following:: supervisor or systemd configurations should be created for these in production environments as appropriate. -Add support for LetsEncrypt -=========================== +Add support for LetsEncrypt/ACME +================================ LetsEncrypt is a free, limited-feature certificate authority that offers publicly trusted certificates that are valid for 90 days. LetsEncrypt does not use organizational validation (OV), and instead relies on domain validation (DV). @@ -424,7 +424,10 @@ LetsEncrypt requires that we prove ownership of a domain before we're able to is time we want a certificate. The most common methods to prove ownership are HTTP validation and DNS validation. Lemur supports DNS validation -through the creation of DNS TXT records. +through the creation of DNS TXT records as well as HTTP validation, reusing the destination concept. + +ACME DNS Challenge +------------------ In a nutshell, when we send a certificate request to LetsEncrypt, they generate a random token and ask us to put that token in a DNS text record to prove ownership of a domain. If a certificate request has multiple domains, we must @@ -462,6 +465,24 @@ possible. To enable this functionality, periodically (or through Cron/Celery) ru This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to Lemur's database (in the dns_providers table). Alternatively, you can manually input this data. +ACME HTTP Challenge +------------------- + +The flow for requesting a certificate using the HTTP challenge is not that different from the one described for the DNS +challenge. The only difference is, that instead of creating a DNS TXT record, a file is uploaded to a Webserver which +serves the file at `http:///.well-known/acme-challenge/` + +Currently the HTTP challenge also works without Celery, since it's done while creating the certificate, and doesn't +rely on celery to create the DNS record. This will change when we implement mix & match of acme challenge types. + +To create a HTTP compatible Authority, you first need to create a new destination that will be used to deploy the +challenge token. Visit `Admin` -> `Destination` and click `Create`. The path you provide for the destination needs to +be the exact path that is called when the ACME providers calls ``http:///.well-known/acme-challenge/`. The +token part will be added dynamically by the acme_upload. +Currently only the SFTP and S3 Bucket destination support the ACME HTTP challenge. + +Afterwards you can create a new certificate authority as described in the DNS challenge, but need to choose +`Acme HTTP-01` as the plugin type, and then the destination you created beforehand. LetsEncrypt: pinning to cross-signed ICA ---------------------------------------- From 72da149fde725486c5dd43b1d72d38be2ff07746 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Mon, 23 Nov 2020 15:32:01 +0100 Subject: [PATCH 07/25] Fix ACME revoke_certificate --- lemur/plugins/lemur_acme/acme_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index c1ab5281..55e4a076 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -224,7 +224,7 @@ class AcmeHandler(object): def revoke_certificate(self, certificate): if not self.reuse_account(certificate.authority): raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") - acme_client, _ = self.acme.setup_acme_client(certificate.authority) + acme_client, _ = self.setup_acme_client(certificate.authority) fullchain_com = jose.ComparableX509( OpenSSL.crypto.load_certificate( From 320e0e75133ea592b792564dda9d93879c32dd99 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 20:10:15 +0000 Subject: [PATCH 08/25] Bump pre-commit from 2.8.2 to 2.9.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.8.2 to 2.9.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.8.2...v2.9.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e2eb7051..886f7187 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,7 +24,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.8.2 # via -r requirements-dev.in +pre-commit==2.9.0 # via -r requirements-dev.in pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 From a0104dd026f524f27a75b4501cf61c63113d40ba Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 20:19:34 +0000 Subject: [PATCH 09/25] Bump sphinx from 3.3.0 to 3.3.1 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.3.0...v3.3.1) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 69c4710c..28cf467d 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -79,6 +79,7 @@ pyrfc3339==1.1 # via -r requirements.txt, acme python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore python-editor==1.0.4 # via -r requirements.txt, alembic python-json-logger==0.1.11 # via -r requirements.txt, logmatic-python +python-ldap==3.3.1 # via -r requirements.txt pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339 pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt @@ -91,7 +92,7 @@ six==1.15.0 # via -r requirements.txt, acme, bcrypt, cryptography, snowballstemmer==2.0.0 # via sphinx soupsieve==2.0.1 # via -r requirements.txt, beautifulsoup4 sphinx-rtd-theme==0.5.0 # via -r requirements-docs.in -sphinx==3.3.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.3.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From 378e34d3d8f0991d600dd24f50524922b372a93e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 20:29:06 +0000 Subject: [PATCH 10/25] Bump faker from 4.14.2 to 4.17.1 Bumps [faker](https://github.com/joke2k/faker) from 4.14.2 to 4.17.1. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.md) - [Commits](https://github.com/joke2k/faker/compare/v4.14.2...v4.17.1) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index b82e2ac8..14549802 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in -faker==4.14.2 # via -r requirements-tests.in, factory-boy +faker==4.17.1 # via -r requirements-tests.in, factory-boy fakeredis==1.4.4 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in From 1474a399b7dc657834feec2573c51f2394e19e51 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 20:47:10 +0000 Subject: [PATCH 11/25] Bump requests from 2.24.0 to 2.25.0 Bumps [requests](https://github.com/psf/requests) from 2.24.0 to 2.25.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/master/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.24.0...v2.25.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 886f7187..adc8304b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,7 +32,7 @@ pygments==2.6.1 # via readme-renderer pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit readme-renderer==25.0 # via twine requests-toolbelt==0.9.1 # via twine -requests==2.24.0 # via requests-toolbelt, twine +requests==2.25.0 # via requests-toolbelt, twine rfc3986==1.4.0 # via twine secretstorage==3.1.2 # via keyring six==1.15.0 # via bleach, cryptography, readme-renderer, virtualenv diff --git a/requirements-docs.txt b/requirements-docs.txt index 28cf467d..97af2a9f 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -85,7 +85,7 @@ pyyaml==5.3.1 # via -r requirements.txt, cloudflare raven[flask]==6.10.0 # via -r requirements.txt redis==3.5.3 # via -r requirements.txt, celery requests-toolbelt==0.9.1 # via -r requirements.txt, acme -requests[security]==2.24.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx +requests[security]==2.25.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx retrying==1.3.3 # via -r requirements.txt s3transfer==0.3.3 # via -r requirements.txt, boto3 six==1.15.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils diff --git a/requirements-tests.txt b/requirements-tests.txt index 14549802..1f89a36a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -69,7 +69,7 @@ pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto redis==3.5.3 # via fakeredis regex==2020.4.4 # via black requests-mock==1.8.0 # via -r requirements-tests.in -requests==2.24.0 # via docker, moto, requests-mock, responses +requests==2.25.0 # via docker, moto, requests-mock, responses responses==0.10.12 # via moto rsa==4.0 # via python-jose s3transfer==0.3.3 # via boto3 diff --git a/requirements.txt b/requirements.txt index d7b56f2b..09a2a9b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -78,7 +78,7 @@ pyyaml==5.3.1 # via -r requirements.in, cloudflare raven[flask]==6.10.0 # via -r requirements.in redis==3.5.3 # via -r requirements.in, celery requests-toolbelt==0.9.1 # via acme -requests[security]==2.24.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt +requests[security]==2.25.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt retrying==1.3.3 # via -r requirements.in s3transfer==0.3.3 # via boto3 six==1.15.0 # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils From f6096c62cf3bc642e6d68a01f21235b40dc8be80 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 20:47:33 +0000 Subject: [PATCH 12/25] Bump boto3 from 1.16.14 to 1.16.24 Bumps [boto3](https://github.com/boto/boto3) from 1.16.14 to 1.16.24. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.14...1.16.24) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 28cf467d..080f7041 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.14 # via -r requirements.txt -botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.24 # via -r requirements.txt +botocore==1.19.24 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 14549802..01211051 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.14 # via aws-sam-translator, moto +boto3==1.16.24 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.24 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index d7b56f2b..cb7e9c02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.14 # via -r requirements.in -botocore==1.19.14 # via -r requirements.in, boto3, s3transfer +boto3==1.16.24 # via -r requirements.in +botocore==1.19.24 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 1d247f3df4651becd01ebd92958d67f26b166d9c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 20:57:15 +0000 Subject: [PATCH 13/25] Bump boto3 from 1.16.14 to 1.16.23 Bumps [boto3](https://github.com/boto/boto3) from 1.16.14 to 1.16.23. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.14...1.16.23) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 97af2a9f..0642dce7 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.14 # via -r requirements.txt -botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.24 # via -r requirements.txt +botocore==1.19.24 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 1f89a36a..4fd96f95 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.14 # via aws-sam-translator, moto +boto3==1.16.24 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.24 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 09a2a9b8..e029b61c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.14 # via -r requirements.in -botocore==1.19.14 # via -r requirements.in, boto3, s3transfer +boto3==1.16.24 # via -r requirements.in +botocore==1.19.24 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 1207de8925509a43d327ab2a9af28062591d7ae3 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 23 Nov 2020 15:24:11 -0800 Subject: [PATCH 14/25] Remove certificate from AWS and cleanup after cert revoke --- lemur/certificates/service.py | 59 ++++++++++ lemur/certificates/views.py | 41 +++++-- lemur/plugins/lemur_aws/elb.py | 32 ++++++ lemur/plugins/lemur_aws/plugin.py | 39 +++++++ lemur/plugins/lemur_aws/tests/test_elb.py | 106 +++++++++++++++++- .../certificates/certificate/revoke.tpl.html | 5 +- 6 files changed, 270 insertions(+), 12 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index ac844120..1aabec48 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -21,6 +21,7 @@ from lemur.certificates.schemas import CertificateOutputSchema, CertificateInput from lemur.common.utils import generate_private_key, truthiness from lemur.destinations.models import Destination from lemur.domains.models import Domain +from lemur.endpoints import service as endpoint_service from lemur.extensions import metrics, sentry, signals from lemur.models import certificate_associations from lemur.notifications.models import Notification @@ -797,3 +798,61 @@ def reissue_certificate(certificate, replace=None, user=None): new_cert = create(**primitives) return new_cert + + +def is_attached_to_endpoint(certificate_name, endpoint_name): + """ + Find if given certificate is attached to the endpoint. Both, certificate and endpoint, are identified by name. + This method talks to elb and finds the real time information. + :param certificate_name: + :param endpoint_name: + :return: True if certificate is attached to the given endpoint, False otherwise + """ + endpoint = endpoint_service.get_by_name(endpoint_name) + attached_certificates = endpoint.source.plugin.get_endpoint_certificate_names(endpoint) + return certificate_name in attached_certificates + + +def remove_from_destination(certificate, destination): + """ + Remove the certificate from given destination if clean() is implemented + :param certificate: + :param destination: + :return: + """ + plugin = plugins.get(destination.plugin_name) + if not hasattr(plugin, "clean"): + info_text = f"Cannot clean certificate {certificate.name}, {destination.plugin_name} plugin does not implement 'clean()'" + current_app.logger.warning(info_text) + else: + plugin.clean(certificate=certificate, options=destination.options) + + +def cleanup_after_revoke(certificate): + """ + Perform the needed cleanup for a revoked certificate. This includes - + 1. Disabling notification + 2. Disabling auto-rotation + 3. Update certificate status to 'revoked' + 4. Remove from AWS + :param certificate: Certificate object to modify and update in DB + :return: None + """ + certificate.notify = False + certificate.rotation = False + certificate.status = 'revoked' + + error_message = "" + + for destination in list(certificate.destinations): + try: + remove_from_destination(certificate, destination) + certificate.destinations.remove(destination) + except Exception as e: + # This cleanup is the best-effort since certificate is already revoked at this point. + # We will capture the exception and move on to the next destination + sentry.captureException() + error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. " + + database.update(certificate) + return error_message diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index a066f20f..8b8b36c3 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -19,6 +19,7 @@ from lemur.auth.permissions import AuthorityPermission, CertificatePermission from lemur.certificates import service from lemur.certificates.models import Certificate +from lemur.extensions import sentry from lemur.plugins.base import plugins from lemur.certificates.schemas import ( certificate_input_schema, @@ -888,8 +889,24 @@ class Certificates(AuthenticatedResource): if cert.owner != data["owner"]: service.cleanup_owner_roles_notification(cert.owner, data) + error_message = "" + # if destination is removed, cleanup the certificate from AWS + for destination in cert.destinations: + if destination not in data["destinations"]: + try: + service.remove_from_destination(cert, destination) + except Exception as e: + sentry.captureException() + # Add the removed destination back + data["destinations"].append(destination) + error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. " + + # go ahead with DB update cert = service.update(certificate_id, **data) log_service.create(g.current_user, "update_cert", certificate=cert) + + if error_message: + return dict(message=f"Edit Successful except -\n\n {error_message}"), 400 return cert @validate_schema(certificate_edit_input_schema, certificate_output_schema) @@ -1429,20 +1446,28 @@ class CertificateRevoke(AuthenticatedResource): 403, ) - if not cert.external_id: - return dict(message="Cannot revoke certificate. No external id found."), 400 + # if not cert.external_id: + # return dict(message="Cannot revoke certificate. No external id found."), 400 if cert.endpoints: - return ( - dict( - message="Cannot revoke certificate. Endpoints are deployed with the given certificate." - ), - 403, - ) + for endpoint in cert.endpoints: + if service.is_attached_to_endpoint(cert.name, endpoint.name): + return ( + dict( + message="Cannot revoke certificate. Endpoints are deployed with the given certificate." + ), + 403, + ) plugin = plugins.get(cert.authority.plugin_name) plugin.revoke_certificate(cert, data) + log_service.create(g.current_user, "revoke_cert", certificate=cert) + + # Perform cleanup after revoke + error_message = service.cleanup_after_revoke(cert) + if error_message: + return dict(message=f"Certificate (id:{cert.id}) is revoked - {error_message}"), 400 return dict(id=cert.id) diff --git a/lemur/plugins/lemur_aws/elb.py b/lemur/plugins/lemur_aws/elb.py index 595a3826..1da29276 100644 --- a/lemur/plugins/lemur_aws/elb.py +++ b/lemur/plugins/lemur_aws/elb.py @@ -149,6 +149,38 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs): raise +@sts_client("elbv2") +@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=5) +def get_load_balancer_arn_from_endpoint(endpoint_name, **kwargs): + """ + Get a load balancer ARN from an endpoint. + :param endpoint_name: + :return: + """ + try: + client = kwargs.pop("client") + elbs = client.describe_load_balancers(Names=[endpoint_name]) + for elb in elbs["LoadBalancers"]: + return elb["LoadBalancerArn"] + + except Exception as e: # noqa + metrics.send( + "get_load_balancer_arn_from_endpoint", + "counter", + 1, + metric_tags={ + "error": str(e), + "endpoint_name": endpoint_name, + }, + ) + sentry.captureException( + extra={ + "endpoint_name": str(endpoint_name), + } + ) + raise + + @sts_client("elb") @retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20) def get_elbs(**kwargs): diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index fcc2e0cf..efcce4d0 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -300,6 +300,41 @@ class AWSSourcePlugin(SourcePlugin): ) return None + def get_endpoint_certificate_names(self, endpoint): + options = endpoint.source.options + account_number = self.get_option("accountNumber", options) + region = get_region_from_dns(endpoint.dnsname) + certificate_names = [] + + if endpoint.type == "elb": + elb_details = elb.get_elbs(account_number=account_number, + region=region, + LoadBalancerNames=[endpoint.name],) + + for lb_description in elb_details["LoadBalancerDescriptions"]: + for listener_description in lb_description["ListenerDescriptions"]: + listener = listener_description.get("Listener") + if not listener.get("SSLCertificateId"): + continue + + certificate_names.append(iam.get_name_from_arn(listener.get("SSLCertificateId"))) + elif endpoint.type == "elbv2": + listeners = elb.describe_listeners_v2( + account_number=account_number, + region=region, + LoadBalancerArn=elb.get_load_balancer_arn_from_endpoint(endpoint.name, + account_number=account_number, + region=region), + ) + for listener in listeners["Listeners"]: + if not listener.get("Certificates"): + continue + + for certificate in listener["Certificates"]: + certificate_names.append(iam.get_name_from_arn(certificate["CertificateArn"])) + + return certificate_names + class AWSDestinationPlugin(DestinationPlugin): title = "AWS" @@ -344,6 +379,10 @@ class AWSDestinationPlugin(DestinationPlugin): def deploy(self, elb_name, account, region, certificate): pass + def clean(self, certificate, options, **kwargs): + account_number = self.get_option("accountNumber", options) + iam.delete_cert(certificate.name, account_number=account_number) + class S3DestinationPlugin(ExportDestinationPlugin): title = "AWS-S3" diff --git a/lemur/plugins/lemur_aws/tests/test_elb.py b/lemur/plugins/lemur_aws/tests/test_elb.py index 4571b87a..2b56a7c5 100644 --- a/lemur/plugins/lemur_aws/tests/test_elb.py +++ b/lemur/plugins/lemur_aws/tests/test_elb.py @@ -1,5 +1,5 @@ import boto3 -from moto import mock_sts, mock_elb +from moto import mock_sts, mock_ec2, mock_elb, mock_elbv2, mock_iam @mock_sts() @@ -27,3 +27,107 @@ def test_get_all_elbs(app, aws_credentials): elbs = get_all_elbs(account_number="123456789012", region="us-east-1") assert elbs + + +@mock_sts() +@mock_ec2 +@mock_elbv2() +@mock_iam +def test_create_elb_with_https_listener_miscellaneous(app, aws_credentials): + from lemur.plugins.lemur_aws import iam, elb + endpoint_name = "example-lbv2" + account_number = "123456789012" + region_ue1 = "us-east-1" + + client = boto3.client("elbv2", region_name="us-east-1") + ec2 = boto3.resource("ec2", region_name="us-east-1") + + # Create VPC + vpc = ec2.create_vpc(CidrBlock="172.28.7.0/24") + + # Create LB (elbv2) in above VPC + assert create_load_balancer(client, ec2, vpc.id, endpoint_name) + # Create target group + target_group_arn = create_target_group(client, vpc.id) + assert target_group_arn + + # Test get_load_balancer_arn_from_endpoint + lb_arn = elb.get_load_balancer_arn_from_endpoint(endpoint_name, + account_number=account_number, + region=region_ue1) + assert lb_arn + + # Test describe_listeners_v2 + listeners = elb.describe_listeners_v2(account_number=account_number, + region=region_ue1, + LoadBalancerArn=lb_arn) + assert listeners + assert not listeners["Listeners"] + + # Upload cert + response = iam.upload_cert("LemurTestCert", "testCert", "cert1", "cert2", + account_number=account_number) + assert response + cert_arn = response["ServerCertificateMetadata"]["Arn"] + assert cert_arn + + # Create https listener using above cert + listeners = client.create_listener( + LoadBalancerArn=lb_arn, + Protocol="HTTPS", + Port=443, + Certificates=[{"CertificateArn": cert_arn}], + DefaultActions=[{"Type": "forward", "TargetGroupArn": target_group_arn}], + ) + assert listeners + listener_arn = listeners["Listeners"][0]["ListenerArn"] + assert listener_arn + + assert listeners["Listeners"] + for listener in listeners["Listeners"]: + if listener["Port"] == 443: + assert listener["Certificates"] + assert cert_arn == listener["Certificates"][0]["CertificateArn"] + + # Test get_listener_arn_from_endpoint + assert listener_arn == elb.get_listener_arn_from_endpoint( + endpoint_name, + 443, + account_number=account_number, + region=region_ue1, + ) + + +@mock_sts() +@mock_elb() +def test_get_all_elbs_v2(): + from lemur.plugins.lemur_aws.elb import get_all_elbs_v2 + + elbs = get_all_elbs_v2(account_number="123456789012", + region="us-east-1") + assert elbs + + +def create_load_balancer(client, ec2, vpc_id, endpoint_name): + subnet1 = ec2.create_subnet( + VpcId=vpc_id, + CidrBlock="172.28.7.192/26", + AvailabilityZone="us-east-1a" + ) + + return client.create_load_balancer( + Name=endpoint_name, + Subnets=[ + subnet1.id, + ], + ) + + +def create_target_group(client, vpc_id): + response = client.create_target_group( + Name="a-target", + Protocol="HTTPS", + Port=443, + VpcId=vpc_id, + ) + return response.get("TargetGroups")[0]["TargetGroupArn"] diff --git a/lemur/static/app/angular/certificates/certificate/revoke.tpl.html b/lemur/static/app/angular/certificates/certificate/revoke.tpl.html index d91c7989..779d2ffd 100644 --- a/lemur/static/app/angular/certificates/certificate/revoke.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/revoke.tpl.html @@ -26,9 +26,8 @@
-

Certificate cannot be revoked, it is associated with the following endpoints. Disassociate this - certificate - before revoking.

+

Certificate might be associated with the following endpoints. Disassociate this + certificate before revoking or continue if you've already done so.

  • From 563c7544920cc3cc5a8586ee0013f305121b0de5 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 23 Nov 2020 17:24:01 -0800 Subject: [PATCH 15/25] Uncomment code --- lemur/certificates/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 8b8b36c3..52937fbd 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -1446,8 +1446,8 @@ class CertificateRevoke(AuthenticatedResource): 403, ) - # if not cert.external_id: - # return dict(message="Cannot revoke certificate. No external id found."), 400 + if not cert.external_id: + return dict(message="Cannot revoke certificate. No external id found."), 400 if cert.endpoints: for endpoint in cert.endpoints: From 0f3357ab46a92583cc394ac357f5e850d4f2eae2 Mon Sep 17 00:00:00 2001 From: sirferl Date: Tue, 24 Nov 2020 12:29:25 +0100 Subject: [PATCH 16/25] moved base64encode to common.utils --- lemur/common/utils.py | 5 +++++ lemur/plugins/lemur_azure_dest/plugin.py | 9 +-------- lemur/plugins/lemur_kubernetes/plugin.py | 8 +------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 19b256e8..5d27fa63 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -10,6 +10,7 @@ import random import re import string import pem +import base64 import sqlalchemy from cryptography import x509 @@ -33,6 +34,10 @@ paginated_parser.add_argument("sortBy", type=str, dest="sort_by", location="args paginated_parser.add_argument("filter", type=str, location="args") paginated_parser.add_argument("owner", type=str, location="args") +def base64encode(string): + # Performs Base64 encoding of string to string using the base64.b64encode() function + # which encodes bytes to bytes. + return base64.b64encode(string.encode()).decode() def get_psuedo_random_string(): """ diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index 53860942..e9521260 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -12,20 +12,13 @@ from flask import current_app from lemur.common.defaults import common_name, bitstrength -from lemur.common.utils import parse_certificate, parse_private_key +from lemur.common.utils import parse_certificate, parse_private_key, base64encode from lemur.plugins.bases import DestinationPlugin from cryptography.hazmat.primitives import serialization import requests import json import sys -import base64 - - -def base64encode(string): - # Performs Base64 encoding of string to string using the base64.b64encode() function - # which encodes bytes to bytes. - return base64.b64encode(string.encode()).decode() def handle_response(my_response): diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index f7ff00f7..05613227 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -18,7 +18,7 @@ import requests from flask import current_app from lemur.common.defaults import common_name -from lemur.common.utils import parse_certificate +from lemur.common.utils import parse_certificate, base64encode from lemur.plugins.bases import DestinationPlugin DEFAULT_API_VERSION = "v1" @@ -73,12 +73,6 @@ def _resolve_uri(k8s_base_uri, namespace, kind, name=None, api_ver=DEFAULT_API_V ) -# Performs Base64 encoding of string to string using the base64.b64encode() function -# which encodes bytes to bytes. -def base64encode(string): - return base64.b64encode(string.encode()).decode() - - def build_secret(secret_format, secret_name, body, private_key, cert_chain): secret = { "apiVersion": "v1", From 56af628c685bfa4296fcf8970aa47ead5d7a9b0c Mon Sep 17 00:00:00 2001 From: sirferl Date: Tue, 24 Nov 2020 12:46:09 +0100 Subject: [PATCH 17/25] moved base64encode to common.utils --- lemur/plugins/lemur_azure_dest/plugin.py | 2 +- lemur/plugins/lemur_kubernetes/plugin.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_azure_dest/plugin.py b/lemur/plugins/lemur_azure_dest/plugin.py index e9521260..9282de40 100755 --- a/lemur/plugins/lemur_azure_dest/plugin.py +++ b/lemur/plugins/lemur_azure_dest/plugin.py @@ -12,7 +12,7 @@ from flask import current_app from lemur.common.defaults import common_name, bitstrength -from lemur.common.utils import parse_certificate, parse_private_key, base64encode +from lemur.common.utils import parse_certificate, parse_private_key from lemur.plugins.bases import DestinationPlugin from cryptography.hazmat.primitives import serialization diff --git a/lemur/plugins/lemur_kubernetes/plugin.py b/lemur/plugins/lemur_kubernetes/plugin.py index 05613227..79207636 100644 --- a/lemur/plugins/lemur_kubernetes/plugin.py +++ b/lemur/plugins/lemur_kubernetes/plugin.py @@ -10,7 +10,6 @@ .. moduleauthor:: Mikhail Khodorovskiy """ -import base64 import itertools import os From 439e888d9e785d67ff808ad5d846c76889e546c2 Mon Sep 17 00:00:00 2001 From: sirferl Date: Tue, 24 Nov 2020 12:59:42 +0100 Subject: [PATCH 18/25] lint errors --- lemur/common/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 5d27fa63..7c8a14aa 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -34,11 +34,13 @@ paginated_parser.add_argument("sortBy", type=str, dest="sort_by", location="args paginated_parser.add_argument("filter", type=str, location="args") paginated_parser.add_argument("owner", type=str, location="args") + def base64encode(string): # Performs Base64 encoding of string to string using the base64.b64encode() function # which encodes bytes to bytes. return base64.b64encode(string.encode()).decode() + def get_psuedo_random_string(): """ Create a random and strongish challenge. From e9f7860816040f4c7b121e0ca4ba5d9c0d437a31 Mon Sep 17 00:00:00 2001 From: sirferl Date: Wed, 25 Nov 2020 14:06:26 +0100 Subject: [PATCH 19/25] Fixed AD-LDAP decode problem --- lemur/auth/ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index 030c7c78..48604785 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -211,7 +211,7 @@ class LdapPrincipal: for group in lgroups: (dn, values) = group if type(values) == dict: - self.ldap_groups.append(values["cn"][0].decode("ascii")) + self.ldap_groups.append(values["cn"][0].decode("utf-8")) else: lgroups = self.ldap_client.search_s( self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs From 8ad82bee532f8f90bb4e0cb3ff2ba2c145b356f6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Nov 2020 13:29:01 +0000 Subject: [PATCH 20/25] Bump pre-commit from 2.9.0 to 2.9.2 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.9.0 to 2.9.2. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.9.0...v2.9.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index adc8304b..5ffcb1e4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,7 +24,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.9.0 # via -r requirements-dev.in +pre-commit==2.9.2 # via -r requirements-dev.in pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi pyflakes==2.2.0 # via flake8 From b00d97b6de140c273179cef4434d13572730fa84 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Nov 2020 16:50:51 +0000 Subject: [PATCH 21/25] Bump fakeredis from 1.4.4 to 1.4.5 Bumps [fakeredis](https://github.com/jamesls/fakeredis) from 1.4.4 to 1.4.5. - [Release notes](https://github.com/jamesls/fakeredis/releases) - [Commits](https://github.com/jamesls/fakeredis/compare/1.4.4...1.4.5) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 4fd96f95..0fc940d8 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in faker==4.17.1 # via -r requirements-tests.in, factory-boy -fakeredis==1.4.4 # via -r requirements-tests.in +fakeredis==1.4.5 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk From 1eb0024b1ffb51ec7a490de0ee6eb3d2a4380719 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Nov 2020 17:00:28 +0000 Subject: [PATCH 22/25] Bump boto3 from 1.16.24 to 1.16.25 Bumps [boto3](https://github.com/boto/boto3) from 1.16.24 to 1.16.25. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.24...1.16.25) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 0642dce7..cae4accd 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.24 # via -r requirements.txt -botocore==1.19.24 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.25 # via -r requirements.txt +botocore==1.19.25 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 0fc940d8..90bb7b5e 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.24 # via aws-sam-translator, moto +boto3==1.16.25 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.24 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.25 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index e029b61c..68d5fdb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.24 # via -r requirements.in -botocore==1.19.24 # via -r requirements.in, boto3, s3transfer +boto3==1.16.25 # via -r requirements.in +botocore==1.19.25 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From fa082145fefc431d167a201a172ccf88a044bea7 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 30 Nov 2020 17:09:42 +0000 Subject: [PATCH 23/25] Bump pyopenssl from 19.1.0 to 20.0.0 Bumps [pyopenssl](https://github.com/pyca/pyopenssl) from 19.1.0 to 20.0.0. - [Release notes](https://github.com/pyca/pyopenssl/releases) - [Changelog](https://github.com/pyca/pyopenssl/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/pyopenssl/compare/19.1.0...20.0.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index cae4accd..c612db5c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -73,7 +73,7 @@ pygments==2.6.1 # via sphinx pyjks==20.0.0 # via -r requirements.txt pyjwt==1.7.1 # via -r requirements.txt pynacl==1.3.0 # via -r requirements.txt, paramiko -pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests +pyopenssl==20.0.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests pyparsing==2.4.7 # via packaging pyrfc3339==1.1 # via -r requirements.txt, acme python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore diff --git a/requirements.txt b/requirements.txt index 68d5fdb6..20698495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -67,7 +67,7 @@ pycryptodomex==3.9.7 # via pyjks pyjks==20.0.0 # via -r requirements.in pyjwt==1.7.1 # via -r requirements.in pynacl==1.3.0 # via paramiko -pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests +pyopenssl==20.0.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests pyrfc3339==1.1 # via acme python-dateutil==2.8.1 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic From 817abb2ca81858a20c93055a6d83219122923677 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 30 Nov 2020 11:15:23 -0800 Subject: [PATCH 24/25] Removed for loop --- lemur/plugins/lemur_aws/elb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/elb.py b/lemur/plugins/lemur_aws/elb.py index 1da29276..cec0a803 100644 --- a/lemur/plugins/lemur_aws/elb.py +++ b/lemur/plugins/lemur_aws/elb.py @@ -160,8 +160,8 @@ def get_load_balancer_arn_from_endpoint(endpoint_name, **kwargs): try: client = kwargs.pop("client") elbs = client.describe_load_balancers(Names=[endpoint_name]) - for elb in elbs["LoadBalancers"]: - return elb["LoadBalancerArn"] + if "LoadBalancers" in elbs and elbs["LoadBalancers"]: + return elbs["LoadBalancers"][0]["LoadBalancerArn"] except Exception as e: # noqa metrics.send( From f22f29c05362d354bee016f0ab0b2883d57914bc Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 1 Dec 2020 18:48:36 -0800 Subject: [PATCH 25/25] Preselect ECCPRIME256V1 on UI for cert minting --- .../app/angular/certificates/certificate/options.tpl.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/static/app/angular/certificates/certificate/options.tpl.html b/lemur/static/app/angular/certificates/certificate/options.tpl.html index 2f28a4de..dbc7f777 100644 --- a/lemur/static/app/angular/certificates/certificate/options.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/options.tpl.html @@ -33,7 +33,7 @@
    + ng-init="certificate.keyType = 'ECCPRIME256V1'">