From e64e2a41d512f9b19f8ffb2ecdf649841a16569b Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 16:36:59 +0200 Subject: [PATCH 001/226] Add update_options to authorities service --- lemur/authorities/service.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index c70c6fc5..c734f408 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -39,6 +39,21 @@ def update(authority_id, description, owner, active, roles): return database.update(authority) +def update_options(authority_id, options): + """ + Update an authority with new options. + + :param authority_id: + :param options: the new options to be saved into the authority + :return: + """ + + authority = get(authority_id) + + authority.options = options + + return database.update(authority) + def mint(**kwargs): """ Creates the authority based on the plugin provided. From 898b5da6613294403da6683f20c45abe3f4bd7f3 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 16:38:38 +0200 Subject: [PATCH 002/226] Add store_account option to acme plugin --- lemur/plugins/lemur_acme/plugin.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 16d61a0f..ec4a5b84 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -240,6 +240,7 @@ class AcmeHandler(object): existing_regr = options.get("acme_regr", current_app.config.get("ACME_REGR")) if existing_key and existing_regr: + current_app.logger.debug("Reusing existing ACME account") # Reuse the same account for each certificate issuance key = jose.JWK.json_loads(existing_key) regr = messages.RegistrationResource.json_loads(existing_regr) @@ -253,6 +254,7 @@ class AcmeHandler(object): # Create an account for each certificate issuance key = jose.JWKRSA(key=generate_private_key("RSA2048")) + current_app.logger.debug("Creating a new ACME account") current_app.logger.debug( "Connecting with directory at {0}".format(directory_url) ) @@ -447,6 +449,13 @@ class ACMEIssuerPlugin(IssuerPlugin): "validation": "/^-----BEGIN CERTIFICATE-----/", "helpMessage": "Certificate to use", }, + { + "name": "store_account", + "type": "bool", + "required": False, + "helpMessage": "Disable to create a new account for each ACME request", + "default": True, + } ] def __init__(self, *args, **kwargs): From eed628dbab1d423cd08ff99027957b06368039ae Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 16:38:57 +0200 Subject: [PATCH 003/226] Implement storage of acme account --- lemur/plugins/lemur_acme/plugin.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index ec4a5b84..b77b1765 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -32,6 +32,7 @@ from lemur.extensions import metrics, sentry from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns +from lemur.authorities import service as authorities_service from retrying import retry @@ -264,6 +265,27 @@ class AcmeHandler(object): registration = client.new_account_and_tos( messages.NewRegistration.from_data(email=email) ) + + # if store_account is checked, add the private_key and registration resources to the options + if options['store_account']: + new_options = json.loads(authority.options) + # the key returned by fields_to_partial_json is missing the key type, so we add it manually + key_dict = key.fields_to_partial_json() + key_dict["kty"] = "RSA" + acme_private_key = { + "name": "acme_private_key", + "value": json.dumps(key_dict) + } + new_options.append(acme_private_key) + + acme_regr = { + "name": "acme_regr", + "value": json.dumps({"body": {}, "uri": registration.uri}) + } + new_options.append(acme_regr) + + authorities_service.update_options(authority.id, options=json.dumps(new_options)) + current_app.logger.debug("Connected: {0}".format(registration.uri)) return client, registration From 7e6fb740b3bf843a65730c60f813e759c37fc217 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 16:57:48 +0200 Subject: [PATCH 004/226] Fix flake8/linting errors --- lemur/authorities/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index c734f408..0913e629 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -54,6 +54,7 @@ def update_options(authority_id, options): return database.update(authority) + def mint(**kwargs): """ Creates the authority based on the plugin provided. From e0708410d0657da42f6acb08ddd3e9a3ecd05c3b Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 23 Sep 2020 17:12:49 +0200 Subject: [PATCH 005/226] Add store_account value to options in test_setup_acme_client_success --- lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 8320a2de..0356175c 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -169,7 +169,8 @@ class TestAcme(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_setup_acme_client_success(self, mock_current_app, mock_acme): mock_authority = Mock() - mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ + '{"name": "store_account", "value": false}] ' mock_client = Mock() mock_registration = Mock() mock_registration.uri = "http://test.com" From bf66de0bfd1a3cd8506a54e5d01304a388cbeb3b Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 12:59:52 +0200 Subject: [PATCH 006/226] Add Test for saving the accound details --- lemur/plugins/lemur_acme/tests/test_acme.py | 32 ++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 0356175c..b8d97489 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -165,12 +165,42 @@ class TestAcme(unittest.TestCase): with self.assertRaises(Exception): self.acme.setup_acme_client(mock_authority) + @patch("lemur.plugins.lemur_acme.plugin.jose.JWKRSA.fields_to_partial_json") + @patch("lemur.plugins.lemur_acme.plugin.authorities_service") + @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_setup_acme_client_success_store_new_account(self, mock_current_app, mock_acme, mock_authorities_service, + mock_key_generation): + mock_authority = Mock() + mock_authority.id = 2 + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ + '{"name": "store_account", "value": true}]' + mock_client = Mock() + mock_registration = Mock() + mock_registration.uri = "http://test.com" + mock_client.register = mock_registration + mock_client.agree_to_tos = Mock(return_value=True) + mock_client.new_account_and_tos.return_value = mock_registration + mock_acme.return_value = mock_client + mock_current_app.config = {} + + mock_key_generation.return_value = {"n": "PwIOkViO"} + + mock_authorities_service.update_options = Mock(return_value=True) + + self.acme.setup_acme_client(mock_authority) + + mock_authorities_service.update_options.assert_called_with(2, options='[{"name": "mock_name", "value": "mock_value"}, ' + '{"name": "store_account", "value": true}, ' + '{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, ' + '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]') + @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_setup_acme_client_success(self, mock_current_app, mock_acme): mock_authority = Mock() mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ - '{"name": "store_account", "value": false}] ' + '{"name": "store_account", "value": false}]' mock_client = Mock() mock_registration = Mock() mock_registration.uri = "http://test.com" From 9abd3e97e7f4d90276a7ba78ca3c66792afdbb21 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 13:21:34 +0200 Subject: [PATCH 007/226] Add test loading acme account from authority --- lemur/plugins/lemur_acme/tests/test_acme.py | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index b8d97489..8cd4783f 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,8 +1,10 @@ import unittest from unittest.mock import patch, Mock +import josepy as jose from cryptography.x509 import DNSName from lemur.plugins.lemur_acme import plugin +from lemur.common.utils import generate_private_key from mock import MagicMock @@ -165,6 +167,28 @@ class TestAcme(unittest.TestCase): with self.assertRaises(Exception): self.acme.setup_acme_client(mock_authority) + @patch("lemur.plugins.lemur_acme.plugin.jose.JWK.json_loads") + @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_setup_acme_client_success_load_account_from_authority(self, mock_current_app, mock_acme, mock_key_json_load): + mock_authority = Mock() + mock_authority.id = 2 + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ + '{"name": "store_account", "value": true},' \ + '{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, ' \ + '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]' + mock_client = Mock() + mock_acme.return_value = mock_client + mock_current_app.config = {} + + mock_key_json_load.return_value = jose.JWKRSA(key=generate_private_key("RSA2048")) + + result_client, result_registration = self.acme.setup_acme_client(mock_authority) + + mock_acme.new_account_and_tos.assert_not_called() + assert result_client + assert not result_registration + @patch("lemur.plugins.lemur_acme.plugin.jose.JWKRSA.fields_to_partial_json") @patch("lemur.plugins.lemur_acme.plugin.authorities_service") @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") From 835339694081258f9e815a5b99d134a33ad55cba Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 13:42:16 +0200 Subject: [PATCH 008/226] Improve tests --- lemur/plugins/lemur_acme/tests/test_acme.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 8cd4783f..ab246563 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -219,9 +219,10 @@ class TestAcme(unittest.TestCase): '{"name": "acme_private_key", "value": "{\\"n\\": \\"PwIOkViO\\", \\"kty\\": \\"RSA\\"}"}, ' '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]') + @patch("lemur.plugins.lemur_acme.plugin.authorities_service") @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_setup_acme_client_success(self, mock_current_app, mock_acme): + def test_setup_acme_client_success(self, mock_current_app, mock_acme, mock_authorities_service): mock_authority = Mock() mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ '{"name": "store_account", "value": false}]' @@ -233,6 +234,7 @@ class TestAcme(unittest.TestCase): mock_acme.return_value = mock_client mock_current_app.config = {} result_client, result_registration = self.acme.setup_acme_client(mock_authority) + mock_authorities_service.update_options.assert_not_called() assert result_client assert result_registration From 57534d86cd03e3d8585ffe52d4d5dd58a5b007fe Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 17:46:14 +0200 Subject: [PATCH 009/226] Disable account saving by default --- lemur/plugins/lemur_acme/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index b77b1765..8bc1485f 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -476,7 +476,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "type": "bool", "required": False, "helpMessage": "Disable to create a new account for each ACME request", - "default": True, + "default": False, } ] From 4f696abb5d6b14e80ab4dea966323525aa80850c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Oct 2020 20:02:27 -0700 Subject: [PATCH 010/226] adding util method to convert PKCS7 to pem --- lemur/common/utils.py | 19 +++++++++- lemur/tests/test_utils.py | 14 ++++++++ lemur/tests/vectors.py | 75 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 283d1eec..19b256e8 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -9,6 +9,7 @@ import random import re import string +import pem import sqlalchemy from cryptography import x509 @@ -16,7 +17,7 @@ from cryptography.exceptions import InvalidSignature, UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import rsa, ec, padding -from cryptography.hazmat.primitives.serialization import load_pem_private_key +from cryptography.hazmat.primitives.serialization import load_pem_private_key, Encoding, pkcs7 from flask_restful.reqparse import RequestParser from sqlalchemy import and_, func @@ -357,3 +358,19 @@ def find_matching_certificates_by_hash(cert, matching_certs): ): matching.append(c) return matching + + +def convert_pkcs7_bytes_to_pem(certs_pkcs7): + """ + Given a list of certificates in pkcs7 encoding (bytes), covert them into a list of PEM encoded files + :raises ValueError or ValidationError + :param certs_pkcs7: + :return: list of certs in PEM format + """ + + certificates = pkcs7.load_pem_pkcs7_certificates(certs_pkcs7) + certificates_pem = [] + for cert in certificates: + certificates_pem.append(pem.parse(cert.public_bytes(encoding=Encoding.PEM))[0]) + + return certificates_pem diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 162e53b0..f4be023b 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -10,6 +10,7 @@ from lemur.tests.vectors import ( ECDSA_SECP384r1_CERT, ECDSA_SECP384r1_CERT_STR, DSA_CERT, + CERT_CHAIN_PKCS7_PEM ) @@ -114,3 +115,16 @@ def test_get_key_type_from_certificate(): from lemur.common.utils import get_key_type_from_certificate assert (get_key_type_from_certificate(SAN_CERT_STR) == "RSA2048") assert (get_key_type_from_certificate(ECDSA_SECP384r1_CERT_STR) == "ECCSECP384R1") + + +def test_convert_pkcs7_bytes_to_pem(): + from lemur.common.utils import convert_pkcs7_bytes_to_pem + from lemur.common.utils import parse_certificate + cert_chain = convert_pkcs7_bytes_to_pem(CERT_CHAIN_PKCS7_PEM) + assert(len(cert_chain) == 3) + + leaf = cert_chain[1] + root = cert_chain[2] + + assert(parse_certificate("\n".join(str(root).splitlines())) == ROOTCA_CERT) + assert (parse_certificate("\n".join(str(leaf).splitlines())) == INTERMEDIATE_CERT) diff --git a/lemur/tests/vectors.py b/lemur/tests/vectors.py index 0768cdac..7a78818c 100644 --- a/lemur/tests/vectors.py +++ b/lemur/tests/vectors.py @@ -512,3 +512,78 @@ BglghkgBZQMEAwIDMAAwLQIVANubSNMSLt8plN9ZV3cp4pe3lMYCAhQPLLE7rTgm -----END CERTIFICATE----- """ DSA_CERT = parse_certificate(DSA_CERT_STR) + + +CERT_CHAIN_PKCS7_STR = """ +-----BEGIN PKCS7----- +MIIMfwYJKoZIhvcNAQcCoIIMcDCCDGwCAQExADALBgkqhkiG9w0BBwGgggxSMIIE +FjCCAv6gAwIBAgIQbIbX/Ap0Roqzf5HeN5akmzANBgkqhkiG9w0BAQsFADCBpDEq +MCgGA1UEAwwhTGVtdXJUcnVzdCBVbml0dGVzdHMgUm9vdCBDQSAyMDE4MSMwIQYD +VQQKDBpMZW11clRydXN0IEVudGVycHJpc2VzIEx0ZDEmMCQGA1UECwwdVW5pdHRl +c3RpbmcgT3BlcmF0aW9ucyBDZW50ZXIxCzAJBgNVBAYTAkVFMQwwCgYDVQQIDANO +L0ExDjAMBgNVBAcMBUVhcnRoMB4XDTE3MTIzMTIyMDAwMFoXDTQ3MTIzMTIyMDAw +MFowgaQxKjAoBgNVBAMMIUxlbXVyVHJ1c3QgVW5pdHRlc3RzIFJvb3QgQ0EgMjAx +ODEjMCEGA1UECgwaTGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsM +HVVuaXR0ZXN0aW5nIE9wZXJhdGlvbnMgQ2VudGVyMQswCQYDVQQGEwJFRTEMMAoG +A1UECAwDTi9BMQ4wDAYDVQQHDAVFYXJ0aDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL8laXtLXyM64t5dz2B9q+4VvOsChefBi2PlGudqxDuRN3l0Kmcf +un6x2Gng24pTlGdtmiTEWA0a2F8HRLv4YBWhuYleVeBPtf1fF1/SuYgkJOWT7S5q +k/od/tUOLHS0Y067st3FydnFQTKpAuYveEkxleFrMS8hX8cuEgbER+8ybiXKn4Gs +yM/om6lsTyBoaLp5yTAoQb4jAWDbiz1xcjPSkvH2lm7rLGtKoylCYwxRsMh2nZcR +r1OXVhYHXwpYHVB/jVAjy7PAWQ316hi6mpPYbBV+yfn2GUfGuytqyoXLEsrM3iEE +AkU0mJjQmYsCDM3r7ONHTM+UFEk47HCZJccCAwEAAaNCMEAwDwYDVR0TAQH/BAUw +AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFFL12SFeOTTDdGKsHKozeByG +HY6nMA0GCSqGSIb3DQEBCwUAA4IBAQAJfe0/uAHobkxth38dqrSFmTo+D5/TMlRt +3hdgjlah6sD2+/DObCyut/XhQWCgTNWyRi4xTKgLh5KSoeJ9EMkADGEgDkU2vjBg +5FmGZsxg6bqjxehK+2HvASJoTH8r41xmTioav7a2i3wNhaNSntw2QRTQBQEDOIzH +RpPDQ2quErjA8nSifE2xmAAr3g+FuookTTJuv37s2cS59zRYsg+WC3+TtPpRssvo +bJ6Xe2D4cCVjUmsqtFEztMgdqgmlcWyGdUKeXdi7CMoeTb4uO+9qRQq46wYWn7K1 +z+W0Kp5yhnnPAoOioAP4vjASDx3z3RnLaZvMmcO7YdCIwhE5oGV0MIIEGjCCAwKg +AwIBAgIRAJ96dbOdrkw/lSTGiwbaagwwDQYJKoZIhvcNAQELBQAwgaQxKjAoBgNV +BAMMIUxlbXVyVHJ1c3QgVW5pdHRlc3RzIFJvb3QgQ0EgMjAxODEjMCEGA1UECgwa +TGVtdXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsMHVVuaXR0ZXN0aW5n +IE9wZXJhdGlvbnMgQ2VudGVyMQswCQYDVQQGEwJFRTEMMAoGA1UECAwDTi9BMQ4w +DAYDVQQHDAVFYXJ0aDAeFw0xNzEyMzEyMjAwMDBaFw00NzEyMzEyMjAwMDBaMIGn +MS0wKwYDVQQDDCRMZW11clRydXN0IFVuaXR0ZXN0cyBDbGFzcyAxIENBIDIwMTgx +IzAhBgNVBAoMGkxlbXVyVHJ1c3QgRW50ZXJwcmlzZXMgTHRkMSYwJAYDVQQLDB1V +bml0dGVzdGluZyBPcGVyYXRpb25zIENlbnRlcjELMAkGA1UEBhMCRUUxDDAKBgNV +BAgMA04vQTEOMAwGA1UEBwwFRWFydGgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDR+qNdfNsLhGvgw3IgCQNakL2B9dpQtkVnvAXhdRZqJETm/tHLkGvO +NWTXAwGdoiKv6+0j3I5InUsW+wzUPewcfj+PLNu4mFMq8jH/gPhTElKiAztPRdm8 +QKchvrqiaU6uEbia8ClM6uPpIi8StxE1aJRYL03p0WeMJjJPrsl6eSSdpR4qL69G +Td1n5je9OuWAcn5utXXnt/jO4vNeFRjlGp/0n3JmTDd9w4vtAyY9UrdGgo37eBmi +6mXt5J9i//NenhaiOVU81RqxZM2Jt1kkg2WSjcqcIQfBEWp9StG46VmHLaL+9/v2 +XAV3tL1VilJGj6PoFMb4gY5MXthfGSiXAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBQstpQr0iMBVfv0lODIsMgT9+9o +ezANBgkqhkiG9w0BAQsFAAOCAQEASYQbv1Qwb5zES6Gb5LEhrAcH81ZB2uIpKd3K +i6AS4fLJVymMGkUs0RZjt39Ep4qX1zf0hn82Yh9YwRalrkgu+tzKrp0JgegNe6+g +yFRrJC0SIGA4zc3M02m/n4tdaouU2lp6jhmWruL3g25ZkgbQ8LO2zjpSMtblR2eu +vR2+bI7TepklyG71qx5y6/N8x5PT+hnTlleiZeE/ji9D96MZlpWB4kBihekWmxup +tED22z/tpQtac+hPBNgt8z1uFVEYN2rKEcCE7V6Qk7icS+M4Vb7M3D8kLyWDubs9 +Yy3l0EWjOXQXxEhTaKEm4gSuY/j+Y35bBVkA2Fcyuq7msiTgrzCCBBYwggL+oAMC +AQICEGyG1/wKdEaKs3+R3jeWpJswDQYJKoZIhvcNAQELBQAwgaQxKjAoBgNVBAMM +IUxlbXVyVHJ1c3QgVW5pdHRlc3RzIFJvb3QgQ0EgMjAxODEjMCEGA1UECgwaTGVt +dXJUcnVzdCBFbnRlcnByaXNlcyBMdGQxJjAkBgNVBAsMHVVuaXR0ZXN0aW5nIE9w +ZXJhdGlvbnMgQ2VudGVyMQswCQYDVQQGEwJFRTEMMAoGA1UECAwDTi9BMQ4wDAYD +VQQHDAVFYXJ0aDAeFw0xNzEyMzEyMjAwMDBaFw00NzEyMzEyMjAwMDBaMIGkMSow +KAYDVQQDDCFMZW11clRydXN0IFVuaXR0ZXN0cyBSb290IENBIDIwMTgxIzAhBgNV +BAoMGkxlbXVyVHJ1c3QgRW50ZXJwcmlzZXMgTHRkMSYwJAYDVQQLDB1Vbml0dGVz +dGluZyBPcGVyYXRpb25zIENlbnRlcjELMAkGA1UEBhMCRUUxDDAKBgNVBAgMA04v +QTEOMAwGA1UEBwwFRWFydGgwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQC/JWl7S18jOuLeXc9gfavuFbzrAoXnwYtj5RrnasQ7kTd5dCpnH7p+sdhp4NuK +U5RnbZokxFgNGthfB0S7+GAVobmJXlXgT7X9Xxdf0rmIJCTlk+0uapP6Hf7VDix0 +tGNOu7LdxcnZxUEyqQLmL3hJMZXhazEvIV/HLhIGxEfvMm4lyp+BrMjP6JupbE8g +aGi6eckwKEG+IwFg24s9cXIz0pLx9pZu6yxrSqMpQmMMUbDIdp2XEa9Tl1YWB18K +WB1Qf41QI8uzwFkN9eoYupqT2GwVfsn59hlHxrsrasqFyxLKzN4hBAJFNJiY0JmL +AgzN6+zjR0zPlBRJOOxwmSXHAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYD +VR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRS9dkhXjk0w3RirByqM3gchh2OpzANBgkq +hkiG9w0BAQsFAAOCAQEACX3tP7gB6G5MbYd/Haq0hZk6Pg+f0zJUbd4XYI5WoerA +9vvwzmwsrrf14UFgoEzVskYuMUyoC4eSkqHifRDJAAxhIA5FNr4wYORZhmbMYOm6 +o8XoSvth7wEiaEx/K+NcZk4qGr+2tot8DYWjUp7cNkEU0AUBAziMx0aTw0NqrhK4 +wPJ0onxNsZgAK94PhbqKJE0ybr9+7NnEufc0WLIPlgt/k7T6UbLL6Gyel3tg+HAl +Y1JrKrRRM7TIHaoJpXFshnVCnl3YuwjKHk2+LjvvakUKuOsGFp+ytc/ltCqecoZ5 +zwKDoqAD+L4wEg8d890Zy2mbzJnDu2HQiMIROaBldKEAMQA= +-----END PKCS7----- +""" + +CERT_CHAIN_PKCS7_PEM = CERT_CHAIN_PKCS7_STR.encode('utf-8') From 1a270cd315f6b298870cc0adb29e1bca613de49f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 7 Oct 2020 20:06:20 -0700 Subject: [PATCH 011/226] switching from static DigiCert ICAs to dynamic ones to support: https://knowledge.digicert.com/alerts/DigiCert-ICA-Update.html --- lemur/plugins/lemur_digicert/plugin.py | 28 +++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 3948acbb..fd94de57 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -21,7 +21,7 @@ import requests import sys from cryptography import x509 from flask import current_app, g -from lemur.common.utils import validate_conf +from lemur.common.utils import validate_conf, convert_pkcs7_bytes_to_pem from lemur.extensions import metrics from lemur.plugins import lemur_digicert as digicert from lemur.plugins.bases import IssuerPlugin, SourcePlugin @@ -235,15 +235,18 @@ def get_certificate_id(session, base_url, order_id): @retry(stop_max_attempt_number=10, wait_fixed=10000) def get_cis_certificate(session, base_url, order_id): - """Retrieve certificate order id from Digicert API.""" - certificate_url = "{0}/platform/cis/certificate/{1}".format(base_url, order_id) - session.headers.update({"Accept": "application/x-pem-file"}) + """Retrieve certificate order id from Digicert API, including the chain""" + certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id) + session.headers.update({"Accept": "application/x-pkcs7-certificates"}) response = session.get(certificate_url) if response.status_code == 404: raise Exception("Order not in issued state.") - return response.content + cert_chain_pem = convert_pkcs7_bytes_to_pem(response.content) + if len(cert_chain_pem) < 3: + raise Exception("Missing the certificate chain") + return cert_chain_pem class DigiCertSourcePlugin(SourcePlugin): @@ -552,22 +555,15 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): data = handle_cis_response(response) # retrieve certificate - certificate_pem = get_cis_certificate(self.session, base_url, data["id"]) + certificate_chain_pem = get_cis_certificate(self.session, base_url, data["id"]) self.session.headers.pop("Accept") - end_entity = pem.parse(certificate_pem)[0] + end_entity = certificate_chain_pem[0] + intermediate = certificate_chain_pem[1] - if "ECC" in issuer_options["key_type"]: - return ( - "\n".join(str(end_entity).splitlines()), - current_app.config.get("DIGICERT_ECC_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name), - data["id"], - ) - - # By default return RSA return ( "\n".join(str(end_entity).splitlines()), - current_app.config.get("DIGICERT_CIS_INTERMEDIATES", {}).get(issuer_options['authority'].name), + "\n".join(str(intermediate).splitlines()), data["id"], ) From 8928e043858832ff5926396662830b6873fd9af3 Mon Sep 17 00:00:00 2001 From: sayali Date: Thu, 8 Oct 2020 11:38:39 -0700 Subject: [PATCH 012/226] Fix disable notify --- lemur/certificates/schemas.py | 2 +- lemur/certificates/service.py | 11 ++ lemur/certificates/views.py | 107 ++++++++++++++++++ .../app/angular/certificates/services.js | 2 +- lemur/tests/test_certificates.py | 26 +++-- 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index a360140e..5da342e5 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -194,7 +194,7 @@ class CertificateEditInputSchema(CertificateSchema): :param data: :return: """ - if data["owner"]: + if data.get("owner"): notification_name = "DEFAULT_{0}".format( data["owner"].split("@")[0].upper() ) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index df73487d..b676cffb 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -256,6 +256,17 @@ def update(cert_id, **kwargs): return database.update(cert) +def update_notify(cert, notify_flag): + """ + Toggle notification value which is a boolean + :param notify_flag: new notify value + :param cert: Certificate object to be updated + :return: + """ + cert.notify = notify_flag + return database.update(cert) + + def create_certificate_roles(**kwargs): # create an role for the owner and assign it owner_role = role_service.get_by_name(kwargs["owner"]) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 51f7f615..0eaba4e5 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -888,6 +888,110 @@ class Certificates(AuthenticatedResource): log_service.create(g.current_user, "update_cert", certificate=cert) return cert + @validate_schema(certificate_edit_input_schema, certificate_output_schema) + def post(self, certificate_id, data=None): + """ + .. http:post:: /certificates/1/update/notify + + Update certificate notification + + **Example request**: + + .. sourcecode:: http + + POST /certificates/1/update/notify HTTP/1.1 + Host: example.com + Accept: application/json, text/javascript + + { + "notify": false + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: text/javascript + + { + "status": null, + "cn": "*.test.example.net", + "chain": "", + "authority": { + "active": true, + "owner": "secure@example.com", + "id": 1, + "description": "verisign test authority", + "name": "verisign" + }, + "owner": "joe@example.com", + "serial": "82311058732025924142789179368889309156", + "id": 2288, + "issuer": "SymantecCorporation", + "dateCreated": "2016-06-03T06:09:42.133769+00:00", + "notBefore": "2016-06-03T00:00:00+00:00", + "notAfter": "2018-01-12T23:59:59+00:00", + "destinations": [], + "bits": 2048, + "body": "-----BEGIN CERTIFICATE-----...", + "description": null, + "deleted": null, + "notify": false, + "notifications": [{ + "id": 1 + }] + "signingAlgorithm": "sha256", + "user": { + "username": "jane", + "active": true, + "email": "jane@example.com", + "id": 2 + }, + "active": true, + "domains": [{ + "sensitive": false, + "id": 1090, + "name": "*.test.example.net" + }], + "replaces": [], + "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112", + "roles": [{ + "id": 464, + "description": "This is a google group based role created by Lemur", + "name": "joe@example.com" + }], + "rotation": true, + "rotationPolicy": {"name": "default"}, + "san": null + } + + :reqheader Authorization: OAuth token to authenticate + :statuscode 200: no error + :statuscode 403: unauthenticated + + """ + cert = service.get(certificate_id) + + if not cert: + return dict(message="Cannot find specified certificate"), 404 + + # allow creators + if g.current_user != cert.user: + owner_role = role_service.get_by_name(cert.owner) + permission = CertificatePermission(owner_role, [x.name for x in cert.roles]) + + if not permission.can(): + return ( + dict(message="You are not authorized to update this certificate"), + 403, + ) + + cert = service.update_notify(cert, data.get("notify")) + log_service.create(g.current_user, "update_cert", certificate=cert) + return cert + def delete(self, certificate_id, data=None): """ .. http:delete:: /certificates/1 @@ -1354,6 +1458,9 @@ api.add_resource( api.add_resource( Certificates, "/certificates/", endpoint="certificate" ) +api.add_resource( + Certificates, "/certificates//update/notify", endpoint="certificateUpdateNotify" +) api.add_resource(CertificatesStats, "/certificates/stats", endpoint="certificateStats") api.add_resource( CertificatesUpload, "/certificates/upload", endpoint="certificateUpload" diff --git a/lemur/static/app/angular/certificates/services.js b/lemur/static/app/angular/certificates/services.js index 0c8eb7cc..ce88ccb3 100644 --- a/lemur/static/app/angular/certificates/services.js +++ b/lemur/static/app/angular/certificates/services.js @@ -301,7 +301,7 @@ angular.module('lemur') }; CertificateService.updateNotify = function (certificate) { - return certificate.put(); + return certificate.post(); }; CertificateService.export = function (certificate) { diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index a0a3b54e..c19e3120 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -922,19 +922,25 @@ def test_certificate_get_body(client): @pytest.mark.parametrize( "token,status", [ - (VALID_USER_HEADER_TOKEN, 405), - (VALID_ADMIN_HEADER_TOKEN, 405), - (VALID_ADMIN_API_TOKEN, 405), - ("", 405), + (VALID_USER_HEADER_TOKEN, 403), + (VALID_ADMIN_HEADER_TOKEN, 200), + (VALID_ADMIN_API_TOKEN, 200), + ("", 401), ], ) -def test_certificate_post(client, token, status): - assert ( - client.post( - api.url_for(Certificates, certificate_id=1), data={}, headers=token - ).status_code - == status +def test_certificate_post_update_notify(client, certificate, token, status): + # negate the current notify flag and pass it to update POST call to flip the notify + toggled_notify = not certificate.notify + + response = client.post( + api.url_for(Certificates, certificate_id=certificate.id), + data=json.dumps({"notify": toggled_notify}), + headers=token ) + + assert response.status_code == status + if status == 200: + assert response.json.get("notify") == toggled_notify @pytest.mark.parametrize( From d5ce38bf71b210f56ab7f6b5d5ba5e9bcff9144d Mon Sep 17 00:00:00 2001 From: sayali Date: Thu, 8 Oct 2020 12:50:30 -0700 Subject: [PATCH 013/226] lint error fix - remove whitespace --- lemur/tests/test_certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index c19e3120..8403461b 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -937,7 +937,7 @@ def test_certificate_post_update_notify(client, certificate, token, status): data=json.dumps({"notify": toggled_notify}), headers=token ) - + assert response.status_code == status if status == 200: assert response.json.get("notify") == toggled_notify From a6a4f458e029f9a54cac57ecf284327d137406b5 Mon Sep 17 00:00:00 2001 From: sirferl Date: Fri, 9 Oct 2020 11:35:04 +0200 Subject: [PATCH 014/226] added Tests and removed problems in test-setup --- lemur/plugins/lemur_entrust/tests/test_entrust.py | 14 ++++++++++---- lemur/tests/conf.py | 6 +++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_entrust/tests/test_entrust.py b/lemur/plugins/lemur_entrust/tests/test_entrust.py index b1cd4c83..b3f2e0c4 100644 --- a/lemur/plugins/lemur_entrust/tests/test_entrust.py +++ b/lemur/plugins/lemur_entrust/tests/test_entrust.py @@ -3,6 +3,7 @@ from unittest.mock import patch, Mock import arrow from cryptography import x509 from lemur.plugins.lemur_entrust import plugin +from freezegun import freeze_time def config_mock(*args): @@ -20,12 +21,17 @@ def config_mock(*args): } return values[args[0]] +@patch("lemur.plugins.lemur_digicert.plugin.current_app") +def test_determine_end_date(mock_current_app): + with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime): + assert arrow.get(2017, 12, 3).format('YYYY-MM-DD') == plugin.determine_end_date(0) # 1 year + 1 month + assert arrow.get(2017, 3, 5).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2017, 3, 5)) + assert arrow.get(2017, 12, 3).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2020, 5, 7)) @patch("lemur.plugins.lemur_entrust.plugin.current_app") def test_process_options(mock_current_app, authority): mock_current_app.config.get = Mock(side_effect=config_mock) - plugin.determine_end_date = Mock(return_value=arrow.get(2020, 10, 7).format('YYYY-MM-DD')) - + plugin.determine_end_date = Mock(return_value=arrow.get(2017, 11, 5).format('YYYY-MM-DD')) authority.name = "Entrust" names = [u"one.example.com", u"two.example.com", u"three.example.com"] options = { @@ -35,7 +41,7 @@ def test_process_options(mock_current_app, authority): "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}}, "organization": "Example, Inc.", "organizational_unit": "Example Org", - "validity_end": arrow.get(2020, 10, 7), + "validity_end": arrow.utcnow().shift(years=1, months=+1), "authority": authority, } @@ -43,7 +49,7 @@ def test_process_options(mock_current_app, authority): "signingAlg": "SHA-2", "eku": "SERVER_AND_CLIENT_AUTH", "certType": "ADVANTAGE_SSL", - "certExpiryDate": arrow.get(2020, 10, 7).format('YYYY-MM-DD'), + "certExpiryDate": arrow.get(2017, 11, 5).format('YYYY-MM-DD'), "tracking": { "requesterName": mock_current_app.config.get("ENTRUST_NAME"), "requesterEmail": mock_current_app.config.get("ENTRUST_EMAIL"), diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index bf033421..0a288327 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -32,9 +32,9 @@ LEMUR_ENCRYPTION_KEYS = "o61sBLNBSGtAckngtNrfVNd8xy8Hp9LBGDstTbMbqCY=" # List of domain regular expressions that non-admin users can issue LEMUR_WHITELISTED_DOMAINS = [ - "^[a-zA-Z0-9-]+\.example\.com$", - "^[a-zA-Z0-9-]+\.example\.org$", - "^example\d+\.long\.com$", + r"^[a-zA-Z0-9-]+\.example\.com$", + r"^[a-zA-Z0-9-]+\.example\.org$", + r"^example\d+\.long\.com$", ] # Mail Server From d43e240a2a2be7ab879e696d026d630232c1544d Mon Sep 17 00:00:00 2001 From: sirferl Date: Fri, 9 Oct 2020 11:41:44 +0200 Subject: [PATCH 015/226] dded ELIF at determine_end_date, becuase of error. --- lemur/plugins/lemur_entrust/plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 9b7848ed..515e2400 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -34,8 +34,7 @@ def determine_end_date(end_date): if not end_date: end_date = max_validity_end - - if end_date > max_validity_end: + elif end_date > max_validity_end: end_date = max_validity_end return end_date.format('YYYY-MM-DD') From 5a968ffe6304dbaedd5a254a77c6c16c632765b5 Mon Sep 17 00:00:00 2001 From: sirferl Date: Fri, 9 Oct 2020 12:05:57 +0200 Subject: [PATCH 016/226] Lint errors --- lemur/plugins/lemur_entrust/tests/test_entrust.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemur/plugins/lemur_entrust/tests/test_entrust.py b/lemur/plugins/lemur_entrust/tests/test_entrust.py index b3f2e0c4..354e204e 100644 --- a/lemur/plugins/lemur_entrust/tests/test_entrust.py +++ b/lemur/plugins/lemur_entrust/tests/test_entrust.py @@ -21,6 +21,7 @@ def config_mock(*args): } return values[args[0]] + @patch("lemur.plugins.lemur_digicert.plugin.current_app") def test_determine_end_date(mock_current_app): with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime): @@ -28,6 +29,7 @@ def test_determine_end_date(mock_current_app): assert arrow.get(2017, 3, 5).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2017, 3, 5)) assert arrow.get(2017, 12, 3).format('YYYY-MM-DD') == plugin.determine_end_date(arrow.get(2020, 5, 7)) + @patch("lemur.plugins.lemur_entrust.plugin.current_app") def test_process_options(mock_current_app, authority): mock_current_app.config.get = Mock(side_effect=config_mock) From 42e9b8b62749adbabc6ec3ea7dd4395d7406107a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 9 Oct 2020 15:40:25 -0700 Subject: [PATCH 017/226] removing the intermediary from being optional --- lemur/plugins/lemur_digicert/plugin.py | 2 -- lemur/tests/conf.py | 1 - 2 files changed, 3 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index fd94de57..cf01c9d1 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -450,7 +450,6 @@ class DigiCertCISSourcePlugin(SourcePlugin): "DIGICERT_CIS_API_KEY", "DIGICERT_CIS_URL", "DIGICERT_CIS_ROOTS", - "DIGICERT_CIS_INTERMEDIATES", "DIGICERT_CIS_PROFILE_NAMES", ] validate_conf(current_app, required_vars) @@ -525,7 +524,6 @@ class DigiCertCISIssuerPlugin(IssuerPlugin): "DIGICERT_CIS_API_KEY", "DIGICERT_CIS_URL", "DIGICERT_CIS_ROOTS", - "DIGICERT_CIS_INTERMEDIATES", "DIGICERT_CIS_PROFILE_NAMES", ] diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index f1019d04..fc6bda98 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -99,7 +99,6 @@ DIGICERT_CIS_URL = "mock://www.digicert.com" DIGICERT_CIS_PROFILE_NAMES = {"sha2-rsa-ecc-root": "ssl_plus"} DIGICERT_CIS_API_KEY = "api-key" DIGICERT_CIS_ROOTS = {"root": "ROOT"} -DIGICERT_CIS_INTERMEDIATES = {"inter": "INTERMEDIATE_CA_CERT"} VERISIGN_URL = "http://example.com" VERISIGN_PEM_PATH = "~/" From d52e0d4e09c742e14fc2dbc3d874f2132029e166 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 9 Oct 2020 16:55:19 -0700 Subject: [PATCH 018/226] Certificate edit: update role and notification with owner change --- lemur/certificates/schemas.py | 32 ++++++++++++++++++++++++++++++-- lemur/certificates/service.py | 23 ++++++++++++++--------- lemur/certificates/views.py | 4 ++++ lemur/roles/service.py | 8 ++++++++ lemur/tests/test_certificates.py | 6 ++++++ 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 5da342e5..72ffb063 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -23,6 +23,7 @@ from lemur.domains.schemas import DomainNestedOutputSchema from lemur.notifications import service as notification_service from lemur.notifications.schemas import NotificationNestedOutputSchema from lemur.policies.schemas import RotationPolicyNestedOutputSchema +from lemur.roles import service as roles_service from lemur.roles.schemas import RoleNestedOutputSchema from lemur.schemas import ( AssociatedAuthoritySchema, @@ -184,13 +185,33 @@ class CertificateEditInputSchema(CertificateSchema): data["replaces"] = data[ "replacements" ] # TODO remove when field is deprecated + + if data.get("owner"): + # Check if role already exists. This avoids adding duplicate role. + if data.get("roles") and any(r.get("name") == data["owner"] for r in data["roles"]): + return data + + # Add required role + owner_role = roles_service.get_or_create( + data["owner"], + description="Auto generated role based on owner: {0}".format(data["owner"]) + ) + + # Put role info in correct format using RoleNestedOutputSchema + owner_role_dict = RoleNestedOutputSchema().dump(owner_role).data + if data.get("roles"): + data["roles"].append(owner_role_dict) + else: + data["roles"] = [owner_role_dict] + return data @post_load def enforce_notifications(self, data): """ - Ensures that when an owner changes, default notifications are added for the new owner. - Old owner notifications are retained unless explicitly removed. + Add default notification for current owner if none exist. + This ensures that the default notifications are added in the even of owner change. + Old owner notifications are retained unless explicitly removed later in the code path. :param data: :return: """ @@ -198,11 +219,18 @@ class CertificateEditInputSchema(CertificateSchema): notification_name = "DEFAULT_{0}".format( data["owner"].split("@")[0].upper() ) + + # Even if one default role exists, return + # This allows a User to remove unwanted default notification for current owner + if any(n.label.startswith(notification_name) for n in data["notifications"]): + return data + data[ "notifications" ] += notification_service.create_default_expiration_notifications( notification_name, [data["owner"]] ) + return data diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index b676cffb..812ec101 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -256,6 +256,14 @@ def update(cert_id, **kwargs): return database.update(cert) +def cleanup_owner_roles_notification(owner_name, kwargs): + kwargs["roles"] = [r for r in kwargs["roles"] if r.name != owner_name] + notification_prefix = "DEFAULT_{0}".format( + owner_name.split("@")[0].upper() + ) + kwargs["notifications"] = [n for n in kwargs["notifications"] if not n.label.startswith(notification_prefix)] + + def update_notify(cert, notify_flag): """ Toggle notification value which is a boolean @@ -268,16 +276,13 @@ def update_notify(cert, notify_flag): def create_certificate_roles(**kwargs): - # create an role for the owner and assign it - owner_role = role_service.get_by_name(kwargs["owner"]) - - if not owner_role: - owner_role = role_service.create( - kwargs["owner"], - description="Auto generated role based on owner: {0}".format( - kwargs["owner"] - ), + # create a role for the owner and assign it + owner_role = role_service.get_or_create( + kwargs["owner"], + description="Auto generated role based on owner: {0}".format( + kwargs["owner"] ) + ) # ensure that the authority's owner is also associated with the certificate if kwargs.get("authority"): diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 0eaba4e5..18746636 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -884,6 +884,10 @@ class Certificates(AuthenticatedResource): 400, ) + # if owner is changed, remove all notifications and roles associated with old owner + if cert.owner != data["owner"]: + service.cleanup_owner_roles_notification(cert.owner, data) + cert = service.update(certificate_id, **data) log_service.create(g.current_user, "update_cert", certificate=cert) return cert diff --git a/lemur/roles/service.py b/lemur/roles/service.py index 51597d6e..fa4c9c97 100644 --- a/lemur/roles/service.py +++ b/lemur/roles/service.py @@ -128,3 +128,11 @@ def render(args): query = database.filter(query, Role, terms) return database.sort_and_page(query, Role, args) + + +def get_or_create(role_name, description): + role = get_by_name(role_name) + if not role: + role = create(name=role_name, description=description) + + return role diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 8403461b..4edc485f 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -180,7 +180,10 @@ def test_certificate_edit_schema(session): input_data = {"owner": "bob@example.com"} data, errors = CertificateEditInputSchema().load(input_data) + + assert not errors assert len(data["notifications"]) == 3 + assert data["roles"][0].name == input_data["owner"] def test_authority_key_identifier_schema(): @@ -970,6 +973,9 @@ def test_certificate_put_with_data(client, certificate, issuer_plugin): headers=VALID_ADMIN_HEADER_TOKEN, ) assert resp.status_code == 200 + assert len(certificate.notifications) == 3 + assert certificate.roles[0].name == "bob@example.com" + assert certificate.notify @pytest.mark.parametrize( From fb4df8865bc703f934bdc5b7db4eb202db585cde Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 9 Oct 2020 17:57:35 -0700 Subject: [PATCH 019/226] Formatting changes and typo --- lemur/certificates/schemas.py | 4 ++-- lemur/certificates/service.py | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 72ffb063..688d6ba4 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -194,7 +194,7 @@ class CertificateEditInputSchema(CertificateSchema): # Add required role owner_role = roles_service.get_or_create( data["owner"], - description="Auto generated role based on owner: {0}".format(data["owner"]) + description=f"Auto generated role based on owner: {data['owner']}" ) # Put role info in correct format using RoleNestedOutputSchema @@ -210,7 +210,7 @@ class CertificateEditInputSchema(CertificateSchema): def enforce_notifications(self, data): """ Add default notification for current owner if none exist. - This ensures that the default notifications are added in the even of owner change. + This ensures that the default notifications are added in the event of owner change. Old owner notifications are retained unless explicitly removed later in the code path. :param data: :return: diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 812ec101..6d1bd2ac 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -258,9 +258,7 @@ def update(cert_id, **kwargs): def cleanup_owner_roles_notification(owner_name, kwargs): kwargs["roles"] = [r for r in kwargs["roles"] if r.name != owner_name] - notification_prefix = "DEFAULT_{0}".format( - owner_name.split("@")[0].upper() - ) + notification_prefix = f"DEFAULT_{owner_name.split('@')[0].upper()}" kwargs["notifications"] = [n for n in kwargs["notifications"] if not n.label.startswith(notification_prefix)] @@ -279,9 +277,7 @@ def create_certificate_roles(**kwargs): # create a role for the owner and assign it owner_role = role_service.get_or_create( kwargs["owner"], - description="Auto generated role based on owner: {0}".format( - kwargs["owner"] - ) + description=f"Auto generated role based on owner: {kwargs['owner']}" ) # ensure that the authority's owner is also associated with the certificate From 770339f94cfa0be2ca45a8cf19d86ad673e7c79f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 9 Oct 2020 18:04:16 -0700 Subject: [PATCH 020/226] cleaning up outdated phrases --- docker/src/lemur.conf.py | 2 +- docs/administration.rst | 2 +- lemur/common/validators.py | 8 ++++---- lemur/manage.py | 2 +- lemur/tests/conf.py | 2 +- lemur/tests/test_certificates.py | 16 ++++++++-------- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/src/lemur.conf.py b/docker/src/lemur.conf.py index 4cb3ae0c..4bcaeef9 100644 --- a/docker/src/lemur.conf.py +++ b/docker/src/lemur.conf.py @@ -24,7 +24,7 @@ LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET', LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS', base64.b64encode(get_random_secret(32).encode('utf8')))) -LEMUR_WHITELISTED_DOMAINS = [] +LEMUR_ALLOWED_DOMAINS = [] LEMUR_EMAIL = '' LEMUR_SECURITY_TEAM_EMAIL = [] diff --git a/docs/administration.rst b/docs/administration.rst index f44ad1a3..00da0c8a 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -100,7 +100,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c Specifies whether to allow certificates created by Lemur to expire on weekends. Default is True. -.. data:: LEMUR_WHITELISTED_DOMAINS +.. data:: LEMUR_ALLOWED_DOMAINS :noindex: List of regular expressions for domain restrictions; if the list is not empty, normal users can only issue diff --git a/lemur/common/validators.py b/lemur/common/validators.py index e1dfe3c1..e004a971 100644 --- a/lemur/common/validators.py +++ b/lemur/common/validators.py @@ -22,7 +22,7 @@ def common_name(value): def sensitive_domain(domain): """ - Checks if user has the admin role, the domain does not match sensitive domains and whitelisted domain patterns. + Checks if user has the admin role, the domain does not match sensitive domains and allowed domain patterns. :param domain: domain name (str) :return: """ @@ -30,10 +30,10 @@ def sensitive_domain(domain): # User has permission, no need to check anything return - whitelist = current_app.config.get("LEMUR_WHITELISTED_DOMAINS", []) - if whitelist and not any(re.match(pattern, domain) for pattern in whitelist): + allowlist = current_app.config.get("LEMUR_ALLOWED_DOMAINS", []) + if allowlist and not any(re.match(pattern, domain) for pattern in allowlist): raise ValidationError( - "Domain {0} does not match whitelisted domain patterns. " + "Domain {0} does not match allowed domain patterns. " "Contact an administrator to issue the certificate.".format(domain) ) diff --git a/lemur/manage.py b/lemur/manage.py index 2fbbe893..e53f8bd6 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -95,7 +95,7 @@ LEMUR_TOKEN_SECRET = '{secret_token}' LEMUR_ENCRYPTION_KEYS = '{encryption_key}' # List of domain regular expressions that non-admin users can issue -LEMUR_WHITELISTED_DOMAINS = [] +LEMUR_ALLOWED_DOMAINS = [] # Mail Server diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index 8255e674..8c1e65d3 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -36,7 +36,7 @@ LEMUR_ENCRYPTION_KEYS = base64.urlsafe_b64encode(get_random_secret(length=32).en # List of domain regular expressions that non-admin users can issue -LEMUR_WHITELISTED_DOMAINS = [ +LEMUR_ALLOWED_DOMAINS = [ r"^[a-zA-Z0-9-]+\.example\.com$", r"^[a-zA-Z0-9-]+\.example\.org$", r"^example\d+\.long\.com$", diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 8403461b..4e8072ff 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -397,7 +397,7 @@ def test_certificate_cn_admin(client, authority, logged_in_admin): from lemur.certificates.schemas import CertificateInputSchema input_data = { - "commonName": "*.admin-overrides-whitelist.com", + "commonName": "*.admin-overrides-allowlist.com", "owner": "jim@example.com", "authority": {"id": authority.id}, "description": "testtestest", @@ -458,7 +458,7 @@ def test_certificate_incative_authority(client, authority, session, logged_in_us def test_certificate_disallowed_names(client, authority, session, logged_in_user): - """The CN and SAN are disallowed by LEMUR_WHITELISTED_DOMAINS.""" + """The CN and SAN are disallowed by LEMUR_ALLOWED_DOMAINS.""" from lemur.certificates.schemas import CertificateInputSchema input_data = { @@ -481,10 +481,10 @@ def test_certificate_disallowed_names(client, authority, session, logged_in_user data, errors = CertificateInputSchema().load(input_data) assert errors["common_name"][0].startswith( - "Domain *.example.com does not match whitelisted domain patterns" + "Domain *.example.com does not match allowed domain patterns" ) assert errors["extensions"]["sub_alt_names"]["names"][0].startswith( - "Domain evilhacker.org does not match whitelisted domain patterns" + "Domain evilhacker.org does not match allowed domain patterns" ) @@ -671,7 +671,7 @@ def test_csr_empty_san(client): def test_csr_disallowed_cn(client, logged_in_user): - """Domain name CN is disallowed via LEMUR_WHITELISTED_DOMAINS.""" + """Domain name CN is disallowed via LEMUR_ALLOWED_DOMAINS.""" from lemur.common import validators request, pkey = create_csr( @@ -680,12 +680,12 @@ def test_csr_disallowed_cn(client, logged_in_user): with pytest.raises(ValidationError) as err: validators.csr(request) assert str(err.value).startswith( - "Domain evilhacker.org does not match whitelisted domain patterns" + "Domain evilhacker.org does not match allowed domain patterns" ) def test_csr_disallowed_san(client, logged_in_user): - """SAN name is disallowed by LEMUR_WHITELISTED_DOMAINS.""" + """SAN name is disallowed by LEMUR_ALLOWED_DOMAINS.""" from lemur.common import validators request, pkey = create_csr( @@ -701,7 +701,7 @@ def test_csr_disallowed_san(client, logged_in_user): with pytest.raises(ValidationError) as err: validators.csr(request) assert str(err.value).startswith( - "Domain evilhacker.org does not match whitelisted domain patterns" + "Domain evilhacker.org does not match allowed domain patterns" ) From 8b8f9e652bf70886b81b908a3b97b78f9f4c2b2c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2020 06:24:35 +0000 Subject: [PATCH 021/226] Bump botocore from 1.18.12 to 1.18.16 Bumps [botocore](https://github.com/boto/botocore) from 1.18.12 to 1.18.16. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.18.12...1.18.16) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 18338b5a..41f70e6b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,7 +18,7 @@ beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.15.12 # via -r requirements.txt -botocore==1.18.12 # via -r requirements.txt, boto3, s3transfer +botocore==1.18.16 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 2a81f432..07bd2c0b 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in boto3==1.15.12 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.18.12 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.18.16 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 83013eac..e3f3516a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.15.12 # via -r requirements.in -botocore==1.18.12 # via -r requirements.in, boto3, s3transfer +botocore==1.18.16 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From b677e6e3253b81fc8467803f2697ac99b35fe3e2 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 13 Oct 2020 19:40:01 -0700 Subject: [PATCH 022/226] Copy subject details for non-CAB-compliant authorities --- lemur/authorities/models.py | 17 +++++++++++++++++ lemur/authorities/schemas.py | 1 + lemur/certificates/schemas.py | 23 ++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index ccd1fab8..e8c0e03a 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -6,6 +6,8 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +import json + from sqlalchemy.orm import relationship from sqlalchemy import ( Column, @@ -80,5 +82,20 @@ class Authority(db.Model): def plugin(self): return plugins.get(self.plugin_name) + @property + def is_cab_compliant(self): + """ + Parse the options to find whether authority is CAB Compliant. Returns None if + option is not available + """ + if not self.options: + return None + + for option in json.loads(self.options): + if option["name"] == 'cab_compliant': + return option["value"] + + return None + def __repr__(self): return "Authority(name={name})".format(name=self.name) diff --git a/lemur/authorities/schemas.py b/lemur/authorities/schemas.py index f80d1581..6c48a183 100644 --- a/lemur/authorities/schemas.py +++ b/lemur/authorities/schemas.py @@ -139,6 +139,7 @@ class AuthorityNestedOutputSchema(LemurOutputSchema): plugin = fields.Nested(PluginOutputSchema) active = fields.Boolean() authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["max_issuance_days", "default_validity_days"]) + is_cab_compliant = fields.Boolean() authority_update_schema = AuthorityUpdateSchema() diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 688d6ba4..21abd214 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -8,7 +8,7 @@ from flask import current_app from flask_restful import inputs from flask_restful.reqparse import RequestParser -from marshmallow import fields, validate, validates_schema, post_load, pre_load +from marshmallow import fields, validate, validates_schema, post_load, pre_load, post_dump from marshmallow.exceptions import ValidationError from lemur.authorities.schemas import AuthorityNestedOutputSchema @@ -332,6 +332,27 @@ class CertificateOutputSchema(LemurOutputSchema): ) rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema) + country = fields.String() + location = fields.String() + state = fields.String() + organization = fields.String() + organizational_unit = fields.String() + + @post_dump + def handle_subject_details(self, data): + # Remove subject details if authority is CAB compliant. The code will use default set of values in that case. + # If CAB compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below + # condition checks for 'not False' ==> 'True or None' + if data.get("authority"): + is_cab_compliant = data.get("authority").get("isCabCompliant") + + if is_cab_compliant is not False: + data.pop("country", None) + data.pop("state", None) + data.pop("location", None) + data.pop("organization", None) + data.pop("organizational_unit", None) + class CertificateShortOutputSchema(LemurOutputSchema): id = fields.Integer() From 28381737dc8401d683063a47e9abc60831039ffb Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 13 Oct 2020 19:19:12 -0700 Subject: [PATCH 023/226] Removed OU from digicert plugin --- lemur/plugins/lemur_digicert/plugin.py | 1 - lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index cf01c9d1..f28279a6 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -175,7 +175,6 @@ def map_cis_fields(options, csr): }, "organization": { "name": options["organization"], - "units": [options["organizational_unit"]], }, } # possibility to default to a SIGNING_ALGORITHM for a given profile diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 4abfcf54..34dcef71 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -121,7 +121,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority): "csr": CSR_STR, "additional_dns_names": names, "signature_hash": "sha256", - "organization": {"name": "Example, Inc.", "units": ["Example Org"]}, + "organization": {"name": "Example, Inc."}, "validity": { "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM") + "Z" }, @@ -157,7 +157,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho "csr": CSR_STR, "additional_dns_names": names, "signature_hash": "sha256", - "organization": {"name": "Example, Inc.", "units": ["Example Org"]}, + "organization": {"name": "Example, Inc."}, "validity": { "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z" }, From 82dd6639424865cb5324db199783f7c8ef931917 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 13 Oct 2020 19:20:08 -0700 Subject: [PATCH 024/226] Moving default key_type to getDefaults --- .../app/angular/certificates/certificate/certificate.js | 3 --- lemur/static/app/angular/certificates/services.js | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lemur/static/app/angular/certificates/certificate/certificate.js b/lemur/static/app/angular/certificates/certificate/certificate.js index 9fadb655..4bdbf60e 100644 --- a/lemur/static/app/angular/certificates/certificate/certificate.js +++ b/lemur/static/app/angular/certificates/certificate/certificate.js @@ -255,9 +255,6 @@ angular.module('lemur') $scope.certificate.replacedBy = []; // should not clone 'replaced by' info $scope.certificate.removeReplaces(); // should not clone 'replacement cert' info - if(!$scope.certificate.keyType) { - $scope.certificate.keyType = 'RSA2048'; // default algo to select during clone if backend did not return algo - } CertificateService.getDefaults($scope.certificate); }); diff --git a/lemur/static/app/angular/certificates/services.js b/lemur/static/app/angular/certificates/services.js index ce88ccb3..280d6078 100644 --- a/lemur/static/app/angular/certificates/services.js +++ b/lemur/static/app/angular/certificates/services.js @@ -289,6 +289,11 @@ angular.module('lemur') if (certificate.dnsProviderId) { certificate.dnsProvider = {id: certificate.dnsProviderId}; } + + if(!certificate.keyType) { + certificate.keyType = 'RSA2048'; // default algo to select during clone if backend did not return algo + } + }); }; From 97cf54433b5cc0d00819735ff60a87f0b711eb87 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 14 Oct 2020 09:45:13 -0700 Subject: [PATCH 025/226] Update models.py language --- lemur/authorities/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index e8c0e03a..ba0516fd 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -85,8 +85,9 @@ class Authority(db.Model): @property def is_cab_compliant(self): """ - Parse the options to find whether authority is CAB Compliant. Returns None if - option is not available + Parse the options to find whether authority is CAB Forum Compliant + i.e., adhering to the CA/Browser Forum Baseline Requirements. + Returns None if option is not available """ if not self.options: return None From 894e35b4e2a8f310a024bdf7da89009bc47b1f29 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 14 Oct 2020 09:48:40 -0700 Subject: [PATCH 026/226] Update schemas.py minor language --- lemur/certificates/schemas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 21abd214..cc0a607e 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -340,8 +340,8 @@ class CertificateOutputSchema(LemurOutputSchema): @post_dump def handle_subject_details(self, data): - # Remove subject details if authority is CAB compliant. The code will use default set of values in that case. - # If CAB compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below + # Remove subject details if authority is CA/Browser Forum compliant. The code will use default set of values in that case. + # If CA/Browser Forum compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below # condition checks for 'not False' ==> 'True or None' if data.get("authority"): is_cab_compliant = data.get("authority").get("isCabCompliant") From 2fefbb6dea6a2a47dc3dacd2b06e203511afc233 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 16:58:24 +0000 Subject: [PATCH 027/226] Bump arrow from 0.16.0 to 0.17.0 Bumps [arrow](https://github.com/arrow-py/arrow) from 0.16.0 to 0.17.0. - [Release notes](https://github.com/arrow-py/arrow/releases) - [Changelog](https://github.com/arrow-py/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/arrow-py/arrow/compare/0.16.0...0.17.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 41f70e6b..42701db6 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -10,7 +10,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate amqp==2.5.2 # via -r requirements.txt, kombu aniso8601==8.0.0 # via -r requirements.txt, flask-restful -arrow==0.16.0 # via -r requirements.txt +arrow==0.17.0 # via -r requirements.txt asyncpool==1.0 # via -r requirements.txt babel==2.8.0 # via sphinx bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko diff --git a/requirements.txt b/requirements.txt index e3f3516a..350735cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful -arrow==0.16.0 # via -r requirements.in +arrow==0.17.0 # via -r requirements.in asyncpool==1.0 # via -r requirements.in bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare From 409e12a9d60901ef35449058774bb610c6ddbdeb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 14 Oct 2020 10:03:44 -0700 Subject: [PATCH 028/226] Update models.py lint --- lemur/authorities/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index ba0516fd..cced3ed1 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -85,7 +85,7 @@ class Authority(db.Model): @property def is_cab_compliant(self): """ - Parse the options to find whether authority is CAB Forum Compliant + Parse the options to find whether authority is CAB Forum Compliant, i.e., adhering to the CA/Browser Forum Baseline Requirements. Returns None if option is not available """ From 499bbca42ebd8a430bf41f29fcdd35d870856c6a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 17:16:30 +0000 Subject: [PATCH 029/226] Bump acme from 1.8.0 to 1.9.0 Bumps [acme](https://github.com/letsencrypt/letsencrypt) from 1.8.0 to 1.9.0. - [Release notes](https://github.com/letsencrypt/letsencrypt/releases) - [Commits](https://github.com/letsencrypt/letsencrypt/compare/v1.8.0...v1.9.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 42701db6..e35ead88 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==1.8.0 # via -r requirements.txt +acme==1.9.0 # via -r requirements.txt alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 # via -r requirements.txt alembic==1.4.2 # via -r requirements.txt, flask-migrate diff --git a/requirements.txt b/requirements.txt index 350735cd..fb43b5a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==1.8.0 # via -r requirements.in +acme==1.9.0 # via -r requirements.in alembic-autogenerate-enums==0.0.2 # via -r requirements.in alembic==1.4.2 # via flask-migrate amqp==2.5.2 # via kombu From f08b50a952b5988b1939ba35f392321f10f64d17 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 17:45:18 +0000 Subject: [PATCH 030/226] Bump boto3 from 1.15.12 to 1.15.16 Bumps [boto3](https://github.com/boto/boto3) from 1.15.12 to 1.15.16. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.15.12...1.15.16) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index e35ead88..38b62198 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,7 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.15.12 # via -r requirements.txt +boto3==1.15.16 # via -r requirements.txt botocore==1.18.16 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests diff --git a/requirements-tests.txt b/requirements-tests.txt index 07bd2c0b..39ef0e66 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.15.12 # via aws-sam-translator, moto +boto3==1.15.16 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.18.16 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests diff --git a/requirements.txt b/requirements.txt index fb43b5a3..d323b40f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.15.12 # via -r requirements.in +boto3==1.15.16 # via -r requirements.in botocore==1.18.16 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests From ddf94e04da75dabf599e7e846f78f0fe40e251f1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 18:09:48 +0000 Subject: [PATCH 031/226] Bump faker from 4.4.0 to 4.14.0 Bumps [faker](https://github.com/joke2k/faker) from 4.4.0 to 4.14.0. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.4.0...v4.14.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 39ef0e66..8df4f5d1 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in -faker==4.4.0 # via -r requirements-tests.in, factory-boy +faker==4.14.0 # via -r requirements-tests.in, factory-boy fakeredis==1.4.3 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in From 62d099b50073d8723a2c465c43e7a5f60c2fc37a Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 14 Oct 2020 12:41:49 -0700 Subject: [PATCH 032/226] Unit tests to check cab_compliant option --- lemur/tests/test_certificates.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index c4140f03..c0f59f2f 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -9,7 +9,6 @@ from cryptography import x509 from cryptography.hazmat.backends import default_backend from marshmallow import ValidationError from freezegun import freeze_time -# from mock import patch from unittest.mock import patch from lemur.certificates.service import create_csr @@ -171,10 +170,33 @@ def test_certificate_output_schema(session, certificate, issuer_plugin): ) as wrapper: data, errors = CertificateOutputSchema().dump(certificate) assert data["issuer"] == "LemurTrustUnittestsClass1CA2018" + assert data["distinguishedName"] == "L=Earth,ST=N/A,C=EE,OU=Karate Lessons,O=Daniel San & co,CN=san.example.org" + # Authority does not have 'cab_compliant', thus subject details should not be returned + assert "organization" not in data assert wrapper.call_count == 1 +def test_certificate_output_schema_subject_details(session, certificate, issuer_plugin): + from lemur.certificates.schemas import CertificateOutputSchema + from lemur.authorities.service import update_options + + # Mark authority as non-cab-compliant + update_options(certificate.authority.id, '[{"name": "cab_compliant","value":false}]') + + data, errors = CertificateOutputSchema().dump(certificate) + assert not errors + assert data["issuer"] == "LemurTrustUnittestsClass1CA2018" + assert data["distinguishedName"] == "L=Earth,ST=N/A,C=EE,OU=Karate Lessons,O=Daniel San & co,CN=san.example.org" + + # Original subject details should be returned because of cab_compliant option update above + assert data["country"] == "EE" + assert data["state"] == "N/A" + assert data["location"] == "Earth" + assert data["organization"] == "Daniel San & co" + assert data["organizationalUnit"] == "Karate Lessons" + + def test_certificate_edit_schema(session): from lemur.certificates.schemas import CertificateEditInputSchema @@ -921,6 +943,14 @@ def test_certificate_get_body(client): "CN=LemurTrust Unittests Class 1 CA 2018" ) + # No authority details are provided in this test, no information about being cab_compliant is available. + # Thus original subject details should be returned. + assert response_body["country"] == "EE" + assert response_body["state"] == "N/A" + assert response_body["location"] == "Earth" + assert response_body["organization"] == "LemurTrust Enterprises Ltd" + assert response_body["organizationalUnit"] == "Unittesting Operations Center" + @pytest.mark.parametrize( "token,status", From 90839b4d4b1b467409fb84827ab957bd039578e4 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 14 Oct 2020 14:48:54 -0700 Subject: [PATCH 033/226] Unit test for cab_compliant = true --- lemur/tests/test_certificates.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index c0f59f2f..fbe24e26 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -196,6 +196,16 @@ def test_certificate_output_schema_subject_details(session, certificate, issuer_ assert data["organization"] == "Daniel San & co" assert data["organizationalUnit"] == "Karate Lessons" + # Mark authority as cab-compliant + update_options(certificate.authority.id, '[{"name": "cab_compliant","value":true}]') + data, errors = CertificateOutputSchema().dump(certificate) + assert not errors + assert "country" not in data + assert "state" not in data + assert "location" not in data + assert "organization" not in data + assert "organizationalUnit" not in data + def test_certificate_edit_schema(session): from lemur.certificates.schemas import CertificateEditInputSchema From ee1d07000a4c827a061979d49d2b2369110e20f5 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 14 Oct 2020 14:49:46 -0700 Subject: [PATCH 034/226] Test subject details in reissue with cab_compliant option --- lemur/tests/test_certificates.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index fbe24e26..9c50c438 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -791,12 +791,25 @@ def test_reissue_certificate( issuer_plugin, crypto_authority, certificate, logged_in_user ): from lemur.certificates.service import reissue_certificate + from lemur.authorities.service import update_options + from lemur.tests.conf import LEMUR_DEFAULT_ORGANIZATION # test-authority would return a mismatching private key, so use 'cryptography-issuer' plugin instead. certificate.authority = crypto_authority new_cert = reissue_certificate(certificate) assert new_cert - assert (new_cert.key_type == "RSA2048") + assert new_cert.key_type == "RSA2048" + assert new_cert.organization != certificate.organization + # Check for default value since authority does not have cab_compliant option set + assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION + + # update cab_compliant option to false for crypto_authority to maintain subject details + update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]') + new_cert = reissue_certificate(certificate) + assert new_cert.organization == certificate.organization + + # reset options + update_options(crypto_authority.id, None) def test_create_csr(): From 4d5e712e854367c8d42d0e02ac407bd07c7f4e8c Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 14 Oct 2020 15:40:23 -0700 Subject: [PATCH 035/226] Remove option reset from test --- lemur/tests/test_certificates.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 9c50c438..c271a97e 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -808,9 +808,6 @@ def test_reissue_certificate( new_cert = reissue_certificate(certificate) assert new_cert.organization == certificate.organization - # reset options - update_options(crypto_authority.id, None) - def test_create_csr(): csr, private_key = create_csr( From f38380d156a29d5403ba43c2ef7e07b0d1b57c40 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 14 Oct 2020 17:38:32 -0700 Subject: [PATCH 036/226] Check if option is present --- lemur/authorities/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index cced3ed1..d1b41a21 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -93,7 +93,7 @@ class Authority(db.Model): return None for option in json.loads(self.options): - if option["name"] == 'cab_compliant': + if "name" in option and option["name"] == 'cab_compliant': return option["value"] return None From 9dc476f39317679fa03df741bf5fc8166d93aae6 Mon Sep 17 00:00:00 2001 From: sayali Date: Thu, 15 Oct 2020 10:44:46 -0700 Subject: [PATCH 037/226] Use cab_compliant option instead of authority name list --- docs/administration.rst | 13 ++++--------- lemur/authorities/models.py | 13 +++++++++++++ lemur/authorities/schemas.py | 8 +++++--- lemur/certificates/models.py | 14 -------------- .../certificates/certificate/certificate.js | 4 ++-- .../certificates/certificate/tracking.tpl.html | 2 +- lemur/static/app/angular/certificates/services.js | 4 ++-- .../app/angular/pending_certificates/services.js | 4 ++-- 8 files changed, 29 insertions(+), 33 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 00da0c8a..6e53c826 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -155,17 +155,12 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c LEMUR_ENCRYPTION_KEYS = ['1YeftooSbxCiX2zo8m1lXtpvQjy27smZcUUaGmffhMY=', 'LAfQt6yrkLqOK5lwpvQcT4jf2zdeTQJV1uYeh9coT5s='] -.. data:: PUBLIC_CA_AUTHORITY_NAMES - :noindex: - A list of public issuers which would be checked against to determine whether limit of max validity of 397 days - should be applied to the certificate. Configure public CA authority names in this list to enforce validity check. - This is an optional setting. Using this will allow the sanity check as mentioned. The name check is a case-insensitive - string comparision. .. data:: PUBLIC_CA_MAX_VALIDITY_DAYS :noindex: - Use this config to override the limit of 397 days of validity for certificates issued by public issuers configured - using PUBLIC_CA_AUTHORITY_NAMES. Below example overrides the default validity of 397 days and sets it to 365 days. + Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities. + The authorities with cab_compliant option set to true will use this config. Below example overrides the default validity + of 397 days and sets it to 365 days. :: @@ -175,7 +170,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c .. data:: DEFAULT_VALIDITY_DAYS :noindex: Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which - is not listed in PUBLIC_CA_AUTHORITY_NAMES will be using this value as default validity to be displayed on UI. Please + is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please note that this config is used for cert issuance only through Lemur UI. Below example overrides the default validity of 365 days and sets it to 1095 days (3 years). diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index d1b41a21..f042f773 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -8,6 +8,7 @@ """ import json +from flask import current_app from sqlalchemy.orm import relationship from sqlalchemy import ( Column, @@ -98,5 +99,17 @@ class Authority(db.Model): return None + @property + def max_issuance_days(self): + if self.is_cab_compliant: + return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) + + @property + def default_validity_days(self): + if self.is_cab_compliant: + return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) + + return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365) # 1 year default + def __repr__(self): return "Authority(name={name})".format(name=self.name) diff --git a/lemur/authorities/schemas.py b/lemur/authorities/schemas.py index 6c48a183..555ba931 100644 --- a/lemur/authorities/schemas.py +++ b/lemur/authorities/schemas.py @@ -111,8 +111,6 @@ class RootAuthorityCertificateOutputSchema(LemurOutputSchema): cn = fields.String() not_after = fields.DateTime() not_before = fields.DateTime() - max_issuance_days = fields.Integer() - default_validity_days = fields.Integer() owner = fields.Email() status = fields.Boolean() user = fields.Nested(UserNestedOutputSchema) @@ -127,6 +125,8 @@ class AuthorityOutputSchema(LemurOutputSchema): active = fields.Boolean() options = fields.Dict() roles = fields.List(fields.Nested(AssociatedRoleSchema)) + max_issuance_days = fields.Integer() + default_validity_days = fields.Integer() authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema) @@ -138,8 +138,10 @@ class AuthorityNestedOutputSchema(LemurOutputSchema): owner = fields.Email() plugin = fields.Nested(PluginOutputSchema) active = fields.Boolean() - authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["max_issuance_days", "default_validity_days"]) + authority_certificate = fields.Nested(RootAuthorityCertificateOutputSchema, only=["not_after", "not_before"]) is_cab_compliant = fields.Boolean() + max_issuance_days = fields.Integer() + default_validity_days = fields.Integer() authority_update_schema = AuthorityUpdateSchema() diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 60442de2..f6562b3f 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -317,20 +317,6 @@ class Certificate(db.Model): def validity_range(self): return self.not_after - self.not_before - @property - def max_issuance_days(self): - public_CA = current_app.config.get("PUBLIC_CA_AUTHORITY_NAMES", []) - if self.name.lower() in [ca.lower() for ca in public_CA]: - return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) - - @property - def default_validity_days(self): - public_CA = current_app.config.get("PUBLIC_CA_AUTHORITY_NAMES", []) - if self.name.lower() in [ca.lower() for ca in public_CA]: - return current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397) - - return current_app.config.get("DEFAULT_VALIDITY_DAYS", 365) # 1 year default - @property def subject(self): return self.parsed_cert.subject diff --git a/lemur/static/app/angular/certificates/certificate/certificate.js b/lemur/static/app/angular/certificates/certificate/certificate.js index 4bdbf60e..41e04d55 100644 --- a/lemur/static/app/angular/certificates/certificate/certificate.js +++ b/lemur/static/app/angular/certificates/certificate/certificate.js @@ -190,7 +190,7 @@ angular.module('lemur') function populateValidityDateAsPerDefault(certificate) { // calculate start and end date as per default validity let startDate = new Date(), endDate = new Date(); - endDate.setDate(startDate.getDate() + certificate.authority.authorityCertificate.defaultValidityDays); + endDate.setDate(startDate.getDate() + certificate.authority.defaultValidityDays); certificate.validityStart = startDate; certificate.validityEnd = endDate; } @@ -359,7 +359,7 @@ angular.module('lemur') function populateValidityDateAsPerDefault(certificate) { // calculate start and end date as per default validity let startDate = new Date(), endDate = new Date(); - endDate.setDate(startDate.getDate() + certificate.authority.authorityCertificate.defaultValidityDays); + endDate.setDate(startDate.getDate() + certificate.authority.defaultValidityDays); certificate.validityStart = startDate; certificate.validityEnd = endDate; } diff --git a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html index d60a1a6a..c50d40ba 100644 --- a/lemur/static/app/angular/certificates/certificate/tracking.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/tracking.tpl.html @@ -139,7 +139,7 @@
+ Default ({{certificate.authority.defaultValidityDays}} days)
diff --git a/lemur/static/app/angular/certificates/services.js b/lemur/static/app/angular/certificates/services.js index 280d6078..be19bafd 100644 --- a/lemur/static/app/angular/certificates/services.js +++ b/lemur/static/app/angular/certificates/services.js @@ -172,12 +172,12 @@ angular.module('lemur') // Minimum end date will be same as selected start date this.authority.authorityCertificate.minValidityEnd = value; - if(!this.authority.authorityCertificate || !this.authority.authorityCertificate.maxIssuanceDays) { + if(!this.authority.maxIssuanceDays) { this.authority.authorityCertificate.maxValidityEnd = this.authority.authorityCertificate.notAfter; } else { // Move max end date by maxIssuanceDays let endDate = new Date(value); - endDate.setDate(endDate.getDate() + this.authority.authorityCertificate.maxIssuanceDays); + endDate.setDate(endDate.getDate() + this.authority.maxIssuanceDays); this.authority.authorityCertificate.maxValidityEnd = endDate; } } diff --git a/lemur/static/app/angular/pending_certificates/services.js b/lemur/static/app/angular/pending_certificates/services.js index 9b32c1d3..7f20355b 100644 --- a/lemur/static/app/angular/pending_certificates/services.js +++ b/lemur/static/app/angular/pending_certificates/services.js @@ -152,12 +152,12 @@ angular.module('lemur') // Minimum end date will be same as selected start date this.authority.authorityCertificate.minValidityEnd = value; - if(!this.authority.authorityCertificate || !this.authority.authorityCertificate.maxIssuanceDays) { + if(!this.authority.maxIssuanceDays) { this.authority.authorityCertificate.maxValidityEnd = this.authority.authorityCertificate.notAfter; } else { // Move max end date by maxIssuanceDays let endDate = new Date(value); - endDate.setDate(endDate.getDate() + this.authority.authorityCertificate.maxIssuanceDays); + endDate.setDate(endDate.getDate() + this.authority.maxIssuanceDays); this.authority.authorityCertificate.maxValidityEnd = endDate; } } From 29f3dd43f2e1779866e61ba96547ca1b561d093f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 15 Oct 2020 15:18:04 -0700 Subject: [PATCH 038/226] Update administration.rst language --- docs/administration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 6e53c826..c2f20362 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -159,7 +159,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c .. data:: PUBLIC_CA_MAX_VALIDITY_DAYS :noindex: Use this config to override the limit of 397 days of validity for certificates issued by CA/Browser compliant authorities. - The authorities with cab_compliant option set to true will use this config. Below example overrides the default validity + The authorities with cab_compliant option set to true will use this config. The example below overrides the default validity of 397 days and sets it to 365 days. :: @@ -171,7 +171,7 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c :noindex: Use this config to override the default validity of 365 days for certificates offered through Lemur UI. Any CA which is not CA/Browser Forum compliant will be using this value as default validity to be displayed on UI. Please - note that this config is used for cert issuance only through Lemur UI. Below example overrides the default validity + note that this config is used for cert issuance only through Lemur UI. The example below overrides the default validity of 365 days and sets it to 1095 days (3 years). :: From bfe89e131e71115a779359dd9a389ba8ec306bdb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 15 Oct 2020 18:13:50 -0700 Subject: [PATCH 039/226] adding delete and put interfaces for the S3 plugin --- lemur/plugins/lemur_aws/s3.py | 40 ++++++++++++++++++++++-- lemur/plugins/lemur_aws/tests/test_s3.py | 38 ++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 lemur/plugins/lemur_aws/tests/test_s3.py diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 43faa28f..c868c7a3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -9,9 +9,12 @@ from flask import current_app from .sts import sts_client +from botocore.exceptions import ClientError +from lemur.extensions import sentry + @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ @@ -32,4 +35,37 @@ def put(bucket_name, region, prefix, data, encrypt, **kwargs): ServerSideEncryption="AES256", ) else: - bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + try: + bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + except ClientError: + sentry.captureException() + + +@sts_client("s3", service_type="client") +def delete(bucket_name, prefix, **kwargs): + """ + Use STS to delete an object + """ + try: + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix) + current_app.logger.debug(f"Delete data from S3." + f"Bucket: {bucket_name}," + f"Prefix: {prefix}," + f"Status_code: {response}") + return response['ResponseMetadata']['HTTPStatusCode'] < 300 + except ClientError: + sentry.captureException() + + +@sts_client("s3", service_type="client") +def get(bucket_name, prefix, **kwargs): + """ + Use STS to get an object + """ + try: + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix) + current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," + f"Prefix: {prefix}") + return response['Body'].read().decode("utf-8") + except ClientError: + sentry.captureException() diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py new file mode 100644 index 00000000..f7a36496 --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -0,0 +1,38 @@ +import boto3 +from moto import mock_sts, mock_s3 + + +@mock_sts() +@mock_s3() +def test_put_delete_s3_object(app): + from lemur.plugins.lemur_aws.s3 import put, delete, get + + bucket = "public-bucket" + account = "123456789012" + path = "some_path/foo" + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + + data = "dummy data" + put(bucket_name=bucket, + prefix=path, + data=data, + encrypt=None, + account_number=account) + + response = get(bucket_name=bucket, prefix=path, account_number=account) + + # put data, and getting the same data + assert (response == data) + + response = get(bucket_name="wrong-bucket", prefix=path, account_number=account) + + # attempting to get thccle wrong data + assert (response is None) + + delete(bucket_name=bucket, prefix=path, account_number=account) + response = get(bucket_name=bucket, prefix=path, account_number=account) + + # delete data, and getting the same data + assert (response is None) From 55658c5f23870c44f01754bfc0b8df8dffc7f361 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Fri, 16 Oct 2020 10:43:52 +0200 Subject: [PATCH 040/226] Add double % for escaped SQLALCHEMY_DATABASE_URI --- lemur/migrations/env.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/migrations/env.py b/lemur/migrations/env.py index 3acefc3a..91fa5fcb 100644 --- a/lemur/migrations/env.py +++ b/lemur/migrations/env.py @@ -20,8 +20,9 @@ fileConfig(config.config_file_name) # target_metadata = mymodel.Base.metadata from flask import current_app +db_url_escaped = current_app.config.get('SQLALCHEMY_DATABASE_URI').replace('%', '%%') config.set_main_option( - "sqlalchemy.url", current_app.config.get("SQLALCHEMY_DATABASE_URI") + "sqlalchemy.url", db_url_escaped ) target_metadata = current_app.extensions["migrate"].db.metadata From d73db59d2352ad54d230948cb4e323cbc46cd30b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:48:47 -0700 Subject: [PATCH 041/226] revsering removing region --- lemur/plugins/lemur_aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index c868c7a3..0e9db182 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -14,7 +14,7 @@ from lemur.extensions import sentry @sts_client("s3", service_type="resource") -def put(bucket_name, prefix, data, encrypt, **kwargs): +def put(bucket_name, region, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ From 6aad37e1f9d115da065c321d88e21b38045ae80c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:00 -0700 Subject: [PATCH 042/226] cleaning up code --- lemur/plugins/lemur_aws/s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 0e9db182..186b715d 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -6,12 +6,12 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -from flask import current_app -from .sts import sts_client - from botocore.exceptions import ClientError +from flask import current_app from lemur.extensions import sentry +from .sts import sts_client + @sts_client("s3", service_type="resource") def put(bucket_name, region, prefix, data, encrypt, **kwargs): From 7d8eb1c61edba2656257fbe2c42f864ce9b87107 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:26 -0700 Subject: [PATCH 043/226] improving test --- lemur/plugins/lemur_aws/tests/test_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index f7a36496..7b8b4ac3 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -9,16 +9,16 @@ def test_put_delete_s3_object(app): bucket = "public-bucket" account = "123456789012" - path = "some_path/foo" + path = "some-path/foo" + data = "dummy data" s3_client = boto3.client('s3') s3_client.create_bucket(Bucket=bucket) - data = "dummy data" put(bucket_name=bucket, prefix=path, data=data, - encrypt=None, + encrypt=False, account_number=account) response = get(bucket_name=bucket, prefix=path, account_number=account) From d705e3ae3b5cce2fd8cfd34ebc2283217d180557 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:49:56 -0700 Subject: [PATCH 044/226] expanding the S3 destination plugin to support the acme token upload inteface --- lemur/plugins/lemur_aws/plugin.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8692348a..6d161ac3 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -406,3 +406,25 @@ class S3DestinationPlugin(ExportDestinationPlugin): self.get_option("encrypt", options), account_number=self.get_option("accountNumber", options), ) + + def upload_acme_token(self, token_path, token, options, **kwargs): + """ + This is called from the acme http challenge + :param self: + :param token_path: + :param token: + :param options: + :param kwargs: + :return: + """ + current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge") + + account_number = self.get_option("accountNumber", options) + bucket_name = self.get_option("bucket", options) + prefix = self.get_option("prefix", options) + region = self.get_option("region", options) + filename = token_path.split("/")[-1] + if not prefix.endswith("/"): + prefix + "/" + + s3.put(bucket_name, region, prefix + filename, token, encrypt=False, account_number=account_number) From 17e528b5dd56284b097f85bb2f6d1e895ac7bfc6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:50:35 -0700 Subject: [PATCH 045/226] adding testing for acme_upload method --- lemur/plugins/lemur_aws/tests/test_plugin.py | 77 ++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index dbad7b02..a471f7c8 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -1,5 +1,82 @@ +import boto3 +from moto import mock_sts, mock_s3 + + def test_get_certificates(app): from lemur.plugins.base import plugins p = plugins.get("aws-s3") assert p + + +@mock_sts() +@mock_s3() +def test_upload_acme_token(app): + from lemur.plugins.base import plugins + from lemur.plugins.lemur_aws.s3 import get + + bucket = "public-bucket" + account = "123456789012" + prefix = "some-path/more-path/" + token_content = "Challenge" + token_name = "TOKEN" + token_path = ".well-known/acme-challenge/" + token_name + + additional_options = [ + { + "name": "bucket", + "value": bucket, + "type": "str", + "required": True, + "validation": "[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account, + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + s3_client = boto3.client('s3') + s3_client.create_bucket(Bucket=bucket) + p = plugins.get("aws-s3") + + p.upload_acme_token(token_path=token_path, + token_content=token_content, + token=token_content, + options=additional_options) + + response = get(bucket_name=bucket, + prefix=prefix + token_name, + encrypt=False, + account_number=account) + + # put data, and getting the same data + assert (response == token_content) From 9c04a888d8122f100aefe646137cb8c0908ae489 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 09:52:04 -0700 Subject: [PATCH 046/226] adjusting the S3 test --- lemur/plugins/lemur_aws/tests/test_s3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index 7b8b4ac3..bfb5a9f9 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -16,6 +16,7 @@ def test_put_delete_s3_object(app): s3_client.create_bucket(Bucket=bucket) put(bucket_name=bucket, + region=None, prefix=path, data=data, encrypt=False, From 11ce540246676b9441fde40b198d8ca0ea7ac2a7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 10:31:19 -0700 Subject: [PATCH 047/226] formatting --- lemur/plugins/lemur_aws/plugin.py | 7 ++++++- lemur/plugins/lemur_aws/s3.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 6d161ac3..ad80d87f 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -427,4 +427,9 @@ class S3DestinationPlugin(ExportDestinationPlugin): if not prefix.endswith("/"): prefix + "/" - s3.put(bucket_name, region, prefix + filename, token, encrypt=False, account_number=account_number) + s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 186b715d..7c4177ff 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -14,7 +14,7 @@ from .sts import sts_client @sts_client("s3", service_type="resource") -def put(bucket_name, region, prefix, data, encrypt, **kwargs): +def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): """ Use STS to write to an S3 bucket """ From 503530e93512291985d5d865b9b7af55d2e05488 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 16 Oct 2020 10:32:10 -0700 Subject: [PATCH 048/226] the test requires region param for sts --- lemur/plugins/lemur_aws/tests/test_s3.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index bfb5a9f9..88bd30d2 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -8,6 +8,7 @@ def test_put_delete_s3_object(app): from lemur.plugins.lemur_aws.s3 import put, delete, get bucket = "public-bucket" + region = "us-east-1" account = "123456789012" path = "some-path/foo" data = "dummy data" @@ -16,11 +17,12 @@ def test_put_delete_s3_object(app): s3_client.create_bucket(Bucket=bucket) put(bucket_name=bucket, - region=None, + region_name=region, prefix=path, data=data, encrypt=False, - account_number=account) + account_number=account, + region=region) response = get(bucket_name=bucket, prefix=path, account_number=account) From a04cce6044360f4bc0987259d4f34d2823c027f0 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 16 Oct 2020 10:40:11 -0700 Subject: [PATCH 049/226] Initial implementation --- docs/administration.rst | 17 +- docs/developer/plugins/index.rst | 13 +- lemur/notifications/messaging.py | 162 ++++++++---------- lemur/plugins/bases/notification.py | 11 +- lemur/plugins/lemur_aws/plugin.py | 59 ++++++- lemur/plugins/lemur_aws/sns.py | 56 ++++++ lemur/plugins/lemur_aws/tests/test_sns.py | 50 ++++++ lemur/plugins/lemur_email/plugin.py | 24 ++- lemur/plugins/lemur_email/tests/test_email.py | 12 ++ lemur/plugins/lemur_slack/plugin.py | 5 +- lemur/tests/plugins/notification_plugin.py | 1 + lemur/tests/test_messaging.py | 23 ++- setup.py | 1 + 13 files changed, 330 insertions(+), 104 deletions(-) create mode 100644 lemur/plugins/lemur_aws/sns.py create mode 100644 lemur/plugins/lemur_aws/tests/test_sns.py diff --git a/docs/administration.rst b/docs/administration.rst index 00da0c8a..cdbf0037 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1441,7 +1441,7 @@ Slack Adds support for slack notifications. -AWS +AWS (Source) ---- :Authors: @@ -1454,7 +1454,7 @@ AWS Uses AWS IAM as a source of certificates to manage. Supports a multi-account deployment. -AWS +AWS (Destination) ---- :Authors: @@ -1467,6 +1467,19 @@ AWS Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment. +AWS (Notification) +----- + +:Authors: + Jasmine Schladen +:Type: + Notification +:Description: + Adds support for SNS notifications. SNS notifications (like other notification plugins) are currently only supported + for certificate expiration. Configuration requires a region, account number, and SNS topic name; these elements + are then combined to build the topic ARN. Lemur must have access to publish messages to the specified SNS topic. + + Kubernetes ---------- diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index 8af5e1c8..0223d9ca 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -215,18 +215,21 @@ Notification ------------ Lemur includes the ability to create Email notifications by **default**. These notifications -currently come in the form of expiration notices. Lemur periodically checks certifications expiration dates and +currently come in the form of expiration and rotation notices. Lemur periodically checks certifications expiration dates and determines if a given certificate is eligible for notification. There are currently only two parameters used to determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number of days the current date (UTC) is from that expiration date. -There are currently two objects that available for notification plugins the first is `NotficationPlugin`. This is the base object for -any notification within Lemur. Currently the only support notification type is an certificate expiration notification. If you +Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable. +Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email. + +There are currently two objects that available for notification plugins. The first is `NotificationPlugin`, which is the base object for +any notification within Lemur. Currently the only supported notification type is an certificate expiration notification. If you are trying to create a new notification type (audit, failed logins, etc.) this would be the object to base your plugin on. You would also then need to build additional code to trigger the new notification type. -The second is `ExpirationNotificationPlugin`, this object inherits from `NotificationPlugin` object. -You will most likely want to base your plugin on, if you want to add new channels for expiration notices (Slack, HipChat, Jira, etc.). It adds default options that are required by +The second is `ExpirationNotificationPlugin`, which inherits from `NotificationPlugin` object. +You will most likely want to base your plugin on this object if you want to add new channels for expiration notices (HipChat, Jira, etc.). It adds default options that are required by all expiration notifications (interval, unit). This interface expects for the child to define the following function:: def send(self, notification_type, message, targets, options, **kwargs): diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 82db7b6e..6bcf6bd3 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -29,13 +29,15 @@ from lemur.plugins.utils import get_plugin_option def get_certificates(exclude=None): """ - Finds all certificates that are eligible for notifications. + Finds all certificates that are eligible for expiration notifications. :param exclude: :return: """ now = arrow.utcnow() max = now + timedelta(days=90) + print("ALPACA: Checking for certs not after {0} with notify enabled and not expired".format(max)) + q = ( database.db.session.query(Certificate) .filter(Certificate.not_after <= max) @@ -43,6 +45,8 @@ def get_certificates(exclude=None): .filter(Certificate.expired == False) ) # noqa + print("ALPACA: Excluding {0}".format(exclude)) + exclude_conditions = [] if exclude: for e in exclude: @@ -56,51 +60,64 @@ def get_certificates(exclude=None): if needs_notification(c): certs.append(c) + print("ALPACA: Found {0} eligible certs".format(len(certs))) + return certs def get_eligible_certificates(exclude=None): """ - Finds all certificates that are eligible for certificate expiration. + Finds all certificates that are eligible for certificate expiration notification. + Returns the set of all eligible certificates, grouped by owner, with a list of applicable notifications. :param exclude: :return: """ certificates = defaultdict(dict) certs = get_certificates(exclude=exclude) + print("ALPACA: Found {0} certificates to check for notifications".format(len(certs))) + # group by owner for owner, items in groupby(certs, lambda x: x.owner): notification_groups = [] for certificate in items: notifications = needs_notification(certificate) + print("ALPACA: Considering sending {0} notifications for cert {1}".format(len(notifications), certificate)) if notifications: for notification in notifications: + print("ALPACA: Will send notification {0} for certificate {1}".format(notification, certificate)) notification_groups.append((notification, certificate)) # group by notification for notification, items in groupby(notification_groups, lambda x: x[0].label): certificates[owner][notification] = list(items) + print("ALPACA: Certificates that need notifications: {0}".format(certificates)) + return certificates -def send_notification(event_type, data, targets, notification): +def send_plugin_notification(event_type, data, recipients, notification): """ Executes the plugin and handles failure. :param event_type: :param data: - :param targets: + :param recipients: :param notification: :return: """ status = FAILURE_METRIC_STATUS try: - notification.plugin.send(event_type, data, targets, notification.options) + print("ALPACA: Trying to send notification {0} (plugin: {1})".format(notification, notification.plugin)) + notification.plugin.send(event_type, data, recipients, notification.options) status = SUCCESS_METRIC_STATUS except Exception as e: + current_app.logger.error( + "Unable to send notification {}.".format(notification), exc_info=True + ) sentry.captureException() metrics.send( @@ -140,36 +157,31 @@ def send_expiration_notifications(exclude): notification_data.append(cert_data) security_data.append(cert_data) - if send_notification( - "expiration", notification_data, [owner], notification + print("ALPACA: Sending owner notification to {0} for certificate {1}. Data: {2}".format(owner, certificates, notification_data)) + + if send_default_notification( + "expiration", notification_data, [owner], notification.options ): success += 1 else: failure += 1 - notification_recipient = get_plugin_option( - "recipients", notification.options - ) - if notification_recipient: - notification_recipient = notification_recipient.split(",") - # removing owner and security_email from notification_recipient - notification_recipient = [i for i in notification_recipient if i not in security_email and i != owner] + recipients = notification.plugin.filter_recipients(security_email + [owner], notification.options) - if ( - notification_recipient + print("ALPACA: Sending plugin notification {0} for certificate {1} to recipients {2}".format(notification, certificates, recipients)) + if send_plugin_notification( + "expiration", + notification_data, + recipients, + notification, ): - if send_notification( - "expiration", - notification_data, - notification_recipient, - notification, - ): - success += 1 - else: - failure += 1 + success += 1 + else: + failure += 1 - if send_notification( - "expiration", security_data, security_email, notification + print("ALPACA: Sending security notification to {0}".format(security_email)) + if send_default_notification( + "expiration", security_data, security_email, notification.options ): success += 1 else: @@ -178,29 +190,29 @@ def send_expiration_notifications(exclude): return success, failure -def send_rotation_notification(certificate, notification_plugin=None): +def send_default_notification(notification_type, data, targets, notification_options=None): """ - Sends a report to certificate owners when their certificate has been - rotated. + Sends a report to the specified target via the default notification plugin. Applicable for any notification_type. + At present, "default" means email, as the other notification plugins do not support dynamically configured targets. - :param certificate: - :param notification_plugin: + :param notification_type: + :param data: + :param targets: + :param notification_options: :return: """ status = FAILURE_METRIC_STATUS - if not notification_plugin: - notification_plugin = plugins.get( - current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN") - ) - - data = certificate_notification_output_schema.dump(certificate).data + notification_plugin = plugins.get( + current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification") + ) try: - notification_plugin.send("rotation", data, [data["owner"]]) + # we need the notification.options here because the email templates utilize the interval/unit info + notification_plugin.send(notification_type, data, targets, notification_options) status = SUCCESS_METRIC_STATUS except Exception as e: current_app.logger.error( - "Unable to send notification to {}.".format(data["owner"]), exc_info=True + "Unable to send notification to {}.".format(targets), exc_info=True ) sentry.captureException() @@ -208,77 +220,49 @@ def send_rotation_notification(certificate, notification_plugin=None): "notification", "counter", 1, - metric_tags={"status": status, "event_type": "rotation"}, + metric_tags={"status": status, "event_type": notification_type}, ) if status == SUCCESS_METRIC_STATUS: return True +def send_rotation_notification(certificate): + data = certificate_notification_output_schema.dump(certificate).data + return send_default_notification("rotation", data, [data["owner"]]) + + def send_pending_failure_notification( - pending_cert, notify_owner=True, notify_security=True, notification_plugin=None + pending_cert, notify_owner=True, notify_security=True ): """ Sends a report to certificate owners when their pending certificate failed to be created. :param pending_cert: - :param notification_plugin: + :param notify_owner: + :param notify_security: :return: """ - status = FAILURE_METRIC_STATUS - - if not notification_plugin: - notification_plugin = plugins.get( - current_app.config.get( - "LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification" - ) - ) data = pending_certificate_output_schema.dump(pending_cert).data data["security_email"] = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") + notify_owner_success = False if notify_owner: - try: - notification_plugin.send("failed", data, [data["owner"]], pending_cert) - status = SUCCESS_METRIC_STATUS - except Exception as e: - current_app.logger.error( - "Unable to send pending failure notification to {}.".format( - data["owner"] - ), - exc_info=True, - ) - sentry.captureException() + notify_owner_success = send_default_notification("failed", data, [data["owner"]], pending_cert) + notify_security_success = False if notify_security: - try: - notification_plugin.send( - "failed", data, data["security_email"], pending_cert - ) - status = SUCCESS_METRIC_STATUS - except Exception as e: - current_app.logger.error( - "Unable to send pending failure notification to " - "{}.".format(data["security_email"]), - exc_info=True, - ) - sentry.captureException() + notify_security_success = send_default_notification("failed", data, data["security_email"], pending_cert) - metrics.send( - "notification", - "counter", - 1, - metric_tags={"status": status, "event_type": "rotation"}, - ) - - if status == SUCCESS_METRIC_STATUS: - return True + return notify_owner_success or notify_security_success def needs_notification(certificate): """ - Determine if notifications for a given certificate should - currently be sent + Determine if notifications for a given certificate should currently be sent. + For each notification configured for the cert, verifies it is active, properly configured, + and that the configured expiration period is currently met. :param certificate: :return: @@ -288,9 +272,13 @@ def needs_notification(certificate): notifications = [] + print("ALPACA: Considering if cert {0} needs notifications".format(certificate)) + print("ALPACA: Notifications for {0}: {1}".format(certificate, certificate.notifications)) + for notification in certificate.notifications: + print("ALPACA: Considering if cert {0} needs notification {1}".format(certificate, notification)) if not notification.active or not notification.options: - return + continue interval = get_plugin_option("interval", notification.options) unit = get_plugin_option("unit", notification.options) @@ -309,6 +297,8 @@ def needs_notification(certificate): "Invalid base unit for expiration interval: {0}".format(unit) ) + print("ALPACA: Considering if cert {0} is applicable for notification {1}: {2} days remaining, configured as " + "{3} days".format(certificate, notification, days, interval)) if days == interval: notifications.append(notification) return notifications diff --git a/lemur/plugins/bases/notification.py b/lemur/plugins/bases/notification.py index 730f68be..0da0dad2 100644 --- a/lemur/plugins/bases/notification.py +++ b/lemur/plugins/bases/notification.py @@ -20,6 +20,15 @@ class NotificationPlugin(Plugin): def send(self, notification_type, message, targets, options, **kwargs): raise NotImplementedError + def filter_recipients(self, options, excluded_recipients): + """ + Given a set of options (which should include configured recipient info), filters out recipients that + we do NOT want to notify. + + For any notification types where recipients can't be dynamically modified, this returns an empty list. + """ + return [] + class ExpirationNotificationPlugin(NotificationPlugin): """ @@ -50,5 +59,5 @@ class ExpirationNotificationPlugin(NotificationPlugin): def options(self): return self.default_options + self.additional_options - def send(self, notification_type, message, targets, options, **kwargs): + def send(self, notification_type, message, excluded_targets, options, **kwargs): raise NotImplementedError diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8692348a..bd18fe52 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -32,13 +32,14 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Harm Weites """ + from acme.errors import ClientError from flask import current_app -from lemur.extensions import sentry, metrics -from lemur.plugins import lemur_aws as aws +from lemur.extensions import sentry, metrics +from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin -from lemur.plugins.lemur_aws import iam, s3, elb, ec2 +from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns def get_region_from_dns(dns): @@ -406,3 +407,55 @@ class S3DestinationPlugin(ExportDestinationPlugin): self.get_option("encrypt", options), account_number=self.get_option("accountNumber", options), ) + + +class SNSNotificationPlugin(ExpirationNotificationPlugin): + title = "AWS SNS" + slug = "aws-sns" + description = "Sends notifications to AWS SNS" + version = aws.VERSION + + author = "Jasmine Schladen " + author_url = "https://github.com/Netflix/lemur" + + additional_options = [ + { + "name": "accountNumber", + "type": "str", + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access the SNS topic", + }, + { + "name": "region", + "type": "str", + "required": True, + "validation": "[0-9a-z\\-]{1,25}", + "helpMessage": "Region in which the SNS topic is located, e.g. \"us-east-1\"", + }, + { + "name": "Topic Name", + "type": "str", + "required": True, + # base topic name is 1-256 characters (alphanumeric plus underscore and hyphen) + "validation": "^[a-zA-Z0-9_\\-]{1,256}$", + "helpMessage": "The name of the topic to use for expiration notifications", + } + ] + + def send(self, notification_type, message, excluded_targets, options, **kwargs): + """ + While we receive a `targets` parameter here, it is unused, as the SNS topic is pre-configured in the + plugin configuration, and can't reasonably be changed dynamically. + """ + + topic_arn = "arn:aws:sns:{0}:{1}:{2}".format(self.get_option("region", options), + self.get_option("accountNumber", options), + self.get_option("Topic Name", options)) + + current_app.logger.debug("Publishing {0} notification to topic {1}".format(notification_type, topic_arn)) + print("ALPACA: Trying to send {0} SNS notification to topic {1}".format(notification_type, topic_arn)) + try: + sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) + except Exception: + current_app.logger.exception("Error publishing {0} notification to topic {1}".format(notification_type, topic_arn)) \ No newline at end of file diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py new file mode 100644 index 00000000..6e264acb --- /dev/null +++ b/lemur/plugins/lemur_aws/sns.py @@ -0,0 +1,56 @@ +""" +.. module: lemur.plugins.lemur_aws.sts + :platform: Unix + :copyright: (c) 2020 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +import json + +import arrow +import boto3 +from flask import current_app + + +def publish(topic_arn, certificates, notification_type, **kwargs): + sns_client = boto3.client("sns", **kwargs) + print("ALPACA: SNS client: {0}, certificates: {1}".format(sns_client, certificates)) + message_ids = {} + for certificate in certificates: + message_ids[certificate["name"]] = publish_single(sns_client, topic_arn, certificate, notification_type) + + return message_ids + + +def publish_single(sns_client, topic_arn, certificate, notification_type): + response = sns_client.publish( + TopicArn=topic_arn, + Message=format_message(certificate, notification_type), + ) + + response_code = response["ResponseMetadata"]["HTTPStatusCode"] + if response_code != 200: + raise Exception("Failed to publish notification to SNS, response code was {}".format(response_code)) + + current_app.logger.debug( + "AWS SNS message published to topic [{0}]: [{1}]".format(topic_arn, response) + ) + + return response["MessageId"] + + +def create_certificate_url(name): + return "https://{hostname}/#/certificates/{name}".format( + hostname=current_app.config.get("LEMUR_HOSTNAME"), name=name + ) + + +def format_message(certificate, notification_type): + json_message = { + "notification_type": notification_type, + "certificate_name": certificate["name"], + "expires": arrow.get(certificate["validityEnd"]).format("dddd, MMMM D, YYYY"), + "endpoints_detected": len(certificate["endpoints"]), + "details": create_certificate_url(certificate["name"]) + } + return json.dumps(json_message) diff --git a/lemur/plugins/lemur_aws/tests/test_sns.py b/lemur/plugins/lemur_aws/tests/test_sns.py new file mode 100644 index 00000000..67c230f7 --- /dev/null +++ b/lemur/plugins/lemur_aws/tests/test_sns.py @@ -0,0 +1,50 @@ +from moto import mock_sts, mock_sns, mock_sqs +import boto3 +import json + +import arrow +from lemur.plugins.lemur_aws.sns import format_message +from lemur.plugins.lemur_aws.sns import publish +from lemur.certificates.schemas import certificate_notification_output_schema + +@mock_sns() +def test_format(certificate, endpoint): + + data = [certificate_notification_output_schema.dump(certificate).data] + + for certificate in data: + expected_message = { + "notification_type": "expiration", + "certificate_name": certificate["name"], + "expires": arrow.get(certificate["validityEnd"]).format("dddd, MMMM D, YYYY"), + "endpoints_detected": 0, + "details": "https://lemur.example.com/#/certificates/{name}".format(name=certificate["name"]) + } + assert expected_message == json.loads(format_message(certificate, "expiration")) + + +@mock_sns() +@mock_sqs() +def test_publish(certificate, endpoint): + + data = [certificate_notification_output_schema.dump(certificate).data] + + sns_client = boto3.client("sns", region_name="us-east-1") + topic_arn = sns_client.create_topic(Name='lemursnstest')["TopicArn"] + + sqs_client = boto3.client("sqs", region_name="us-east-1") + queue = sqs_client.create_queue(QueueName="lemursnstestqueue") + queue_url = queue["QueueUrl"] + queue_arn = sqs_client.get_queue_attributes(QueueUrl=queue_url)["Attributes"]["QueueArn"] + sns_client.subscribe(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_arn) + + message_ids = publish(topic_arn, data, "expiration", region_name="us-east-1") + assert len(message_ids) == len(data) + received_messages = sqs_client.receive_message(QueueUrl=queue_url)["Messages"] + + print("ALPACA: Received messages = {}".format(received_messages)) + + for certificate in data: + expected_message_id = message_ids[certificate["name"]] + actual_message = next((m for m in received_messages if json.loads(m["Body"])["MessageId"] == expected_message_id), None) + assert json.loads(actual_message["Body"])["Message"] == format_message(certificate, "expiration") diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 241aa1b0..b74679be 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -17,6 +17,7 @@ from lemur.plugins.bases import ExpirationNotificationPlugin from lemur.plugins import lemur_email as email from lemur.plugins.lemur_email.templates.config import env +from lemur.plugins.utils import get_plugin_option def render_html(template_name, message): @@ -105,8 +106,23 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): s_type = current_app.config.get("LEMUR_EMAIL_SENDER", "ses").lower() - if s_type == "ses": - send_via_ses(subject, body, targets) + current_app.logger.info("ALPACA: Would send an email to {0} with subject \"{1}\" here".format(targets, subject)) - elif s_type == "smtp": - send_via_smtp(subject, body, targets) + # if s_type == "ses": + # send_via_ses(subject, body, targets) + # + # elif s_type == "smtp": + # send_via_smtp(subject, body, targets) + + @staticmethod + def filter_recipients(options, excluded_recipients): + print("ALPACA: Getting recipients for notification {0}".format(options)) + notification_recipients = get_plugin_option("recipients", options) + print( + "ALPACA: Sending certificate notifications to recipients {0}".format(notification_recipients.split(","))) + if notification_recipients: + notification_recipients = notification_recipients.split(",") + # removing owner and security_email from notification_recipient + notification_recipients = [i for i in notification_recipients if i not in excluded_recipients] + + return notification_recipients diff --git a/lemur/plugins/lemur_email/tests/test_email.py b/lemur/plugins/lemur_email/tests/test_email.py index 43168cab..9555be86 100644 --- a/lemur/plugins/lemur_email/tests/test_email.py +++ b/lemur/plugins/lemur_email/tests/test_email.py @@ -34,3 +34,15 @@ def test_render(certificate, endpoint): hostname="lemur.test.example.com", ) ) + + +def test_filter_recipients(certificate, endpoint): + from lemur.plugins.lemur_email.plugin import EmailNotificationPlugin + + options = [{"name": "recipients", "value": "security@netflix.com,bob@netflix.com,joe@netflix.com"}] + assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@netflix.com", "bob@netflix.com", + "joe@netflix.com"] + assert EmailNotificationPlugin.filter_recipients(options, ["security@netflix.com"]) == ["bob@netflix.com", + "joe@netflix.com"] + assert EmailNotificationPlugin.filter_recipients(options, ["security@netflix.com", "bob@netflix.com", + "joe@netflix.com"]) == [] diff --git a/lemur/plugins/lemur_slack/plugin.py b/lemur/plugins/lemur_slack/plugin.py index 7569d295..f62ebd3f 100644 --- a/lemur/plugins/lemur_slack/plugin.py +++ b/lemur/plugins/lemur_slack/plugin.py @@ -119,6 +119,9 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): """ A typical check can be performed using the notify command: `lemur notify` + + While we receive a `targets` parameter here, it is unused, as Slack webhooks do not allow + dynamic re-targeting of messages. The webhook itself specifies a channel. """ attachments = None if notification_type == "expiration": @@ -142,6 +145,6 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): if r.status_code not in [200]: raise Exception("Failed to send message") - current_app.logger.error( + current_app.logger.info( "Slack response: {0} Message Body: {1}".format(r.status_code, body) ) diff --git a/lemur/tests/plugins/notification_plugin.py b/lemur/tests/plugins/notification_plugin.py index 4ad79704..5078e1e0 100644 --- a/lemur/tests/plugins/notification_plugin.py +++ b/lemur/tests/plugins/notification_plugin.py @@ -14,4 +14,5 @@ class TestNotificationPlugin(NotificationPlugin): @staticmethod def send(notification_type, message, targets, options, **kwargs): + print("TODO REMOVE: sending email to {}".format(targets)) return diff --git a/lemur/tests/test_messaging.py b/lemur/tests/test_messaging.py index 98e9ebf3..e5975638 100644 --- a/lemur/tests/test_messaging.py +++ b/lemur/tests/test_messaging.py @@ -87,7 +87,9 @@ def test_send_expiration_notification(certificate, notification, notification_pl delta = certificate.not_after - timedelta(days=10) with freeze_time(delta.datetime): - assert send_expiration_notifications([]) == (2, 0) + # this will only send owner and security emails (no additional recipients), + # but it executes 3 successful send attempts + assert send_expiration_notifications([]) == (3, 0) @mock_ses @@ -103,6 +105,23 @@ def test_send_expiration_notification_with_no_notifications( @mock_ses def test_send_rotation_notification(notification_plugin, certificate): + from lemur.tests.factories import UserFactory + from lemur.tests.factories import CertificateFactory from lemur.notifications.messaging import send_rotation_notification - send_rotation_notification(certificate, notification_plugin=notification_plugin) + user = UserFactory(email="jschladen@netflix.com") + + new_cert = CertificateFactory(user=user) + assert send_rotation_notification(new_cert) + + +@mock_ses +def test_send_pending_failure_notification(certificate, endpoint): + from lemur.tests.factories import UserFactory + from lemur.tests.factories import PendingCertificateFactory + from lemur.notifications.messaging import send_pending_failure_notification + + user = UserFactory(email="jschladen@netflix.com") + + pending_cert = PendingCertificateFactory(user=user) + assert send_pending_failure_notification(pending_cert) diff --git a/setup.py b/setup.py index 4da14c3d..58367f14 100644 --- a/setup.py +++ b/setup.py @@ -135,6 +135,7 @@ setup( 'aws_destination = lemur.plugins.lemur_aws.plugin:AWSDestinationPlugin', 'aws_source = lemur.plugins.lemur_aws.plugin:AWSSourcePlugin', 'aws_s3 = lemur.plugins.lemur_aws.plugin:S3DestinationPlugin', + 'aws_sns = lemur.plugins.lemur_aws.plugin:SNSNotificationPlugin', 'email_notification = lemur.plugins.lemur_email.plugin:EmailNotificationPlugin', 'slack_notification = lemur.plugins.lemur_slack.plugin:SlackNotificationPlugin', 'java_truststore_export = lemur.plugins.lemur_jks.plugin:JavaTruststoreExportPlugin', From 60bb0037f0b3bd774b935dbc7991e32f2f6386fc Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 16 Oct 2020 15:13:12 -0700 Subject: [PATCH 050/226] Miscellaneous notification fixes and tests --- lemur/notifications/messaging.py | 9 +- lemur/plugins/lemur_email/plugin.py | 9 +- .../lemur_email/templates/rotation.html | 18 ++-- lemur/plugins/lemur_email/tests/test_email.py | 89 ++++++++++++++----- lemur/plugins/lemur_slack/plugin.py | 31 +++---- lemur/plugins/lemur_slack/tests/test_slack.py | 45 ++++++++++ lemur/tests/conf.py | 2 +- lemur/tests/test_messaging.py | 15 +++- 8 files changed, 157 insertions(+), 61 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 82db7b6e..82a1ff1e 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -101,6 +101,9 @@ def send_notification(event_type, data, targets, notification): notification.plugin.send(event_type, data, targets, notification.options) status = SUCCESS_METRIC_STATUS except Exception as e: + current_app.logger.error( + "Unable to send notification to {}.".format(targets), exc_info=True + ) sentry.captureException() metrics.send( @@ -190,13 +193,13 @@ def send_rotation_notification(certificate, notification_plugin=None): status = FAILURE_METRIC_STATUS if not notification_plugin: notification_plugin = plugins.get( - current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN") + current_app.config.get("LEMUR_DEFAULT_NOTIFICATION_PLUGIN", "email-notification") ) data = certificate_notification_output_schema.dump(certificate).data try: - notification_plugin.send("rotation", data, [data["owner"]]) + notification_plugin.send("rotation", data, [data["owner"]], []) status = SUCCESS_METRIC_STATUS except Exception as e: current_app.logger.error( @@ -290,7 +293,7 @@ def needs_notification(certificate): for notification in certificate.notifications: if not notification.active or not notification.options: - return + continue interval = get_plugin_option("interval", notification.options) unit = get_plugin_option("unit", notification.options) diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 241aa1b0..08332ef1 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -19,14 +19,16 @@ from lemur.plugins import lemur_email as email from lemur.plugins.lemur_email.templates.config import env -def render_html(template_name, message): +def render_html(template_name, options, certificates): """ Renders the html for our email notification. :param template_name: - :param message: + :param options: + :param certificates: :return: """ + message = {"options": options, "certificates": certificates} template = env.get_template("{}.html".format(template_name)) return template.render( dict(message=message, hostname=current_app.config.get("LEMUR_HOSTNAME")) @@ -100,8 +102,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): subject = "Lemur: {0} Notification".format(notification_type.capitalize()) - data = {"options": options, "certificates": message} - body = render_html(notification_type, data) + body = render_html(notification_type, options, message) s_type = current_app.config.get("LEMUR_EMAIL_SENDER", "ses").lower() diff --git a/lemur/plugins/lemur_email/templates/rotation.html b/lemur/plugins/lemur_email/templates/rotation.html index 521eb327..9ce7ff33 100644 --- a/lemur/plugins/lemur_email/templates/rotation.html +++ b/lemur/plugins/lemur_email/templates/rotation.html @@ -83,12 +83,12 @@ - {{ certificate.name }} + {{ message.certificates.name }}
-
{{ certificate.owner }} -
{{ certificate.validityEnd | time }} - Details +
{{ message.certificates.owner }} +
{{ message.certificates.validityEnd | time }} + Details
@@ -110,12 +110,12 @@ - {{ certificate.replacedBy[0].name }} + {{ message.certificates.name }}
-
{{ certificate.replacedBy[0].owner }} -
{{ certificate.replacedBy[0].validityEnd | time }} - Details +
{{ message.certificates.owner }} +
{{ message.certificates.validityEnd | time }} + Details
@@ -133,7 +133,7 @@ - {% for endpoint in certificate.endpoints %} + {% for endpoint in message.certificates.endpoints %} diff --git a/lemur/plugins/lemur_email/tests/test_email.py b/lemur/plugins/lemur_email/tests/test_email.py index 43168cab..4f1ea187 100644 --- a/lemur/plugins/lemur_email/tests/test_email.py +++ b/lemur/plugins/lemur_email/tests/test_email.py @@ -1,36 +1,83 @@ import os -from lemur.plugins.lemur_email.templates.config import env +from datetime import timedelta +import arrow +import boto3 +from moto import mock_ses + +from lemur.certificates.schemas import certificate_notification_output_schema +from lemur.plugins.lemur_email.plugin import render_html from lemur.tests.factories import CertificateFactory dir_path = os.path.dirname(os.path.realpath(__file__)) -def test_render(certificate, endpoint): - from lemur.certificates.schemas import certificate_notification_output_schema +@mock_ses +def verify_sender_email(): + ses_client = boto3.client("ses", region_name="us-east-1") + ses_client.verify_email_identity(EmailAddress="lemur@example.com") + + +def get_options(): + return [ + {"name": "interval", "value": 10}, + {"name": "unit", "value": "days"}, + ] + + +def test_render_expiration(certificate, endpoint): new_cert = CertificateFactory() new_cert.replaces.append(certificate) - data = { - "certificates": [certificate_notification_output_schema.dump(certificate).data], - "options": [ - {"name": "interval", "value": 10}, - {"name": "unit", "value": "days"}, - ], - } + assert render_html("expiration", get_options(), [certificate_notification_output_schema.dump(certificate).data]) - template = env.get_template("{}.html".format("expiration")) - - body = template.render(dict(message=data, hostname="lemur.test.example.com")) - - template = env.get_template("{}.html".format("rotation")) +def test_render_rotation(certificate, endpoint): certificate.endpoints.append(endpoint) - body = template.render( - dict( - certificate=certificate_notification_output_schema.dump(certificate).data, - hostname="lemur.test.example.com", - ) - ) + assert render_html("rotation", get_options(), certificate_notification_output_schema.dump(certificate).data) + + +def test_render_rotation_failure(pending_certificate): + assert render_html("failed", get_options(), certificate_notification_output_schema.dump(pending_certificate).data) + + +@mock_ses +def test_send_expiration_notification(): + from lemur.notifications.messaging import send_expiration_notifications + from lemur.tests.factories import CertificateFactory + from lemur.tests.factories import NotificationFactory + + now = arrow.utcnow() + in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future + certificate = CertificateFactory() + notification = NotificationFactory(plugin_name="email-notification") + + certificate.not_after = in_ten_days + certificate.notifications.append(notification) + certificate.notifications[0].options = get_options() + + verify_sender_email() + assert send_expiration_notifications([]) == (2, 0) + + +@mock_ses +def test_send_rotation_notification(endpoint, source_plugin): + from lemur.notifications.messaging import send_rotation_notification + from lemur.deployment.service import rotate_certificate + + new_certificate = CertificateFactory() + rotate_certificate(endpoint, new_certificate) + assert endpoint.certificate == new_certificate + + verify_sender_email() + assert send_rotation_notification(new_certificate) + + +@mock_ses +def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin): + from lemur.notifications.messaging import send_pending_failure_notification + + verify_sender_email() + assert send_pending_failure_notification(pending_certificate) diff --git a/lemur/plugins/lemur_slack/plugin.py b/lemur/plugins/lemur_slack/plugin.py index 7569d295..67c3fd84 100644 --- a/lemur/plugins/lemur_slack/plugin.py +++ b/lemur/plugins/lemur_slack/plugin.py @@ -58,26 +58,19 @@ def create_rotation_attachments(certificate): "title": certificate["name"], "title_link": create_certificate_url(certificate["name"]), "fields": [ + {"title": "Owner", "value": certificate["owner"], "short": True}, { - {"title": "Owner", "value": certificate["owner"], "short": True}, - { - "title": "Expires", - "value": arrow.get(certificate["validityEnd"]).format( - "dddd, MMMM D, YYYY" - ), - "short": True, - }, - { - "title": "Replaced By", - "value": len(certificate["replaced"][0]["name"]), - "short": True, - }, - { - "title": "Endpoints Rotated", - "value": len(certificate["endpoints"]), - "short": True, - }, - } + "title": "Expires", + "value": arrow.get(certificate["validityEnd"]).format( + "dddd, MMMM D, YYYY" + ), + "short": True, + }, + { + "title": "Endpoints Rotated", + "value": len(certificate["endpoints"]), + "short": True, + }, ], } diff --git a/lemur/plugins/lemur_slack/tests/test_slack.py b/lemur/plugins/lemur_slack/tests/test_slack.py index 86add25f..77abd542 100644 --- a/lemur/plugins/lemur_slack/tests/test_slack.py +++ b/lemur/plugins/lemur_slack/tests/test_slack.py @@ -21,3 +21,48 @@ def test_formatting(certificate): } assert attachment == create_expiration_attachments(data)[0] + + +def get_options(): + return [ + {"name": "interval", "value": 10}, + {"name": "unit", "value": "days"}, + ] + + +# Currently disabled as we have no good way to mock Slack webhooks +# def test_send_expiration_notification(): +# from lemur.notifications.messaging import send_expiration_notifications +# from lemur.tests.factories import CertificateFactory +# +# now = arrow.utcnow() +# in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future +# certificate = CertificateFactory() +# notification = NotificationFactory(plugin_name="slack-notification") +# +# certificate.not_after = in_ten_days +# certificate.notifications.append(notification) +# certificate.notifications[0].options = get_options() +# +# assert send_expiration_notifications([]) == (2, 0) + + +# Currently disabled as we have no good way to mock Slack webhooks +# def test_send_rotation_notification(endpoint, source_plugin): +# from lemur.notifications.messaging import send_rotation_notification +# from lemur.deployment.service import rotate_certificate +# +# notification = NotificationFactory(plugin_name="slack-notification") +# +# new_certificate = CertificateFactory() +# rotate_certificate(endpoint, new_certificate) +# assert endpoint.certificate == new_certificate +# +# assert send_rotation_notification(new_certificate, notification_plugin=notification.plugin) + + +# Currently disabled as the Slack plugin doesn't support this type of notification +# def test_send_pending_failure_notification(user, pending_certificate, async_issuer_plugin): +# from lemur.notifications.messaging import send_pending_failure_notification +# +# assert send_pending_failure_notification(pending_certificate, notification_plugin=plugins.get("slack-notification")) diff --git a/lemur/tests/conf.py b/lemur/tests/conf.py index 38b8bade..3dfb5621 100644 --- a/lemur/tests/conf.py +++ b/lemur/tests/conf.py @@ -46,7 +46,7 @@ LEMUR_ALLOWED_DOMAINS = [ # Lemur currently only supports SES for sending email, this address # needs to be verified -LEMUR_EMAIL = "" +LEMUR_EMAIL = "lemur@example.com" LEMUR_SECURITY_TEAM_EMAIL = ["security@example.com"] LEMUR_HOSTNAME = "lemur.example.com" diff --git a/lemur/tests/test_messaging.py b/lemur/tests/test_messaging.py index 98e9ebf3..dd8f339f 100644 --- a/lemur/tests/test_messaging.py +++ b/lemur/tests/test_messaging.py @@ -1,8 +1,8 @@ +from datetime import timedelta + +import arrow import pytest from freezegun import freeze_time - -from datetime import timedelta -import arrow from moto import mock_ses @@ -105,4 +105,11 @@ def test_send_expiration_notification_with_no_notifications( def test_send_rotation_notification(notification_plugin, certificate): from lemur.notifications.messaging import send_rotation_notification - send_rotation_notification(certificate, notification_plugin=notification_plugin) + assert send_rotation_notification(certificate, notification_plugin=notification_plugin) + + +@mock_ses +def test_send_pending_failure_notification(notification_plugin, async_issuer_plugin, pending_certificate): + from lemur.notifications.messaging import send_pending_failure_notification + + assert send_pending_failure_notification(pending_certificate, notification_plugin=notification_plugin) From 072b337f37487a5750e339ffdf05100aec24162c Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 16 Oct 2020 16:21:43 -0700 Subject: [PATCH 051/226] Restructure log messages --- lemur/notifications/messaging.py | 54 ++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 82a1ff1e..78809dbc 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -8,6 +8,7 @@ .. moduleauthor:: Kevin Glisson """ +import sys from collections import defaultdict from datetime import timedelta from itertools import groupby @@ -96,14 +97,20 @@ def send_notification(event_type, data, targets, notification): :param notification: :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "Sending expiration notification for to targets {}".format(targets), + "notification_type": "rotation", + "targets": targets, + } status = FAILURE_METRIC_STATUS try: notification.plugin.send(event_type, data, targets, notification.options) status = SUCCESS_METRIC_STATUS except Exception as e: - current_app.logger.error( - "Unable to send notification to {}.".format(targets), exc_info=True - ) + log_data["message"] = "Unable to send expiration notification to targets {}".format(targets) + current_app.logger.error(log_data, exc_info=True) sentry.captureException() metrics.send( @@ -190,6 +197,14 @@ def send_rotation_notification(certificate, notification_plugin=None): :param notification_plugin: :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "Sending rotation notification for certificate {}".format(certificate.name), + "notification_type": "rotation", + "name": certificate.name, + "owner": certificate.owner, + } status = FAILURE_METRIC_STATUS if not notification_plugin: notification_plugin = plugins.get( @@ -202,9 +217,9 @@ def send_rotation_notification(certificate, notification_plugin=None): notification_plugin.send("rotation", data, [data["owner"]], []) status = SUCCESS_METRIC_STATUS except Exception as e: - current_app.logger.error( - "Unable to send notification to {}.".format(data["owner"]), exc_info=True - ) + log_data["message"] = "Unable to send rotation notification for certificate {0} to ownner {1}" \ + .format(certificate.name, data["owner"]) + current_app.logger.error(log_data) sentry.captureException() metrics.send( @@ -228,6 +243,14 @@ def send_pending_failure_notification( :param notification_plugin: :return: """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "Sending pending failure notification for pending certificate {}".format(pending_cert.name), + "notification_type": "rotation", + "name": pending_cert.name, + "owner": pending_cert.owner, + } status = FAILURE_METRIC_STATUS if not notification_plugin: @@ -245,12 +268,10 @@ def send_pending_failure_notification( notification_plugin.send("failed", data, [data["owner"]], pending_cert) status = SUCCESS_METRIC_STATUS except Exception as e: - current_app.logger.error( - "Unable to send pending failure notification to {}.".format( - data["owner"] - ), - exc_info=True, - ) + log_data["recipient"] = data["owner"] + log_data["message"] = "Unable to send pending failure notification for certificate {0} to owner {1}" \ + .format(pending_cert.name, pending_cert.owner) + current_app.logger.error(log_data, exc_info=True) sentry.captureException() if notify_security: @@ -260,11 +281,10 @@ def send_pending_failure_notification( ) status = SUCCESS_METRIC_STATUS except Exception as e: - current_app.logger.error( - "Unable to send pending failure notification to " - "{}.".format(data["security_email"]), - exc_info=True, - ) + log_data["recipient"] = data["security_email"] + log_data["message"] = "Unable to send pending failure notification for certificate {0} to security email " \ + "{1}".format(pending_cert.name, pending_cert.owner) + current_app.logger.error(log_data, exc_info=True) sentry.captureException() metrics.send( From 6a1889787dc2d5df5836a9f19ca2b503c80515b5 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 16 Oct 2020 16:30:21 -0700 Subject: [PATCH 052/226] Correct log attributes --- lemur/notifications/messaging.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 78809dbc..5452f4fc 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -101,8 +101,8 @@ def send_notification(event_type, data, targets, notification): log_data = { "function": function, "message": "Sending expiration notification for to targets {}".format(targets), - "notification_type": "rotation", - "targets": targets, + "notification_type": "expiration", + "certificate_targets": targets, } status = FAILURE_METRIC_STATUS try: @@ -202,8 +202,8 @@ def send_rotation_notification(certificate, notification_plugin=None): "function": function, "message": "Sending rotation notification for certificate {}".format(certificate.name), "notification_type": "rotation", - "name": certificate.name, - "owner": certificate.owner, + "certificate_name": certificate.name, + "certificate_owner": certificate.owner, } status = FAILURE_METRIC_STATUS if not notification_plugin: @@ -247,9 +247,9 @@ def send_pending_failure_notification( log_data = { "function": function, "message": "Sending pending failure notification for pending certificate {}".format(pending_cert.name), - "notification_type": "rotation", - "name": pending_cert.name, - "owner": pending_cert.owner, + "notification_type": "failed", + "certificate_name": pending_cert.name, + "certificate_owner": pending_cert.owner, } status = FAILURE_METRIC_STATUS From e90b08b3633c7a277da84ad19cde45324ca9bf5a Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 16 Oct 2020 17:08:44 -0700 Subject: [PATCH 053/226] Correct typo and enable Slack notification test --- lemur/notifications/messaging.py | 4 +- lemur/plugins/lemur_slack/tests/test_slack.py | 40 +++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 5452f4fc..aa85123d 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -217,9 +217,9 @@ def send_rotation_notification(certificate, notification_plugin=None): notification_plugin.send("rotation", data, [data["owner"]], []) status = SUCCESS_METRIC_STATUS except Exception as e: - log_data["message"] = "Unable to send rotation notification for certificate {0} to ownner {1}" \ + log_data["message"] = "Unable to send rotation notification for certificate {0} to owner {1}" \ .format(certificate.name, data["owner"]) - current_app.logger.error(log_data) + current_app.logger.error(log_data, exc_info=True) sentry.captureException() metrics.send( diff --git a/lemur/plugins/lemur_slack/tests/test_slack.py b/lemur/plugins/lemur_slack/tests/test_slack.py index 77abd542..da232d61 100644 --- a/lemur/plugins/lemur_slack/tests/test_slack.py +++ b/lemur/plugins/lemur_slack/tests/test_slack.py @@ -1,3 +1,10 @@ +from datetime import timedelta + +import arrow + +from lemur.tests.factories import NotificationFactory, CertificateFactory + + def test_formatting(certificate): from lemur.plugins.lemur_slack.plugin import create_expiration_attachments from lemur.certificates.schemas import certificate_notification_output_schema @@ -27,32 +34,33 @@ def get_options(): return [ {"name": "interval", "value": 10}, {"name": "unit", "value": "days"}, + {"name": "webhook", "value": "https://slack.com/api/api.test"}, ] -# Currently disabled as we have no good way to mock Slack webhooks -# def test_send_expiration_notification(): -# from lemur.notifications.messaging import send_expiration_notifications -# from lemur.tests.factories import CertificateFactory -# -# now = arrow.utcnow() -# in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future -# certificate = CertificateFactory() -# notification = NotificationFactory(plugin_name="slack-notification") -# -# certificate.not_after = in_ten_days -# certificate.notifications.append(notification) -# certificate.notifications[0].options = get_options() -# -# assert send_expiration_notifications([]) == (2, 0) +def test_send_expiration_notification(): + from lemur.notifications.messaging import send_expiration_notifications + + notification = NotificationFactory(plugin_name="slack-notification") + notification.options = get_options() + + now = arrow.utcnow() + in_ten_days = now + timedelta(days=10, hours=1) # a bit more than 10 days since we'll check in the future + + certificate = CertificateFactory() + certificate.not_after = in_ten_days + certificate.notifications.append(notification) + + assert send_expiration_notifications([]) == (2, 0) -# Currently disabled as we have no good way to mock Slack webhooks +# Currently disabled as the Slack plugin doesn't support this type of notification # def test_send_rotation_notification(endpoint, source_plugin): # from lemur.notifications.messaging import send_rotation_notification # from lemur.deployment.service import rotate_certificate # # notification = NotificationFactory(plugin_name="slack-notification") +# notification.options = get_options() # # new_certificate = CertificateFactory() # rotate_certificate(endpoint, new_certificate) From 591c8cf5249cb5d8b1b075f8b0c52863795913d0 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Mon, 19 Oct 2020 22:03:43 +0200 Subject: [PATCH 054/226] Do not add urlContextPath to relative path --- gulp/build.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gulp/build.js b/gulp/build.js index eed59503..5aca8094 100644 --- a/gulp/build.js +++ b/gulp/build.js @@ -237,7 +237,7 @@ gulp.task('addUrlContextPath',['addUrlContextPath:revreplace'], function(){ .forEach(function(file){ return gulp.src(file) .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/'))) - .pipe(gulpif(urlContextPathExists, replace('angular/', argv.urlContextPath + '/angular/'))) + .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/'))) .pipe(gulp.dest(function(file){ return file.base; })) @@ -256,10 +256,9 @@ gulp.task('addUrlContextPath:revreplace', ['addUrlContextPath:revision'], functi var manifest = gulp.src("lemur/static/dist/rev-manifest.json"); var urlContextPathExists = argv.urlContextPath ? true : false; return gulp.src( "lemur/static/dist/index.html") - .pipe(gulpif(urlContextPathExists, revReplace({prefix: argv.urlContextPath + '/', manifest: manifest}, revReplace({manifest: manifest})))) .pipe(gulp.dest('lemur/static/dist')); }) gulp.task('build', ['build:ngviews', 'build:inject', 'build:images', 'build:fonts', 'build:html', 'build:extras']); -gulp.task('package', ['addUrlContextPath', 'package:strip']); \ No newline at end of file +gulp.task('package', ['addUrlContextPath', 'package:strip']); From ecd4d6ebe3489cb9f2ad7988eb3eb373fb053714 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Mon, 19 Oct 2020 15:12:48 -0700 Subject: [PATCH 055/226] Change string formatting pattern --- lemur/notifications/messaging.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index aa85123d..6c8599aa 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -100,7 +100,7 @@ def send_notification(event_type, data, targets, notification): function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, - "message": "Sending expiration notification for to targets {}".format(targets), + "message": f"Sending expiration notification for to targets {targets}", "notification_type": "expiration", "certificate_targets": targets, } @@ -109,7 +109,7 @@ def send_notification(event_type, data, targets, notification): notification.plugin.send(event_type, data, targets, notification.options) status = SUCCESS_METRIC_STATUS except Exception as e: - log_data["message"] = "Unable to send expiration notification to targets {}".format(targets) + log_data["message"] = f"Unable to send expiration notification to targets {targets}" current_app.logger.error(log_data, exc_info=True) sentry.captureException() @@ -200,7 +200,7 @@ def send_rotation_notification(certificate, notification_plugin=None): function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, - "message": "Sending rotation notification for certificate {}".format(certificate.name), + "message": f"Sending rotation notification for certificate {certificate.name}", "notification_type": "rotation", "certificate_name": certificate.name, "certificate_owner": certificate.owner, @@ -217,8 +217,8 @@ def send_rotation_notification(certificate, notification_plugin=None): notification_plugin.send("rotation", data, [data["owner"]], []) status = SUCCESS_METRIC_STATUS except Exception as e: - log_data["message"] = "Unable to send rotation notification for certificate {0} to owner {1}" \ - .format(certificate.name, data["owner"]) + log_data["message"] = f"Unable to send rotation notification for certificate {certificate.name} " \ + f"to owner {data['owner']}" current_app.logger.error(log_data, exc_info=True) sentry.captureException() @@ -246,7 +246,7 @@ def send_pending_failure_notification( function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, - "message": "Sending pending failure notification for pending certificate {}".format(pending_cert.name), + "message": f"Sending pending failure notification for pending certificate {pending_cert}" "notification_type": "failed", "certificate_name": pending_cert.name, "certificate_owner": pending_cert.owner, @@ -269,8 +269,8 @@ def send_pending_failure_notification( status = SUCCESS_METRIC_STATUS except Exception as e: log_data["recipient"] = data["owner"] - log_data["message"] = "Unable to send pending failure notification for certificate {0} to owner {1}" \ - .format(pending_cert.name, pending_cert.owner) + log_data["message"] = f"Unable to send pending failure notification for certificate {pending_cert.name} " \ + f"to owner {pending_cert.owner}" current_app.logger.error(log_data, exc_info=True) sentry.captureException() @@ -282,8 +282,8 @@ def send_pending_failure_notification( status = SUCCESS_METRIC_STATUS except Exception as e: log_data["recipient"] = data["security_email"] - log_data["message"] = "Unable to send pending failure notification for certificate {0} to security email " \ - "{1}".format(pending_cert.name, pending_cert.owner) + log_data["message"] = f"Unable to send pending failure notification for certificate {pending_cert.name} " \ + f"to security email {pending_cert.owner}" current_app.logger.error(log_data, exc_info=True) sentry.captureException() @@ -291,7 +291,7 @@ def send_pending_failure_notification( "notification", "counter", 1, - metric_tags={"status": status, "event_type": "rotation"}, + metric_tags={"status": status, "event_type": "failed"}, ) if status == SUCCESS_METRIC_STATUS: @@ -329,7 +329,7 @@ def needs_notification(certificate): else: raise Exception( - "Invalid base unit for expiration interval: {0}".format(unit) + f"Invalid base unit for expiration interval: {unit}" ) if days == interval: From b5f0fc5a195f464885a3f808edcfe4f95b054f4d Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Mon, 19 Oct 2020 15:21:34 -0700 Subject: [PATCH 056/226] Fix syntax error --- lemur/notifications/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 6c8599aa..ca955b69 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -246,7 +246,7 @@ def send_pending_failure_notification( function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, - "message": f"Sending pending failure notification for pending certificate {pending_cert}" + "message": f"Sending pending failure notification for pending certificate {pending_cert}", "notification_type": "failed", "certificate_name": pending_cert.name, "certificate_owner": pending_cert.owner, From 855baadfee15cf40610603c79fc6c8ead0164999 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 16 Oct 2020 17:36:32 -0700 Subject: [PATCH 057/226] Show only few supported ECC algorithms on UI --- .../angular/authorities/authority/options.tpl.html | 13 +++++++++++-- .../certificates/certificate/options.tpl.html | 12 ++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lemur/static/app/angular/authorities/authority/options.tpl.html b/lemur/static/app/angular/authorities/authority/options.tpl.html index bf1ad70c..01928fc3 100644 --- a/lemur/static/app/angular/authorities/authority/options.tpl.html +++ b/lemur/static/app/angular/authorities/authority/options.tpl.html @@ -20,8 +20,17 @@ Key Type
- +
diff --git a/lemur/static/app/angular/certificates/certificate/options.tpl.html b/lemur/static/app/angular/certificates/certificate/options.tpl.html index 7e6ad428..2c02c693 100644 --- a/lemur/static/app/angular/certificates/certificate/options.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/options.tpl.html @@ -32,10 +32,14 @@
From 58798fbc2ee97d949765364dbc700eeff149167a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2020 09:21:21 +0000 Subject: [PATCH 058/226] Bump fakeredis from 1.4.3 to 1.4.4 Bumps [fakeredis](https://github.com/jamesls/fakeredis) from 1.4.3 to 1.4.4. - [Release notes](https://github.com/jamesls/fakeredis/releases) - [Commits](https://github.com/jamesls/fakeredis/compare/1.4.3...1.4.4) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 8df4f5d1..51553d1d 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in faker==4.14.0 # via -r requirements-tests.in, factory-boy -fakeredis==1.4.3 # via -r requirements-tests.in +fakeredis==1.4.4 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in future==0.18.2 # via aws-xray-sdk From a3b90c1a6b1a118771a8498c33d99efb9346c2b2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2020 09:23:10 +0000 Subject: [PATCH 059/226] Bump botocore from 1.18.16 to 1.18.18 Bumps [botocore](https://github.com/boto/botocore) from 1.18.16 to 1.18.18. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.18.16...1.18.18) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 38b62198..24504233 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,7 +18,7 @@ beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.15.16 # via -r requirements.txt -botocore==1.18.16 # via -r requirements.txt, boto3, s3transfer +botocore==1.18.18 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 8df4f5d1..94bdabc4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in boto3==1.15.16 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.18.16 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.18.18 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index d323b40f..0949fbb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.15.16 # via -r requirements.in -botocore==1.18.16 # via -r requirements.in, boto3, s3transfer +botocore==1.18.18 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 4f552cb636252c77054879cc736555321084ff8c Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Tue, 20 Oct 2020 12:02:36 -0700 Subject: [PATCH 060/226] Code cleanup --- docs/administration.rst | 2 +- lemur/notifications/messaging.py | 4 ++-- lemur/plugins/lemur_aws/sns.py | 2 +- lemur/plugins/lemur_email/plugin.py | 10 ++++------ lemur/plugins/lemur_email/tests/test_email.py | 14 +++++++------- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index ee504865..ef3f8e38 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -1462,7 +1462,7 @@ AWS (Destination) Uses AWS IAM as a destination for Lemur generated certificates. Support a multi-account deployment. -AWS (Notification) +AWS (SNS Notification) ----- :Authors: diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 51c9f18a..1fce7636 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -107,6 +107,7 @@ def send_plugin_notification(event_type, data, recipients, notification): } status = FAILURE_METRIC_STATUS try: + current_app.logger.debug(log_data) notification.plugin.send(event_type, data, recipients, notification.options) status = SUCCESS_METRIC_STATUS except Exception as e: @@ -203,6 +204,7 @@ def send_default_notification(notification_type, data, targets, notification_opt ) try: + current_app.logger.debug(log_data) # we need the notification.options here because the email templates utilize the interval/unit info notification_plugin.send(notification_type, data, targets, notification_options) status = SUCCESS_METRIC_STATUS @@ -288,8 +290,6 @@ def needs_notification(certificate): raise Exception( f"Invalid base unit for expiration interval: {unit}" ) - print(f"Does cert {certificate.name} need a notification {notification.label}? Actual: {days}, " - f"configured: {interval}") # TODO REMOVE if days == interval: notifications.append(notification) return notifications diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index 96c44f28..3aeb14da 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -3,7 +3,7 @@ :platform: Unix :copyright: (c) 2020 by Netflix Inc., see AUTHORS for more :license: Apache, see LICENSE for more details. -.. moduleauthor:: Kevin Glisson +.. moduleauthor:: Jasmine Schladen """ import json diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index 62e6b2d4..5b9c188e 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -107,13 +107,11 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): s_type = current_app.config.get("LEMUR_EMAIL_SENDER", "ses").lower() - print(f"Would send {s_type} email to {targets}: {subject}") + if s_type == "ses": + send_via_ses(subject, body, targets) -# if s_type == "ses": - # send_via_ses(subject, body, targets) - - # elif s_type == "smtp": - # send_via_smtp(subject, body, targets) + elif s_type == "smtp": + send_via_smtp(subject, body, targets) @staticmethod def filter_recipients(options, excluded_recipients, **kwargs): diff --git a/lemur/plugins/lemur_email/tests/test_email.py b/lemur/plugins/lemur_email/tests/test_email.py index d7f7a17d..fd4dc575 100644 --- a/lemur/plugins/lemur_email/tests/test_email.py +++ b/lemur/plugins/lemur_email/tests/test_email.py @@ -81,10 +81,10 @@ def test_send_pending_failure_notification(user, pending_certificate, async_issu def test_filter_recipients(certificate, endpoint): from lemur.plugins.lemur_email.plugin import EmailNotificationPlugin - options = [{"name": "recipients", "value": "security@netflix.com,bob@netflix.com,joe@netflix.com"}] - assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@netflix.com", "bob@netflix.com", - "joe@netflix.com"] - assert EmailNotificationPlugin.filter_recipients(options, ["security@netflix.com"]) == ["bob@netflix.com", - "joe@netflix.com"] - assert EmailNotificationPlugin.filter_recipients(options, ["security@netflix.com", "bob@netflix.com", - "joe@netflix.com"]) == [] + options = [{"name": "recipients", "value": "security@example.com,bob@example.com,joe@example.com"}] + assert EmailNotificationPlugin.filter_recipients(options, []) == ["security@example.com", "bob@example.com", + "joe@example.com"] + assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com"]) == ["bob@example.com", + "joe@example.com"] + assert EmailNotificationPlugin.filter_recipients(options, ["security@example.com", "bob@example.com", + "joe@example.com"]) == [] From 788703ce12b9e3783fe6cd3ce00d3ebf98caf9e3 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 20 Oct 2020 16:43:57 -0700 Subject: [PATCH 061/226] Fix cert reissue when L/OU is not set get_certificate_primitives complains with None L/OU --- lemur/certificates/schemas.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index cc0a607e..77f49c9b 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -353,6 +353,12 @@ class CertificateOutputSchema(LemurOutputSchema): data.pop("organization", None) data.pop("organizational_unit", None) + # Removing optional fields if None, else it complains in de-serialization + if "location" in data and data["location"] is None: + data.pop("location") + if "organizational_unit" in data and data["organizational_unit"] is None: + data.pop("organizational_unit") + class CertificateShortOutputSchema(LemurOutputSchema): id = fields.Integer() From 01dddd2a557286cbf8ecf1229ec5ed51518fd65f Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 20 Oct 2020 17:17:28 -0700 Subject: [PATCH 062/226] iterate over subject details --- lemur/certificates/schemas.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 77f49c9b..3dc864e7 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -340,6 +340,8 @@ class CertificateOutputSchema(LemurOutputSchema): @post_dump def handle_subject_details(self, data): + subject_details = ["country", "state", "location", "organization", "organizational_unit"] + # Remove subject details if authority is CA/Browser Forum compliant. The code will use default set of values in that case. # If CA/Browser Forum compliance of an authority is unknown (None), it is safe to fallback to default values. Thus below # condition checks for 'not False' ==> 'True or None' @@ -347,17 +349,13 @@ class CertificateOutputSchema(LemurOutputSchema): is_cab_compliant = data.get("authority").get("isCabCompliant") if is_cab_compliant is not False: - data.pop("country", None) - data.pop("state", None) - data.pop("location", None) - data.pop("organization", None) - data.pop("organizational_unit", None) + for field in subject_details: + data.pop(field, None) - # Removing optional fields if None, else it complains in de-serialization - if "location" in data and data["location"] is None: - data.pop("location") - if "organizational_unit" in data and data["organizational_unit"] is None: - data.pop("organizational_unit") + # Removing subject fields if None, else it complains in de-serialization + for field in subject_details: + if field in data and data[field] is None: + data.pop(field) class CertificateShortOutputSchema(LemurOutputSchema): From 49971652351ec487961234c353cad0ed62158984 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 20 Oct 2020 17:59:50 -0700 Subject: [PATCH 063/226] Removing ECC 192 and 521 from UI not CAB supported. Keeping 521 for authority --- .../static/app/angular/authorities/authority/options.tpl.html | 1 - .../app/angular/certificates/certificate/options.tpl.html | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/lemur/static/app/angular/authorities/authority/options.tpl.html b/lemur/static/app/angular/authorities/authority/options.tpl.html index 01928fc3..adf8eacc 100644 --- a/lemur/static/app/angular/authorities/authority/options.tpl.html +++ b/lemur/static/app/angular/authorities/authority/options.tpl.html @@ -24,7 +24,6 @@ ng-options="option.value as option.name for option in [ {'name': 'RSA-2048', 'value': 'RSA2048'}, {'name': 'RSA-4096', 'value': 'RSA4096'}, - {'name': 'ECC-PRIME192V1', 'value': 'ECCPRIME192V1'}, {'name': 'ECC-PRIME256V1', 'value': 'ECCPRIME256V1'}, {'name': 'ECC-SECP384R1', 'value': 'ECCSECP384R1'}, {'name': 'ECC-SECP521R1', 'value': 'ECCSECP521R1'}]" diff --git a/lemur/static/app/angular/certificates/certificate/options.tpl.html b/lemur/static/app/angular/certificates/certificate/options.tpl.html index 2c02c693..11b8fe68 100644 --- a/lemur/static/app/angular/certificates/certificate/options.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/options.tpl.html @@ -35,10 +35,8 @@ ng-options="option.value as option.name for option in [ {'name': 'RSA-2048', 'value': 'RSA2048'}, {'name': 'RSA-4096', 'value': 'RSA4096'}, - {'name': 'ECC-PRIME192V1', 'value': 'ECCPRIME192V1'}, {'name': 'ECC-PRIME256V1', 'value': 'ECCPRIME256V1'}, - {'name': 'ECC-SECP384R1', 'value': 'ECCSECP384R1'}, - {'name': 'ECC-SECP521R1', 'value': 'ECCSECP521R1'}]" + {'name': 'ECC-SECP384R1', 'value': 'ECCSECP384R1'}]" ng-init="certificate.keyType = 'RSA2048'"> From 9374adaa4632ed4e477f875cf036f8bb364d7707 Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Wed, 21 Oct 2020 08:45:12 +0200 Subject: [PATCH 064/226] do not create db_upgrade.log during migrations --- lemur/migrations/versions/c301c59688d2_.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 669c934f..8712d60c 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -35,11 +35,11 @@ from lemur.common import utils import time import datetime -log_file = open('db_upgrade.log', 'a') - +import logging +log = logging.getLogger(__name__) def upgrade(): - log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) + log.info("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) start_time = time.time() # Update RSA keys using the key length information @@ -50,8 +50,7 @@ def upgrade(): # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs. update_key_type() - log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time)) - log_file.close() + log.info("--- Total %s seconds ---\n" % (time.time() - start_time)) def downgrade(): @@ -69,18 +68,18 @@ def downgrade(): def update_key_type_rsa(bits): - log_file.write("Processing certificate with key type RSA %s\n" % bits) + log.info("Processing certificate with key type RSA %s\n" % bits) stmt = text( f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null" ) - log_file.write("Query: %s\n" % stmt) + log.info("Query: %s\n" % stmt) start_time = time.time() op.execute(stmt) commit() - log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + log.info("--- %s seconds ---\n" % (time.time() - start_time)) def update_key_type(): @@ -95,9 +94,9 @@ def update_key_type(): try: cert_key_type = utils.get_key_type_from_certificate(body) except ValueError as e: - log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) + log.info("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) else: - log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) stmt = text( "update certificates set key_type=:key_type where id=:id" ) @@ -106,7 +105,7 @@ def update_key_type(): commit() - log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + log.info("--- %s seconds ---\n" % (time.time() - start_time)) def commit(): From 757e190b6094966ff16113d2e82b5677ca8bb025 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 21 Oct 2020 12:11:41 -0700 Subject: [PATCH 065/226] Check if OU and L is present in subject fixing index out of range --- lemur/common/defaults.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index b9c88e49..d94c3563 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -110,9 +110,11 @@ def organizational_unit(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME)[ - 0 - ].value.strip() + ou = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATIONAL_UNIT_NAME) + if not ou: + return None + + return ou[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get organizational unit! {0}".format(e)) @@ -155,9 +157,11 @@ def location(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME)[ - 0 - ].value.strip() + loc = cert.subject.get_attributes_for_oid(x509.OID_LOCALITY_NAME) + if not loc: + return None + + return loc[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get location! {0}".format(e)) From 43483cb1c7c6c29bfe0dde757e501faecd31a493 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 21 Oct 2020 15:11:10 -0700 Subject: [PATCH 066/226] Check if present - Organization, State, Country --- lemur/common/defaults.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lemur/common/defaults.py b/lemur/common/defaults.py index d94c3563..d7b37292 100644 --- a/lemur/common/defaults.py +++ b/lemur/common/defaults.py @@ -95,9 +95,11 @@ def organization(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)[ - 0 - ].value.strip() + o = cert.subject.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME) + if not o: + return None + + return o[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get organization! {0}".format(e)) @@ -127,9 +129,11 @@ def country(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME)[ - 0 - ].value.strip() + c = cert.subject.get_attributes_for_oid(x509.OID_COUNTRY_NAME) + if not c: + return None + + return c[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get country! {0}".format(e)) @@ -142,9 +146,11 @@ def state(cert): :return: """ try: - return cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME)[ - 0 - ].value.strip() + s = cert.subject.get_attributes_for_oid(x509.OID_STATE_OR_PROVINCE_NAME) + if not s: + return None + + return s[0].value.strip() except Exception as e: sentry.captureException() current_app.logger.error("Unable to get state! {0}".format(e)) From 92eec5cc9c014aae35dc65ba0bc145f47b0d0acd Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 21 Oct 2020 18:52:55 -0700 Subject: [PATCH 067/226] revocation should only check for not expired and not revoked certs --- lemur/certificates/service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 6d1bd2ac..6daaa641 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -105,7 +105,7 @@ def get_all_certs(): def get_all_valid_certs(authority_plugin_name): """ - Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names + Retrieves all valid (not expired & not revoked) certificates within Lemur, for the given authority plugin names ignored if no authority_plugin_name provided. Note that depending on the DB size retrieving all certificates might an expensive operation @@ -116,11 +116,12 @@ def get_all_valid_certs(authority_plugin_name): return ( Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter( Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter( - Authority.plugin_name.in_(authority_plugin_name)).all() + Authority.plugin_name.in_(authority_plugin_name)).filter(Certificate.revoked.is_(False)).all() ) else: return ( - Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all() + Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter( + Certificate.revoked.is_(False)).all() ) From 906b3b2337c56486584a4751a0a3b77270c0ebcf Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 21 Oct 2020 19:52:25 -0700 Subject: [PATCH 068/226] better handling of status code --- lemur/plugins/lemur_entrust/plugin.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 515e2400..d3f2c202 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -109,7 +109,12 @@ def handle_response(my_response): "response": d } current_app.logger.info(log_data) - return d + if d == {'response': 'No detailed message'}: + # status if no data + return s + else: + # return data from the response + return d class EntrustIssuerPlugin(IssuerPlugin): @@ -211,7 +216,7 @@ class EntrustIssuerPlugin(IssuerPlugin): deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" response = self.session.post(deactivate_url) metrics.send("entrust_deactivate_certificate", "counter", 1) - return handle_response(response) + return response.status_code @staticmethod def create_authority(options): From a4dba0cb35a02960e6d63d6aa7bc7291673a4943 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 21 Oct 2020 19:52:51 -0700 Subject: [PATCH 069/226] creating a cli to handle entrust deactivation --- lemur/certificates/cli.py | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index b883dee0..224f02a2 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -735,3 +735,44 @@ def automatically_enable_autorotate(): }) cert.rotation = True database.update(cert) + + +@manager.command +def deactivate_entrust_certificates(): + """ + Attempt to deactivate test certificates issued by Entrust + """ + + log_data = { + "function": f"{__name__}.{sys._getframe().f_code.co_name}", + "message": "Deactivating Entrust certificates" + } + + certificates = get_all_valid_certs(['entrust-issuer']) + entrust_plugin = plugins.get('entrust-issuer') + for cert in certificates: + try: + response = entrust_plugin.deactivate_certificate(cert) + if response == 200: + cert.status = "revoked" + else: + cert.status = "unknown" + + log_data["valid"] = cert.status + log_data["certificate_name"] = cert.name + log_data["certificate_id"] = cert.id + metrics.send( + "certificate_deactivate", + "counter", + 1, + metric_tags={"status": log_data["valid"], + "certificate_name": log_data["certificate_name"], + "certificate_id": log_data["certificate_id"]}, + ) + current_app.logger.info(log_data) + + database.update(cert) + + except Exception as e: + sentry.captureException() + current_app.logger.exception(e) From 2cc03088cdba41bfaffb5ebacd28379a521f235b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 21 Oct 2020 19:53:08 -0700 Subject: [PATCH 070/226] creating a celery task --- lemur/common/celery.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index a490b13b..f72fd207 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -759,7 +759,7 @@ def check_revoked(): log_data = { "function": function, - "message": "check if any certificates are revoked revoked", + "message": "check if any valid certificate is revoked", "task_id": task_id, } @@ -842,3 +842,39 @@ def enable_autorotate_for_certs_attached_to_endpoint(): cli_certificate.automatically_enable_autorotate() metrics.send(f"{function}.success", "counter", 1) return log_data + + +@celery.task(soft_time_limit=3600) +def deactivate_entrust(): + """ + This celery task attempts to deactivate all not yet deactivated Entrust certificates, and should only run in TEST + :return: + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + task_id = None + if celery.current_task: + task_id = celery.current_task.request.id + + log_data = { + "function": function, + "message": "deactivate entrust certificates", + "task_id": task_id, + } + + if task_id and is_task_active(function, task_id, None): + log_data["message"] = "Skipping task: Task is already active" + current_app.logger.debug(log_data) + return + + current_app.logger.debug(log_data) + try: + cli_certificate.deactivate_entrust_certificates() + except SoftTimeLimitExceeded: + log_data["message"] = "Time limit exceeded." + current_app.logger.error(log_data) + sentry.captureException() + metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function}) + return + + metrics.send(f"{function}.success", "counter", 1) + return log_data From c40ecd12cbe8df3913897df5fd2fb95ce6e559d6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 10:58:16 -0700 Subject: [PATCH 071/226] improved naming --- lemur/common/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/common/celery.py b/lemur/common/celery.py index f72fd207..f9d58bd9 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -845,7 +845,7 @@ def enable_autorotate_for_certs_attached_to_endpoint(): @celery.task(soft_time_limit=3600) -def deactivate_entrust(): +def deactivate_entrust_test_certificates(): """ This celery task attempts to deactivate all not yet deactivated Entrust certificates, and should only run in TEST :return: From 2e7e3a82fa0b909ea83b0989e2f32dee084a1bce Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 11:57:54 -0700 Subject: [PATCH 072/226] Update cli.py logging in exception --- lemur/certificates/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index 224f02a2..cf2ff367 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -774,5 +774,7 @@ def deactivate_entrust_certificates(): database.update(cert) except Exception as e: + current_app.logger.info(log_data) sentry.captureException() current_app.logger.exception(e) + From 03d1af16e7725527bbd5b5e80b417c05ddfd3108 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 15:59:38 -0700 Subject: [PATCH 073/226] better logging for exceptions around all plugins --- lemur/certificates/service.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 6d1bd2ac..9c544124 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -359,7 +359,12 @@ def create(**kwargs): try: cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) except Exception: - current_app.logger.error("Exception minting certificate", exc_info=True) + log_data = { + "message": "Exception minting certificate", + "issuer": kwargs["authority"].name, + "cn": kwargs["common_name"], + } + current_app.logger.error(log_data, exc_info=True) sentry.captureException() raise kwargs["body"] = cert_body From c2fe2b5e0384ade0e0f2f2567e8cb1545b62ed85 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 15:59:59 -0700 Subject: [PATCH 074/226] improved logging for all responses --- lemur/plugins/lemur_entrust/plugin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 515e2400..03919686 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -20,7 +20,13 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ + log_data = { + "reason": (r.reason if r.reason else ""), + "status_code": r.status_code, + "url": (r.url if r.url else ""), + } metrics.send(f"entrust_status_code_{r.status_code}", "counter", 1) + current_app.logger.info(log_data) def determine_end_date(end_date): From c60645bec49f23d7c55276d1a17e7c316d683cf0 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 16:00:26 -0700 Subject: [PATCH 075/226] improved logging for all responses --- lemur/plugins/lemur_digicert/plugin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index f28279a6..9a322371 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -37,7 +37,13 @@ def log_status_code(r, *args, **kwargs): :param kwargs: :return: """ + log_data = { + "reason": (r.reason if r.reason else ""), + "status_code": r.status_code, + "url": (r.url if r.url else ""), + } metrics.send("digicert_status_code_{}".format(r.status_code), "counter", 1) + current_app.logger.info(log_data) def signature_hash(signing_algorithm): From 8fa90a2ce54539853ee7ff2769b5f35ad3e2865f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 16:01:09 -0700 Subject: [PATCH 076/226] digicert expects also seconds, though not yet honoring it --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 9a322371..61a274fa 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -177,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:SS") + "Z" }, "organization": { "name": options["organization"], From 02c040865d6ca5a1c5fec2fe1e2cf039515bb08d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 16:05:29 -0700 Subject: [PATCH 077/226] more meaningful message --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 61a274fa..574c8e8e 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -210,7 +210,7 @@ def handle_response(response): :return: """ if response.status_code > 399: - raise Exception(response.json()["errors"][0]["message"]) + raise Exception("DigiCert rejected certificate request with the following error:" + response.json()["errors"][0]["message"]) return response.json() From 1c96ea9ab1ee8f1e8c36331510e3866aace74bcf Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:10:32 -0700 Subject: [PATCH 078/226] better messaging of exceptions --- lemur/plugins/lemur_digicert/plugin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 574c8e8e..a100954f 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -221,10 +221,13 @@ def handle_cis_response(response): :param response: :return: """ - if response.status_code > 399: - raise Exception(response.text) - return response.json() + if response.status_code == 404: + raise Exception("DigiCert: Order not in issued state.") + elif response.status_code == 406: + raise Exception("DigiCert: Wrong Header") + elif response.status_code > 399: + raise Exception("DigiCert rejected request with the error:" + response.text) @retry(stop_max_attempt_number=10, wait_fixed=10000) From 2e7652962cbbe4403ff44dc0df9550882e1e1b10 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:11:02 -0700 Subject: [PATCH 079/226] refactoring of the error handling --- lemur/plugins/lemur_digicert/plugin.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index a100954f..4143019e 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -221,7 +221,6 @@ def handle_cis_response(response): :param response: :return: """ - return response.json() if response.status_code == 404: raise Exception("DigiCert: Order not in issued state.") elif response.status_code == 406: @@ -229,6 +228,11 @@ def handle_cis_response(response): elif response.status_code > 399: raise Exception("DigiCert rejected request with the error:" + response.text) + if response.url.endswith("download"): + return response.content + else: + return response.json() + @retry(stop_max_attempt_number=10, wait_fixed=10000) def get_certificate_id(session, base_url, order_id): @@ -247,11 +251,9 @@ def get_cis_certificate(session, base_url, order_id): certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id) session.headers.update({"Accept": "application/x-pkcs7-certificates"}) response = session.get(certificate_url) + response_content = handle_cis_response(response) - if response.status_code == 404: - raise Exception("Order not in issued state.") - - cert_chain_pem = convert_pkcs7_bytes_to_pem(response.content) + cert_chain_pem = convert_pkcs7_bytes_to_pem(response_content) if len(cert_chain_pem) < 3: raise Exception("Missing the certificate chain") return cert_chain_pem From ae1e9d120b8751c1de9fc7fee706cb79f3bf46d8 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:13:58 -0700 Subject: [PATCH 080/226] consistent messaging --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 4143019e..345bea72 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -210,7 +210,7 @@ def handle_response(response): :return: """ if response.status_code > 399: - raise Exception("DigiCert rejected certificate request with the following error:" + response.json()["errors"][0]["message"]) + raise Exception("DigiCert rejected request with the error:" + response.json()["errors"][0]["message"]) return response.json() From 9acd974b7451f63baf363b0f9e88e77cb2b82219 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:20:47 -0700 Subject: [PATCH 081/226] fixing the test to support seconds --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 34dcef71..fe47c5b8 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -123,7 +123,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority): "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM:SS") + "Z" }, "profile_name": None, } From 97f80b79dcea1601da700c06b77395e67ae2954a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:23:33 -0700 Subject: [PATCH 082/226] adjusting digicert test to support seconds --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index fe47c5b8..059cdd82 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -159,7 +159,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z" + "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM:SS") + "Z" }, "profile_name": None, } From cf87e178c8f70d527478544f6b37a50a901c62cf Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:33:02 -0700 Subject: [PATCH 083/226] making lint happy --- lemur/certificates/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/certificates/cli.py b/lemur/certificates/cli.py index cf2ff367..f23948be 100644 --- a/lemur/certificates/cli.py +++ b/lemur/certificates/cli.py @@ -777,4 +777,3 @@ def deactivate_entrust_certificates(): current_app.logger.info(log_data) sentry.captureException() current_app.logger.exception(e) - From 9ce0010bf1a76a1057a9d3b88f1a51966d552568 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:33:39 -0700 Subject: [PATCH 084/226] handle_respone can also handle the no data response --- lemur/plugins/lemur_entrust/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index d3f2c202..0e9f6b7f 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -216,7 +216,7 @@ class EntrustIssuerPlugin(IssuerPlugin): deactivate_url = f"{base_url}/certificates/{certificate.external_id}/deactivations" response = self.session.post(deactivate_url) metrics.send("entrust_deactivate_certificate", "counter", 1) - return response.status_code + return handle_response(response) @staticmethod def create_authority(options): From 8610af8b8368565c2e48976eb01168bb2ab21c90 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 22 Oct 2020 17:54:46 -0700 Subject: [PATCH 085/226] more precise language --- lemur/plugins/lemur_digicert/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 345bea72..ee917dac 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -222,9 +222,9 @@ def handle_cis_response(response): :return: """ if response.status_code == 404: - raise Exception("DigiCert: Order not in issued state.") + raise Exception("DigiCert: order not in issued state") elif response.status_code == 406: - raise Exception("DigiCert: Wrong Header") + raise Exception("DigiCert: wrong header request format") elif response.status_code > 399: raise Exception("DigiCert rejected request with the error:" + response.text) From 71df6b8560079934802553af929d3b3cc2f0d90c Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Thu, 22 Oct 2020 18:15:26 -0700 Subject: [PATCH 086/226] Fix plugin field on notification edit --- .../notifications/notification/notification.js | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/lemur/static/app/angular/notifications/notification/notification.js b/lemur/static/app/angular/notifications/notification/notification.js index d3cfac9b..9cf88cbf 100644 --- a/lemur/static/app/angular/notifications/notification/notification.js +++ b/lemur/static/app/angular/notifications/notification/notification.js @@ -42,8 +42,8 @@ angular.module('lemur') PluginService.getByType('notification').then(function (plugins) { $scope.plugins = plugins; _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.notification.pluginName) { - plugin.pluginOptions = $scope.notification.notificationOptions; + if (plugin.slug === $scope.notification.plugin.slug) { + plugin.pluginOptions = $scope.notification.plugin.pluginOptions; $scope.notification.plugin = plugin; } }); @@ -51,16 +51,6 @@ angular.module('lemur') NotificationService.getCertificates(notification); }); - PluginService.getByType('notification').then(function (plugins) { - $scope.plugins = plugins; - _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.notification.pluginName) { - plugin.pluginOptions = $scope.notification.notificationOptions; - $scope.notification.plugin = plugin; - } - }); - }); - $scope.save = function (notification) { NotificationService.update(notification).then( function () { From 233f9768e84c583994662f17a841ed73a4d75676 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 23 Oct 2020 09:34:33 -0700 Subject: [PATCH 087/226] Fix error handling --- lemur/notifications/messaging.py | 2 +- lemur/plugins/lemur_aws/plugin.py | 5 +---- lemur/plugins/lemur_aws/sns.py | 3 ++- lemur/plugins/lemur_slack/plugin.py | 6 +++--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 1fce7636..3dd4fff7 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -111,7 +111,7 @@ def send_plugin_notification(event_type, data, recipients, notification): notification.plugin.send(event_type, data, recipients, notification.options) status = SUCCESS_METRIC_STATUS except Exception as e: - log_data["message"] = f"Unable to send expiration notification to recipients {recipients}" + log_data["message"] = f"Unable to send {event_type} notification to recipients {recipients}" current_app.logger.error(log_data, exc_info=True) sentry.captureException() diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index a0b72d94..8a54b035 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -454,7 +454,4 @@ class SNSNotificationPlugin(ExpirationNotificationPlugin): f"{self.get_option('topicName', options)}" current_app.logger.debug(f"Publishing {notification_type} notification to topic {topic_arn}") - try: - sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) - except Exception: - current_app.logger.exception(f"Error publishing {notification_type} notification to topic {topic_arn}") + sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) diff --git a/lemur/plugins/lemur_aws/sns.py b/lemur/plugins/lemur_aws/sns.py index 3aeb14da..f9fd4a07 100644 --- a/lemur/plugins/lemur_aws/sns.py +++ b/lemur/plugins/lemur_aws/sns.py @@ -29,7 +29,8 @@ def publish_single(sns_client, topic_arn, certificate, notification_type): response_code = response["ResponseMetadata"]["HTTPStatusCode"] if response_code != 200: - raise Exception(f"Failed to publish notification to SNS, response code was {response_code}") + raise Exception(f"Failed to publish {notification_type} notification to SNS topic {topic_arn}. " + f"SNS response: {response_code} {response}") current_app.logger.debug(f"AWS SNS message published to topic [{topic_arn}]: [{response}]") diff --git a/lemur/plugins/lemur_slack/plugin.py b/lemur/plugins/lemur_slack/plugin.py index ba2baf40..70d97aa5 100644 --- a/lemur/plugins/lemur_slack/plugin.py +++ b/lemur/plugins/lemur_slack/plugin.py @@ -127,7 +127,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): raise Exception("Unable to create message attachments") body = { - "text": "Lemur {0} Notification".format(notification_type.capitalize()), + "text": f"Lemur {notification_type.capitalize()} Notification", "attachments": attachments, "channel": self.get_option("recipients", options), "username": self.get_option("username", options), @@ -136,8 +136,8 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): r = requests.post(self.get_option("webhook", options), json.dumps(body)) if r.status_code not in [200]: - raise Exception("Failed to send message") + raise Exception(f"Failed to send message. Slack response: {r.status_code} {body}") current_app.logger.info( - "Slack response: {0} Message Body: {1}".format(r.status_code, body) + f"Slack response: {r.status_code} Message Body: {body}" ) From a5cea4fb9a5a4e8eda99c46fdf808af5f3aab32f Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 23 Oct 2020 09:42:03 -0700 Subject: [PATCH 088/226] Skip revoked certs when looking for certs to notify --- lemur/notifications/messaging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 3dd4fff7..3928689e 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -42,6 +42,7 @@ def get_certificates(exclude=None): .filter(Certificate.not_after <= max) .filter(Certificate.notify == True) .filter(Certificate.expired == False) + .filter(Certificate.revoked == False) ) # noqa exclude_conditions = [] From e01863097bf55cd9864a4a2e46ae0a6de700b02e Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:16:23 -0700 Subject: [PATCH 089/226] fixing the time bug, sub-second to second, and month to minute! --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index ee917dac..091539de 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -177,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:ss") + "Z" }, "organization": { "name": options["organization"], From bc6fb02fc2712e461270b28a01d317709d13d4dc Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:16:38 -0700 Subject: [PATCH 090/226] fixing testing --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 059cdd82..fd07ea2b 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -123,7 +123,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority): "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } @@ -159,7 +159,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } From 1495fb3595bc102e6ff2f9400c6981fc8bba80ed Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:18:24 -0700 Subject: [PATCH 091/226] now fixing the month to minute bug --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 091539de..ec3a0792 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -177,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:ss") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "organization": { "name": options["organization"], From fd12d4848c2016a410df717e3d92efaf05e16e64 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Fri, 23 Oct 2020 11:26:11 -0700 Subject: [PATCH 092/226] Grammar fixes --- docs/administration.rst | 2 +- docs/developer/plugins/index.rst | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index ef3f8e38..724b136f 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -269,7 +269,7 @@ Certificates marked as inactive will **not** be notified of upcoming expiration. silence the expiration. If a certificate is active and is expiring the above will be notified according to the `LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS` or 30, 15, 2 days before expiration if no intervals are set. -Lemur supports sending certification expiration notifications through SES and SMTP. +Lemur supports sending certificate expiration notifications through SES and SMTP. .. data:: LEMUR_EMAIL_SENDER diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index 0223d9ca..c2a8c48a 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -215,7 +215,7 @@ Notification ------------ Lemur includes the ability to create Email notifications by **default**. These notifications -currently come in the form of expiration and rotation notices. Lemur periodically checks certifications expiration dates and +currently come in the form of expiration and rotation notices. Lemur periodically checks certificate expiration dates and determines if a given certificate is eligible for notification. There are currently only two parameters used to determine if a certificate is eligible; validity expiration (date the certificate is no longer valid) and the number of days the current date (UTC) is from that expiration date. @@ -223,12 +223,12 @@ of days the current date (UTC) is from that expiration date. Expiration notifications can also be configured for Slack or AWS SNS. Rotation notifications are not configurable. Notifications sent to a certificate owner and security team (`LEMUR_SECURITY_TEAM_EMAIL`) can currently only be sent via email. -There are currently two objects that available for notification plugins. The first is `NotificationPlugin`, which is the base object for -any notification within Lemur. Currently the only supported notification type is an certificate expiration notification. If you +There are currently two objects that are available for notification plugins. The first is `NotificationPlugin`, which is the base object for +any notification within Lemur. Currently the only supported notification type is a certificate expiration notification. If you are trying to create a new notification type (audit, failed logins, etc.) this would be the object to base your plugin on. You would also then need to build additional code to trigger the new notification type. -The second is `ExpirationNotificationPlugin`, which inherits from `NotificationPlugin` object. +The second is `ExpirationNotificationPlugin`, which inherits from the `NotificationPlugin` object. You will most likely want to base your plugin on this object if you want to add new channels for expiration notices (HipChat, Jira, etc.). It adds default options that are required by all expiration notifications (interval, unit). This interface expects for the child to define the following function:: From 2c22d42a57d7ac219fbacd6480d63cbf67af0b63 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 23 Oct 2020 17:06:42 -0700 Subject: [PATCH 093/226] Modify description during reissue Include the certificate ID being reissued and mention that this is created by Lemur as part of reissue --- lemur/certificates/service.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1716ccb2..b90d7e47 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -6,6 +6,7 @@ .. moduleauthor:: Kevin Glisson """ import arrow +import re from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization @@ -778,6 +779,19 @@ def reissue_certificate(certificate, replace=None, user=None): if replace: primitives["replaces"] = [certificate] + # Modify description to include the certificate ID being reissued and mention that this is created by Lemur + # as part of reissue + reissue_message_prefix = "Reissued by Lemur for cert ID " + reissue_message = re.compile(f"{reissue_message_prefix}([0-9]+)") + if primitives["description"]: + match = reissue_message.search(primitives["description"]) + if match: + primitives["description"] = primitives["description"].replace(match.group(1), str(certificate.id)) + else: + primitives["description"] = f"{reissue_message_prefix}{certificate.id}, {primitives['description']}" + else: + primitives["description"] = f"{reissue_message_prefix}{certificate.id}" + new_cert = create(**primitives) return new_cert From 2c1e7b19a225ad71e4a6802fd293469c5601d3d7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 17:59:58 -0700 Subject: [PATCH 094/226] 10x 10s delay might be too long for the load balancer request --- lemur/plugins/lemur_digicert/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index ec3a0792..d8e88fa3 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -234,7 +234,7 @@ def handle_cis_response(response): return response.json() -@retry(stop_max_attempt_number=10, wait_fixed=10000) +@retry(stop_max_attempt_number=10, wait_fixed=1000) def get_certificate_id(session, base_url, order_id): """Retrieve certificate order id from Digicert API.""" order_url = "{0}/services/v2/order/certificate/{1}".format(base_url, order_id) @@ -245,7 +245,7 @@ def get_certificate_id(session, base_url, order_id): return response_data["certificate"]["id"] -@retry(stop_max_attempt_number=10, wait_fixed=10000) +@retry(stop_max_attempt_number=10, wait_fixed=1000) def get_cis_certificate(session, base_url, order_id): """Retrieve certificate order id from Digicert API, including the chain""" certificate_url = "{0}/platform/cis/certificate/{1}/download".format(base_url, order_id) From d233490c8aeb6996bb8fc936c9c86f3614fce142 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 18:01:14 -0700 Subject: [PATCH 095/226] simple retry --- lemur/plugins/lemur_entrust/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index ffb5765d..4700c022 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -200,6 +200,7 @@ class EntrustIssuerPlugin(IssuerPlugin): return cert, chain, external_id + @retry(stop_max_attempt_number=3, wait_fixed=1000) def revoke_certificate(self, certificate, comments): """Revoke an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") @@ -216,6 +217,7 @@ class EntrustIssuerPlugin(IssuerPlugin): metrics.send("entrust_revoke_certificate", "counter", 1) return handle_response(response) + @retry(stop_max_attempt_number=3, wait_fixed=1000) def deactivate_certificate(self, certificate): """Deactivates an Entrust certificate.""" base_url = current_app.config.get("ENTRUST_URL") From 75bc3a5b20d6d0efcf0da236daf301fe29b12bdb Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 18:02:05 -0700 Subject: [PATCH 096/226] refactoring and adding retry --- lemur/plugins/lemur_entrust/plugin.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 4700c022..52860049 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -121,6 +121,24 @@ def handle_response(my_response): else: # return data from the response return d +@retry(stop_max_attempt_number=3, wait_fixed=5000) +def get_certificate_order(session, url, data): + """ + Helper function place a cert order and downloading it + :param session: + :param url: Entrust endpoint url + :param data: CSR, and the required order details, such as validity length + :return: the cert chain + :raise Exception: + """ + try: + response = session.post(url, json=data, timeout=(15, 40)) + except requests.exceptions.Timeout: + raise Exception("Timeout for POST") + except requests.exceptions.RequestException as e: + raise Exception(f"Error for POST {e}") + + return handle_response(response) class EntrustIssuerPlugin(IssuerPlugin): @@ -178,14 +196,8 @@ class EntrustIssuerPlugin(IssuerPlugin): data = process_options(issuer_options) data["csr"] = csr - try: - response = self.session.post(url, json=data, timeout=(15, 40)) - except requests.exceptions.Timeout: - raise Exception("Timeout for POST") - except requests.exceptions.RequestException as e: - raise Exception(f"Error for POST {e}") + response_dict = get_certificate_order(self.session, url, data) - response_dict = handle_response(response) external_id = response_dict['trackingId'] cert = response_dict['endEntityCert'] if len(response_dict['chainCerts']) < 2: From 6891077501a9ca418b6747f0b13d57bc08a79759 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 18:02:35 -0700 Subject: [PATCH 097/226] readability --- lemur/plugins/lemur_entrust/plugin.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 52860049..0e769093 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -100,27 +100,29 @@ def handle_response(my_response): } try: - d = json.loads(my_response.content) + data = json.loads(my_response.content) except ValueError: # catch an empty jason object here - d = {'response': 'No detailed message'} - s = my_response.status_code - if s > 399: - raise Exception(f"ENTRUST error: {msg.get(s, s)}\n{d['errors']}") + data = {'response': 'No detailed message'} + status_code = my_response.status_code + if status_code > 399: + raise Exception(f"ENTRUST error: {msg.get(status_code, status_code)}\n{data['errors']}") log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", "message": "Response", - "status": s, - "response": d + "status": status_code, + "response": data } current_app.logger.info(log_data) - if d == {'response': 'No detailed message'}: + if data == {'response': 'No detailed message'}: # status if no data - return s + return status_code else: # return data from the response - return d + return data + + @retry(stop_max_attempt_number=3, wait_fixed=5000) def get_certificate_order(session, url, data): """ From 7e573d6d517d7f6189ff005ed072f321627b7b6f Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 18:02:54 -0700 Subject: [PATCH 098/226] fixing typo --- lemur/plugins/lemur_entrust/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 0e769093..c785acc1 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -87,7 +87,7 @@ def process_options(options): def handle_response(my_response): """ Helper function for parsing responses from the Entrust API. - :param content: + :param my_response: :return: :raise Exception: """ msg = { From 9957120a7fd2c0befef39f166c6292f5b8f83e86 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 18:03:07 -0700 Subject: [PATCH 099/226] adding missing import --- lemur/plugins/lemur_entrust/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index c785acc1..02e0a2be 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -1,9 +1,9 @@ - import arrow import requests import json import sys from flask import current_app +from retrying import retry from lemur.plugins import lemur_entrust as entrust from lemur.plugins.bases import IssuerPlugin, SourcePlugin From 0e02abbb3791cbd844cd916812a99896c6823fce Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 18:03:27 -0700 Subject: [PATCH 100/226] Entrust just looks into CSR for RSA/EC key type --- lemur/plugins/lemur_entrust/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index 02e0a2be..fcb3e14f 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -78,7 +78,6 @@ def process_options(options): "eku": "SERVER_AND_CLIENT_AUTH", "certType": product_type, "certExpiryDate": validity_end, - # "keyType": "RSA", Entrust complaining about this parameter "tracking": tracking_data } return data From f6554a9a1e79b9a3e688ba59438708b559efd835 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 18:03:55 -0700 Subject: [PATCH 101/226] typo, fixing abstract class complaints --- lemur/plugins/lemur_entrust/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_entrust/plugin.py b/lemur/plugins/lemur_entrust/plugin.py index fcb3e14f..8bb0710c 100644 --- a/lemur/plugins/lemur_entrust/plugin.py +++ b/lemur/plugins/lemur_entrust/plugin.py @@ -259,7 +259,7 @@ class EntrustIssuerPlugin(IssuerPlugin): def get_ordered_certificate(self, order_id): raise NotImplementedError("Not implemented\n", self, order_id) - def canceled_ordered_certificate(self, pending_cert, **kwargs): + def cancel_ordered_certificate(self, pending_cert, **kwargs): raise NotImplementedError("Not implemented\n", self, pending_cert, **kwargs) From d7478a5c5cb00dc619e7f8bc7afc2780a1cb3e1a Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sun, 25 Oct 2020 19:24:17 +0100 Subject: [PATCH 102/226] use an alternative logger for the upgrade --- docs/administration.rst | 7 +++++++ lemur/manage.py | 1 + lemur/migrations/versions/1db4f82bc780_.py | 16 ++++++++++++--- lemur/migrations/versions/c301c59688d2_.py | 23 +++++++++++++++++----- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index c2f20362..62a22dd1 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -28,6 +28,13 @@ Basic Configuration LOG_FILE = "/logs/lemur/lemur-test.log" +.. data:: LOG_UPGRADE_FILE + :noindex: + + :: + + LOG_UPGRADE_FILE = "/logs/lemur/db_upgrade.log" + .. data:: DEBUG :noindex: diff --git a/lemur/manage.py b/lemur/manage.py index e53f8bd6..bff97535 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -120,6 +120,7 @@ METRIC_PROVIDERS = [] LOG_LEVEL = "DEBUG" LOG_FILE = "lemur.log" +LOG_UPGRADE_FILE = "db_upgrade.log" # Database diff --git a/lemur/migrations/versions/1db4f82bc780_.py b/lemur/migrations/versions/1db4f82bc780_.py index e6fb47f0..00b83ceb 100644 --- a/lemur/migrations/versions/1db4f82bc780_.py +++ b/lemur/migrations/versions/1db4f82bc780_.py @@ -10,11 +10,21 @@ Create Date: 2018-08-03 12:56:44.565230 revision = "1db4f82bc780" down_revision = "3adfdd6598df" -import logging - from alembic import op -log = logging.getLogger(__name__) +from flask import current_app +from logging import Formatter, FileHandler, getLogger + +log = getLogger(__name__) +handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log")) +handler.setFormatter( + Formatter( + "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]" + ) +) +handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.addHandler(handler) def upgrade(): diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 8712d60c..d1a30650 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -31,12 +31,25 @@ down_revision = '434c29e40511' from alembic import op from sqlalchemy.sql import text -from lemur.common import utils import time import datetime +from flask import current_app + +from logging import Formatter, FileHandler, getLogger + +from lemur.common import utils + +log = getLogger(__name__) +handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log")) +handler.setFormatter( + Formatter( + "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]" + ) +) +handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG")) +log.addHandler(handler) -import logging -log = logging.getLogger(__name__) def upgrade(): log.info("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) @@ -94,9 +107,9 @@ def update_key_type(): try: cert_key_type = utils.get_key_type_from_certificate(body) except ValueError as e: - log.info("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) + log.error("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) else: - log.info("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + log.error("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) stmt = text( "update certificates set key_type=:key_type where id=:id" ) From 37f05a89f22d041a7d49bfac5f62519b4f5f65f9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2020 09:23:10 +0000 Subject: [PATCH 103/226] Bump botocore from 1.18.16 to 1.18.18 Bumps [botocore](https://github.com/boto/botocore) from 1.18.16 to 1.18.18. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.18.16...1.18.18) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 38b62198..24504233 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,7 +18,7 @@ beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.15.16 # via -r requirements.txt -botocore==1.18.16 # via -r requirements.txt, boto3, s3transfer +botocore==1.18.18 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 8df4f5d1..94bdabc4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in boto3==1.15.16 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.18.16 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.18.18 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index d323b40f..0949fbb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.15.16 # via -r requirements.in -botocore==1.18.16 # via -r requirements.in, boto3, s3transfer +botocore==1.18.18 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From fa62023b2db0a868af31ae380d6f25f30a9f11dc Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:16:23 -0700 Subject: [PATCH 104/226] fixing the time bug, sub-second to second, and month to minute! --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index ee917dac..091539de 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -177,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:ss") + "Z" }, "organization": { "name": options["organization"], From 3290d6634bda9591e76d8f857e5ae8162848f28c Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:16:38 -0700 Subject: [PATCH 105/226] fixing testing --- lemur/plugins/lemur_digicert/tests/test_digicert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_digicert/tests/test_digicert.py b/lemur/plugins/lemur_digicert/tests/test_digicert.py index 059cdd82..fd07ea2b 100644 --- a/lemur/plugins/lemur_digicert/tests/test_digicert.py +++ b/lemur/plugins/lemur_digicert/tests/test_digicert.py @@ -123,7 +123,7 @@ def test_map_cis_fields_with_validity_years(mock_current_app, authority): "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": arrow.get(2018, 11, 3).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } @@ -159,7 +159,7 @@ def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, autho "signature_hash": "sha256", "organization": {"name": "Example, Inc."}, "validity": { - "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM:SS") + "Z" + "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "profile_name": None, } From 6723e3c80dcdee473d1d23be444180bdb6a9f61b Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 23 Oct 2020 10:18:24 -0700 Subject: [PATCH 106/226] now fixing the month to minute bug --- lemur/plugins/lemur_digicert/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index 091539de..ec3a0792 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -177,7 +177,7 @@ def map_cis_fields(options, csr): "csr": csr, "signature_hash": signature_hash(options.get("signing_algorithm")), "validity": { - "valid_to": validity_end.format("YYYY-MM-DDTHH:MM:ss") + "Z" + "valid_to": validity_end.format("YYYY-MM-DDTHH:mm:ss") + "Z" }, "organization": { "name": options["organization"], From 3f765b51efdce6730f608c1762d09212347908d3 Mon Sep 17 00:00:00 2001 From: Jasmine Schladen Date: Mon, 26 Oct 2020 11:27:18 -0700 Subject: [PATCH 107/226] Fix sources and destinations, and allow actually updating the notification type --- lemur/destinations/schemas.py | 3 +++ lemur/destinations/service.py | 9 ++++++++- lemur/destinations/views.py | 1 + lemur/notifications/service.py | 4 +++- lemur/notifications/views.py | 1 + lemur/sources/service.py | 4 +++- lemur/sources/views.py | 1 + .../angular/destinations/destination/destination.js | 12 ++++++------ lemur/static/app/angular/sources/source/source.js | 12 ++---------- 9 files changed, 28 insertions(+), 19 deletions(-) diff --git a/lemur/destinations/schemas.py b/lemur/destinations/schemas.py index cc46ecd4..22187a09 100644 --- a/lemur/destinations/schemas.py +++ b/lemur/destinations/schemas.py @@ -31,6 +31,9 @@ class DestinationOutputSchema(LemurOutputSchema): def fill_object(self, data): if data: data["plugin"]["pluginOptions"] = data["options"] + for option in data["plugin"]["pluginOptions"]: + if "export-plugin" in option["type"]: + option["value"]["pluginOptions"] = option["value"]["plugin_options"] return data diff --git a/lemur/destinations/service.py b/lemur/destinations/service.py index 92162f4b..7bae57f0 100644 --- a/lemur/destinations/service.py +++ b/lemur/destinations/service.py @@ -41,12 +41,14 @@ def create(label, plugin_name, options, description=None): return database.create(destination) -def update(destination_id, label, options, description): +def update(destination_id, label, plugin_name, options, description): """ Updates an existing destination. :param destination_id: Lemur assigned ID :param label: Destination common name + :param plugin_name: + :param options: :param description: :rtype : Destination :return: @@ -54,6 +56,11 @@ def update(destination_id, label, options, description): destination = get(destination_id) destination.label = label + destination.plugin_name = plugin_name + # remove any sub-plugin objects before try to save the json options + for option in options: + if "plugin" in option["type"]: + del option["value"]["plugin_object"] destination.options = options destination.description = description diff --git a/lemur/destinations/views.py b/lemur/destinations/views.py index 0b0559fe..072ff34e 100644 --- a/lemur/destinations/views.py +++ b/lemur/destinations/views.py @@ -338,6 +338,7 @@ class Destinations(AuthenticatedResource): return service.update( destination_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], ) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index ac624d1c..34edccc0 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -104,12 +104,13 @@ def create(label, plugin_name, options, description, certificates): return database.create(notification) -def update(notification_id, label, options, description, active, certificates): +def update(notification_id, label, plugin_name, options, description, active, certificates): """ Updates an existing notification. :param notification_id: :param label: Notification label + :param plugin_name: :param options: :param description: :param active: @@ -120,6 +121,7 @@ def update(notification_id, label, options, description, active, certificates): notification = get(notification_id) notification.label = label + notification.plugin_name = plugin_name notification.options = options notification.description = description notification.active = active diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index cdabb4d4..f6eef655 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -340,6 +340,7 @@ class Notifications(AuthenticatedResource): return service.update( notification_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], data["active"], diff --git a/lemur/sources/service.py b/lemur/sources/service.py index fafa6f5a..be0de049 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -264,13 +264,14 @@ def create(label, plugin_name, options, description=None): return database.create(source) -def update(source_id, label, options, description): +def update(source_id, label, plugin_name, options, description): """ Updates an existing source. :param source_id: Lemur assigned ID :param label: Source common name :param options: + :param plugin_name: :param description: :rtype : Source :return: @@ -278,6 +279,7 @@ def update(source_id, label, options, description): source = get(source_id) source.label = label + source.plugin_name = plugin_name source.options = options source.description = description diff --git a/lemur/sources/views.py b/lemur/sources/views.py index b74c4d80..3b4deab7 100644 --- a/lemur/sources/views.py +++ b/lemur/sources/views.py @@ -284,6 +284,7 @@ class Sources(AuthenticatedResource): return service.update( source_id, data["label"], + data["plugin"]["slug"], data["plugin"]["plugin_options"], data["description"], ) diff --git a/lemur/static/app/angular/destinations/destination/destination.js b/lemur/static/app/angular/destinations/destination/destination.js index 21f624c8..93a7f80e 100644 --- a/lemur/static/app/angular/destinations/destination/destination.js +++ b/lemur/static/app/angular/destinations/destination/destination.js @@ -52,19 +52,19 @@ angular.module('lemur') if (plugin.slug === $scope.destination.plugin.slug) { plugin.pluginOptions = $scope.destination.plugin.pluginOptions; $scope.destination.plugin = plugin; - _.each($scope.destination.plugin.pluginOptions, function (option) { - if (option.type === 'export-plugin') { - PluginService.getByType('export').then(function (plugins) { - $scope.exportPlugins = plugins; + PluginService.getByType('export').then(function (plugins) { + $scope.exportPlugins = plugins; + _.each($scope.destination.plugin.pluginOptions, function (option) { + if (option.type === 'export-plugin') { _.each($scope.exportPlugins, function (plugin) { if (plugin.slug === option.value.slug) { plugin.pluginOptions = option.value.pluginOptions; option.value = plugin; } }); - }); - } + } + }); }); } }); diff --git a/lemur/static/app/angular/sources/source/source.js b/lemur/static/app/angular/sources/source/source.js index 1d5c1641..8ea381f8 100644 --- a/lemur/static/app/angular/sources/source/source.js +++ b/lemur/static/app/angular/sources/source/source.js @@ -41,22 +41,14 @@ angular.module('lemur') PluginService.getByType('source').then(function (plugins) { $scope.plugins = plugins; _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.source.pluginName) { + if (plugin.slug === $scope.source.plugin.slug) { + plugin.pluginOptions = $scope.source.plugin.pluginOptions; $scope.source.plugin = plugin; } }); }); }); - PluginService.getByType('source').then(function (plugins) { - $scope.plugins = plugins; - _.each($scope.plugins, function (plugin) { - if (plugin.slug === $scope.source.pluginName) { - $scope.source.plugin = plugin; - } - }); - }); - $scope.save = function (source) { SourceService.update(source).then( function () { From 749aa772ba459a40dd59cc3b3ebd1e73cee99ef2 Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Mon, 26 Oct 2020 11:57:33 -0700 Subject: [PATCH 108/226] First change to get CNAME redirection working --- docs/administration.rst | 14 ++++++++++++++ lemur/plugins/lemur_acme/plugin.py | 22 +++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 846a4c34..80d88feb 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -620,6 +620,20 @@ If you are not using a metric provider you do not need to configure any of these Plugin Specific Options ----------------------- +ACME Plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. data:: ACME_DNS_PROVIDER_TYPES + :noindex: + + Dictionary of ACME DNS Providers and their requirements. + +.. data:: ACME_ENABLE_DELEGATED_CNAME + :noindex: + + Enables delegated DNS domain validation using CNAMES. When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges. + + Active Directory Certificate Services Plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 16d61a0f..9177d6e8 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -16,6 +16,7 @@ import json import time import OpenSSL.crypto +import dns.resolver import josepy as jose from acme import challenges, errors, messages from acme.client import BackwardsCompatibleClientV2, ClientNetwork @@ -23,7 +24,6 @@ from acme.errors import PollError, TimeoutError, WildcardUnsupportedError from acme.messages import Error as AcmeError from botocore.exceptions import ClientError from flask import current_app - from lemur.authorizations import service as authorization_service from lemur.common.utils import generate_private_key from lemur.dns_providers import service as dns_provider_service @@ -287,6 +287,13 @@ class AcmeHandler(object): authorizations = [] for domain in order_info.domains: + + # Replace domain if doing CNAME delegation + if current_app.config.get("ACME_ENABLE_DELEGATED_CNAME", False): + cname = self.get_cname(domain) + if cname: + domain = cname + if not self.dns_providers_for_domain.get(domain): metrics.send( "get_authorizations_no_dns_provider_for_domain", "counter", 1 @@ -407,6 +414,19 @@ class AcmeHandler(object): raise UnknownProvider("No such DNS provider: {}".format(type)) return provider + def get_cname(self, domain): + """ + :param domain: Domain name to look up a CNAME for. + :param record_type: Type of DNS record to lookup. + :return: First CNAME target or False if no CNAME record exists. + """ + try: + result = dns.resolver.query(domain, 'CNAME') + if len(result) > 0: + return str(result[0].target).rstrip('.') + except dns.exception.DNSException: + return False + class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" From 392725ff309609c7695de5860a409a9f794415d0 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 26 Oct 2020 15:33:20 -0700 Subject: [PATCH 109/226] Add description check in reissue unit test --- lemur/tests/test_certificates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index c271a97e..583022eb 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -802,6 +802,7 @@ def test_reissue_certificate( assert new_cert.organization != certificate.organization # Check for default value since authority does not have cab_compliant option set assert new_cert.organization == LEMUR_DEFAULT_ORGANIZATION + assert new_cert.description.startswith(f"Reissued by Lemur for cert ID {certificate.id}") # update cab_compliant option to false for crypto_authority to maintain subject details update_options(crypto_authority.id, '[{"name": "cab_compliant","value":false}]') From 709a9808aaef5ce5621d25eaff41adc01d6410c6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 26 Oct 2020 18:32:53 -0700 Subject: [PATCH 110/226] better structure of the query and and removing ilike --- lemur/certificates/service.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1716ccb2..1b026f4d 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -560,18 +560,29 @@ def query_common_name(common_name, args): :return: """ owner = args.pop("owner") - if not owner: - owner = "%" - # only not expired certificates current_time = arrow.utcnow() - result = ( - Certificate.query.filter(Certificate.cn.ilike(common_name)) - .filter(Certificate.owner.ilike(owner)) - .filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) - .all() - ) + if common_name == "%" and not owner: + result = ( + Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) + .all() + ) + elif common_name == "%": + # all valid certs from the owner + result = ( + Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) + .filter(Certificate.owner == owner) + .all() + ) + else: + # search based on owner and cn + result = ( + Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) + .filter(Certificate.cn.like(common_name)) + .filter(Certificate.owner == owner) + .all() + ) return result From 6a1b4b48577166a536bb9223e749fcda05e0a5af Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 26 Oct 2020 18:33:33 -0700 Subject: [PATCH 111/226] ignore expired certs --- lemur/certificates/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 1b026f4d..275935b2 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -566,6 +566,7 @@ def query_common_name(common_name, args): if common_name == "%" and not owner: result = ( Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) + .filter(not_(Certificate.revoked)) .all() ) elif common_name == "%": @@ -573,6 +574,7 @@ def query_common_name(common_name, args): result = ( Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) .filter(Certificate.owner == owner) + .filter(not_(Certificate.revoked)) .all() ) else: @@ -581,6 +583,7 @@ def query_common_name(common_name, args): Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) .filter(Certificate.cn.like(common_name)) .filter(Certificate.owner == owner) + .filter(not_(Certificate.revoked)) .all() ) From 1ef6139f9bfc9becee9cb8d32f91c93fb6f1d360 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 26 Oct 2020 18:34:21 -0700 Subject: [PATCH 112/226] ignore rotated certs, since there is a new cert that can be used --- lemur/certificates/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 275935b2..ef82c605 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -567,6 +567,7 @@ def query_common_name(common_name, args): result = ( Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) .filter(not_(Certificate.revoked)) + .filter(not_(Certificate.replaced.any())) # ignore rotated certificates to avoid duplicates .all() ) elif common_name == "%": @@ -575,6 +576,7 @@ def query_common_name(common_name, args): Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) .filter(Certificate.owner == owner) .filter(not_(Certificate.revoked)) + .filter(not_(Certificate.replaced.any())) # ignore rotated certificates to avoid duplicates .all() ) else: @@ -584,6 +586,7 @@ def query_common_name(common_name, args): .filter(Certificate.cn.like(common_name)) .filter(Certificate.owner == owner) .filter(not_(Certificate.revoked)) + .filter(not_(Certificate.replaced.any())) # ignore rotated certificates to avoid duplicates .all() ) From 3b258447dbb46c3bc7f1e3adf48dc0361670c208 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 26 Oct 2020 19:16:40 -0700 Subject: [PATCH 113/226] addressing Chad's feedbakc --- lemur/certificates/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index ef82c605..4484f636 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -583,7 +583,7 @@ def query_common_name(common_name, args): # search based on owner and cn result = ( Certificate.query.filter(Certificate.not_after >= current_time.format("YYYY-MM-DD")) - .filter(Certificate.cn.like(common_name)) + .filter(Certificate.cn.ilike(common_name)) .filter(Certificate.owner == owner) .filter(not_(Certificate.revoked)) .filter(not_(Certificate.replaced.any())) # ignore rotated certificates to avoid duplicates From d00dd9d2956b7a4f28339712344810092600ec65 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 14:29:23 +0200 Subject: [PATCH 114/226] Initial structure for ACME http challenge --- lemur/plugins/lemur_acme/plugin.py | 94 +++++++++++++++++++++++++++++- setup.py | 1 + 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 8bc1485f..d1c86017 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -26,6 +26,7 @@ from flask import current_app from lemur.authorizations import service as authorization_service from lemur.common.utils import generate_private_key +from lemur.destinations import service as destination_service from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider from lemur.extensions import metrics, sentry @@ -436,7 +437,7 @@ class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" slug = "acme-issuer" description = ( - "Enables the creation of certificates via ACME CAs (including Let's Encrypt)" + "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the DNS-01 challenge" ) version = acme.VERSION @@ -746,3 +747,94 @@ class ACMEIssuerPlugin(IssuerPlugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): # Needed to override issuer function. pass + + +class ACMEHttpIssuerPlugin(IssuerPlugin): + title = "Acme HTTP-01" + slug = "acme-http-issuer" + description = ( + "Enables the creation of certificates via ACME CAs (including Let's Encrypt), using the HTTP-01 challenge" + ) + version = acme.VERSION + + author = "Netflix" + author_url = "https://github.com/netflix/lemur.git" + + destination_list = [] + + options = [ + { + "name": "acme_url", + "type": "str", + "required": True, + "validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", + "helpMessage": "Must be a valid web url starting with http[s]://", + }, + { + "name": "telephone", + "type": "str", + "default": "", + "helpMessage": "Telephone to use", + }, + { + "name": "email", + "type": "str", + "default": "", + "validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", + "helpMessage": "Email to use", + }, + { + "name": "certificate", + "type": "textarea", + "default": "", + "validation": "/^-----BEGIN CERTIFICATE-----/", + "helpMessage": "Certificate to use", + }, + { + "name": "tokenDestination", + "type": "select", + "required": True, + "available": destination_list, + "helpMessage": "The destination to use to deploy the token.", + }, + ] + + def __init__(self, *args, **kwargs): + super(ACMEHttpIssuerPlugin, self).__init__(*args, **kwargs) + + if len(self.destination_list) == 0: + destinations = destination_service.get_all() + for destination in destinations: + # we only want to use sftp destinations here + if destination.plugin_name == "sftp-destination": + self.destination_list.append(destination.label) + + def create_certificate(self, csr, issuer_options): + pass + + @staticmethod + def create_authority(options): + """ + Creates an authority, this authority is then used by Lemur to allow a user + to specify which Certificate Authority they want to sign their certificate. + + :param options: + :return: + """ + role = {"username": "", "password": "", "name": "acme"} + plugin_options = options.get("plugin", {}).get("plugin_options") + if not plugin_options: + error = "Invalid options for lemur_acme plugin: {}".format(options) + current_app.logger.error(error) + raise InvalidConfiguration(error) + # Define static acme_root based off configuration variable by default. However, if user has passed a + # certificate, use this certificate as the root. + acme_root = current_app.config.get("ACME_ROOT") + for option in plugin_options: + if option.get("name") == "certificate": + acme_root = option.get("value") + return acme_root, "", [role] + + def cancel_ordered_certificate(self, pending_cert, **kwargs): + # Needed to override issuer function. + pass \ No newline at end of file diff --git a/setup.py b/setup.py index 4da14c3d..7fcbc101 100644 --- a/setup.py +++ b/setup.py @@ -132,6 +132,7 @@ setup( 'lemur.plugins': [ 'verisign_issuer = lemur.plugins.lemur_verisign.plugin:VerisignIssuerPlugin', 'acme_issuer = lemur.plugins.lemur_acme.plugin:ACMEIssuerPlugin', + 'acme_http_issuer = lemur.plugins.lemur_acme.plugin:ACMEHttpIssuerPlugin', 'aws_destination = lemur.plugins.lemur_aws.plugin:AWSDestinationPlugin', 'aws_source = lemur.plugins.lemur_aws.plugin:AWSSourcePlugin', 'aws_s3 = lemur.plugins.lemur_aws.plugin:S3DestinationPlugin', From 348d8477dd3303fe400d0d2eca46cc845a5f1818 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 14:57:32 +0200 Subject: [PATCH 115/226] Refactor destination plugin, to allow upload of ACME http-challenge tokens --- lemur/plugins/lemur_sftp/plugin.py | 57 +++++++++++++++++++----------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 66784048..7c4e93c4 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -16,6 +16,7 @@ .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov """ +from os import path import paramiko @@ -95,18 +96,14 @@ class SFTPDestinationPlugin(DestinationPlugin): }, ] + # this is called when using this as a default destination plugin def upload(self, name, body, private_key, cert_chain, options, **kwargs): current_app.logger.debug("SFTP destination plugin is started") cn = common_name(parse_certificate(body)) - host = self.get_option("host", options) - port = self.get_option("port", options) - user = self.get_option("user", options) - password = self.get_option("password", options) - ssh_priv_key = self.get_option("privateKeyPath", options) - ssh_priv_key_pass = self.get_option("privateKeyPass", options) dst_path = self.get_option("destinationPath", options) + dst_path_cn = dst_path + "/" + cn export_format = self.get_option("exportFormat", options) # prepare files for upload @@ -121,6 +118,31 @@ class SFTPDestinationPlugin(DestinationPlugin): # store chain in the separate file files[cn + ".ca.bundle.pem"] = cert_chain + self.upload_file(dst_path_cn, files, options) + + # this is called from the acme http challenge + def upload_acme_token(self, token, thumbprint, options, **kwargs): + + current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge") + + dst_path = self.get_option("destinationPath", options) + dst_path = path.join(dst_path, "/.well-known/acme-challenge/") + + # prepare files for upload + files = {token: thumbprint} + + self.upload_file(dst_path, files, options) + + # here the file is uploaded for real, this helps to keep this class DRY + def upload_file(self, dst_path, files, options): + + host = self.get_option("host", options) + port = self.get_option("port", options) + user = self.get_option("user", options) + password = self.get_option("password", options) + ssh_priv_key = self.get_option("privateKeyPath", options) + ssh_priv_key_pass = self.get_option("privateKeyPass", options) + # upload files try: current_app.logger.debug( @@ -156,33 +178,26 @@ class SFTPDestinationPlugin(DestinationPlugin): sftp.mkdir(dst_path) except IOError: current_app.logger.debug("{0} already exist, resuming".format(dst_path)) - try: - dst_path_cn = dst_path + "/" + cn - current_app.logger.debug("Creating {0}".format(dst_path_cn)) - sftp.mkdir(dst_path_cn) - except IOError: - current_app.logger.debug( - "{0} already exist, resuming".format(dst_path_cn) - ) # upload certificate files to the sftp destination for filename, data in files.items(): current_app.logger.debug( - "Uploading {0} to {1}".format(filename, dst_path_cn) + "Uploading {0} to {1}".format(filename, dst_path) ) try: - with sftp.open(dst_path_cn + "/" + filename, "w") as f: + with sftp.open(dst_path + "/" + filename, "w") as f: f.write(data) - except (PermissionError) as permerror: + except PermissionError as permerror: if permerror.errno == 13: current_app.logger.debug( - "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format(filename, dst_path_cn) + "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format( + filename, dst_path) ) - sftp.chmod(dst_path_cn + "/" + filename, 0o600) - with sftp.open(dst_path_cn + "/" + filename, "w") as f: + sftp.chmod(dst_path + "/" + filename, 0o600) + with sftp.open(dst_path + "/" + filename, "w") as f: f.write(data) # read only for owner, -r-------- - sftp.chmod(dst_path_cn + "/" + filename, 0o400) + sftp.chmod(dst_path + "/" + filename, 0o400) ssh.close() From 3012995c76e87b29ab30d5c3077c3e228852c4d6 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 18:32:30 +0200 Subject: [PATCH 116/226] Improve naming, make it possible to create directories recursively with SFTP --- lemur/plugins/lemur_sftp/plugin.py | 52 ++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 7c4e93c4..f2a8c9bf 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -16,7 +16,7 @@ .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov """ -from os import path +from os import path, walk import paramiko @@ -121,15 +121,17 @@ class SFTPDestinationPlugin(DestinationPlugin): self.upload_file(dst_path_cn, files, options) # this is called from the acme http challenge - def upload_acme_token(self, token, thumbprint, options, **kwargs): + def upload_acme_token(self, token_path, token, options, **kwargs): current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge") dst_path = self.get_option("destinationPath", options) - dst_path = path.join(dst_path, "/.well-known/acme-challenge/") + dst_path = path.join(dst_path, ".well-known/acme-challenge/") + + _, filename = path.split(token_path) # prepare files for upload - files = {token: thumbprint} + files = {filename: token} self.upload_file(dst_path, files, options) @@ -169,15 +171,37 @@ class SFTPDestinationPlugin(DestinationPlugin): ) raise paramiko.ssh_exception.AuthenticationException + # split the path into it's segments, so we can create it recursively + allparts = [] + path_copy = dst_path + while True: + parts = path.split(path_copy) + if parts[0] == path_copy: # sentinel for absolute paths + allparts.insert(0, parts[0]) + break + elif parts[1] == path_copy: # sentinel for relative paths + allparts.insert(0, parts[1]) + break + else: + path_copy = parts[0] + allparts.insert(0, parts[1]) + # open the sftp session inside the ssh connection sftp = ssh.open_sftp() - # make sure that the destination path exist - try: - current_app.logger.debug("Creating {0}".format(dst_path)) - sftp.mkdir(dst_path) - except IOError: - current_app.logger.debug("{0} already exist, resuming".format(dst_path)) + # make sure that the destination path exists, recursively + remote_path = allparts[0] + for part in allparts: + try: + if part != "/" and part != "": + remote_path = path.join(remote_path, part); + sftp.stat(remote_path) + except IOError: + current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path)) + try: + sftp.mkdir(remote_path) + except IOError as ioerror: + current_app.logger.debug("Couldn't create {0}, error message: {1}".format(remote_path, ioerror)) # upload certificate files to the sftp destination for filename, data in files.items(): @@ -185,7 +209,7 @@ class SFTPDestinationPlugin(DestinationPlugin): "Uploading {0} to {1}".format(filename, dst_path) ) try: - with sftp.open(dst_path + "/" + filename, "w") as f: + with sftp.open(path.join(dst_path, filename), "w") as f: f.write(data) except PermissionError as permerror: if permerror.errno == 13: @@ -193,11 +217,11 @@ class SFTPDestinationPlugin(DestinationPlugin): "Uploading {0} to {1} returned Permission Denied Error, making file writable and retrying".format( filename, dst_path) ) - sftp.chmod(dst_path + "/" + filename, 0o600) - with sftp.open(dst_path + "/" + filename, "w") as f: + sftp.chmod(path.join(dst_path, filename), 0o600) + with sftp.open(path.join(dst_path, filename), "w") as f: f.write(data) # read only for owner, -r-------- - sftp.chmod(dst_path + "/" + filename, 0o400) + sftp.chmod(path.join(dst_path, filename), 0o400) ssh.close() From e06bdcf2a3d75a598a09b73f613d0e59b8604774 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 29 Sep 2020 18:33:40 +0200 Subject: [PATCH 117/226] Implement create_certificate for HTTP-01 challenge --- lemur/plugins/lemur_acme/plugin.py | 73 ++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index d1c86017..f8165c4d 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -30,6 +30,8 @@ from lemur.destinations import service as destination_service from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider from lemur.extensions import metrics, sentry + +from lemur.plugins.base import plugins from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns @@ -83,19 +85,19 @@ class AcmeHandler(object): def maybe_add_extension(self, host, dns_provider_options): if dns_provider_options and dns_provider_options.get( - "acme_challenge_extension" + "acme_challenge_extension" ): host = host + dns_provider_options.get("acme_challenge_extension") return host def start_dns_challenge( - self, - acme_client, - account_number, - host, - dns_provider, - order, - dns_provider_options, + self, + acme_client, + account_number, + host, + dns_provider, + order, + dns_provider_options, ): current_app.logger.debug("Starting DNS challenge for {0}".format(host)) @@ -210,12 +212,12 @@ class AcmeHandler(object): if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: pem_certificate_chain = orderr.fullchain_pem[ - len(pem_certificate) : # noqa - ].lstrip() + len(pem_certificate): # noqa + ].lstrip() current_app.logger.debug( "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) @@ -692,6 +694,7 @@ class ACMEIssuerPlugin(IssuerPlugin): account_number = None provider_type = None + acme_client.new_order() domains = self.acme.get_domains(issuer_options) if not create_immediately: # Create pending authorizations that we'll need to do the creation @@ -810,7 +813,51 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): self.destination_list.append(destination.label) def create_certificate(self, csr, issuer_options): - pass + """ + Creates an ACME certificate using the HTTP-01 challenge. + + :param csr: + :param issuer_options: + :return: :raise Exception: + """ + self.acme = AcmeHandler() + authority = issuer_options.get("authority") + create_immediately = issuer_options.get("create_immediately", False) + acme_client, registration = self.acme.setup_acme_client(authority) + + orderr = acme_client.new_order(csr) + challenge = None + + for authz in orderr.authorizations: + # Choosing challenge. + # authz.body.challenges is a set of ChallengeBody objects. + for i in authz.body.challenges: + # Find the supported challenge. + if isinstance(i.chall, challenges.HTTP01): + challenge = i + + if challenge is None: + raise Exception('HTTP-01 challenge was not offered by the CA server.') + else: + # Here we probably should create a pending certificate and make use of celery, but for now + # I'll ignore all of that + for option in json.loads(issuer_options["authority"].options): + if option["name"] == "tokenDestination": + token_destination = destination_service.get_by_label(option["value"]) + + if token_destination is None: + raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') + + destination_plugin = plugins.get(token_destination.plugin_name) + destination_plugin.upload_acme_token(challenge.chall.path, challenge.chall.token, token_destination.options) + + current_app.logger.info("Uploaded HTTP-01 challenge token, trying to poll and finalize the order") + + pem_certificate, pem_certificate_chain = self.acme.request_certificate( + acme_client, orderr.authorizations, csr + ) + # TODO add external ID (if possible) + return pem_certificate, pem_certificate_chain, None @staticmethod def create_authority(options): @@ -837,4 +884,4 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): # Needed to override issuer function. - pass \ No newline at end of file + pass From b93d271f318d56935f741e1846e8525c890c2f81 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 08:39:41 +0200 Subject: [PATCH 118/226] Fix flake8 --- lemur/plugins/lemur_acme/plugin.py | 2 +- lemur/plugins/lemur_sftp/plugin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f8165c4d..d42573df 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -212,7 +212,7 @@ class AcmeHandler(object): if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: pem_certificate_chain = orderr.fullchain_pem[ diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index f2a8c9bf..e44052d2 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -16,7 +16,7 @@ .. moduleauthor:: Dmitry Zykov https://github.com/DmitryZykov """ -from os import path, walk +from os import path import paramiko @@ -194,7 +194,7 @@ class SFTPDestinationPlugin(DestinationPlugin): for part in allparts: try: if part != "/" and part != "": - remote_path = path.join(remote_path, part); + remote_path = path.join(remote_path, part) sftp.stat(remote_path) except IOError: current_app.logger.debug("{0} doesn't exist, trying to create it".format(remote_path)) From b2de9866528ade2fe815ae2656ab90b602e17b62 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 09:24:26 +0200 Subject: [PATCH 119/226] Split tests into handler, and dns specifics --- .../tests/{test_acme.py => test_acme_dns.py} | 88 +------- .../lemur_acme/tests/test_acme_handler.py | 194 ++++++++++++++++++ 2 files changed, 196 insertions(+), 86 deletions(-) rename lemur/plugins/lemur_acme/tests/{test_acme.py => test_acme_dns.py} (81%) create mode 100644 lemur/plugins/lemur_acme/tests/test_acme_handler.py diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py similarity index 81% rename from lemur/plugins/lemur_acme/tests/test_acme.py rename to lemur/plugins/lemur_acme/tests/test_acme_dns.py index ab246563..8074ca93 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -8,7 +8,7 @@ from lemur.common.utils import generate_private_key from mock import MagicMock -class TestAcme(unittest.TestCase): +class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") def setUp(self, mock_dns_provider_service): self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() @@ -39,19 +39,6 @@ class TestAcme(unittest.TestCase): result = yield self.acme.get_dns_challenges(host, mock_authz) self.assertEqual(result, mock_entry) - def test_strip_wildcard(self): - expected = ("example.com", False) - result = self.acme.strip_wildcard("example.com") - self.assertEqual(expected, result) - - expected = ("example.com", True) - result = self.acme.strip_wildcard("*.example.com") - self.assertEqual(expected, result) - - def test_authz_record(self): - a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") - self.assertEqual(type(a), plugin.AuthorizationRecord) - @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) @@ -68,7 +55,7 @@ class TestAcme(unittest.TestCase): from acme import challenges c = challenges.DNS01() - mock_entry.chall = TestAcme.test_complete_dns_challenge_fail + mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail mock_authz.body.resolved_combinations.append(mock_entry) mock_acme.request_domain_challenges = Mock(return_value=mock_authz) mock_dns_provider = Mock() @@ -336,77 +323,6 @@ class TestAcme(unittest.TestCase): dyn = provider.get_dns_provider("dyn") assert dyn - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificate(mock_cert) - self.assertEqual( - result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} - ) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - mock_cert2 = Mock() - mock_cert2.external_id = 2 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificates([mock_cert, mock_cert2]) - self.assertEqual(len(result), 2) - self.assertEqual( - result[0]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, - ) - self.assertEqual( - result[1]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, - ) - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py new file mode 100644 index 00000000..60ebf409 --- /dev/null +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -0,0 +1,194 @@ +import unittest +from unittest.mock import patch, Mock + +from cryptography.x509 import DNSName +from lemur.plugins.lemur_acme import plugin +from mock import MagicMock + + +class TestAcmeHandler(unittest.TestCase): + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + def setUp(self, mock_dns_provider_service): + self.acme = plugin.AcmeHandler() + mock_dns_provider = Mock() + mock_dns_provider.name = "cloudflare" + mock_dns_provider.credentials = "{}" + mock_dns_provider.provider_type = "cloudflare" + self.acme.dns_providers_for_domain = { + "www.test.com": [mock_dns_provider], + "test.fakedomain.net": [mock_dns_provider], + } + + def test_strip_wildcard(self): + expected = ("example.com", False) + result = self.acme.strip_wildcard("example.com") + self.assertEqual(expected, result) + + expected = ("example.com", True) + result = self.acme.strip_wildcard("*.example.com") + self.assertEqual(expected, result) + + def test_authz_record(self): + a = plugin.AuthorizationRecord("host", "authz", "challenge", "id") + self.assertEqual(type(a), plugin.AuthorizationRecord) + + def test_setup_acme_client_fail(self): + mock_authority = Mock() + mock_authority.options = [] + with self.assertRaises(Exception): + self.acme.setup_acme_client(mock_authority) + + @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_setup_acme_client_success(self, mock_current_app, mock_acme): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_client = Mock() + mock_registration = Mock() + mock_registration.uri = "http://test.com" + mock_client.register = mock_registration + mock_client.agree_to_tos = Mock(return_value=True) + mock_acme.return_value = mock_client + mock_current_app.config = {} + result_client, result_registration = self.acme.setup_acme_client(mock_authority) + assert result_client + assert result_registration + + @patch('lemur.plugins.lemur_acme.plugin.current_app') + def test_get_domains_single(self, mock_current_app): + options = {"common_name": "test.netflix.net"} + result = self.acme.get_domains(options) + self.assertEqual(result, [options["common_name"]]) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_get_domains_multiple(self, mock_current_app): + options = { + "common_name": "test.netflix.net", + "extensions": { + "sub_alt_names": {"names": [DNSName("test2.netflix.net"), DNSName("test3.netflix.net")]} + }, + } + result = self.acme.get_domains(options) + self.assertEqual( + result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] + ) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_get_domains_san(self, mock_current_app): + options = { + "common_name": "test.netflix.net", + "extensions": { + "sub_alt_names": {"names": [DNSName("test.netflix.net"), DNSName("test2.netflix.net")]} + }, + } + result = self.acme.get_domains(options) + self.assertEqual( + result, [options["common_name"], "test2.netflix.net"] + ) + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", + return_value="test", + ) + def test_get_authorizations(self, mock_start_dns_challenge): + mock_order = Mock() + mock_order.body.identifiers = [] + mock_domain = Mock() + mock_order.body.identifiers.append(mock_domain) + mock_order_info = Mock() + mock_order_info.account_number = 1 + mock_order_info.domains = ["test.fakedomain.net"] + result = self.acme.get_authorizations( + "acme_client", mock_order, mock_order_info + ) + self.assertEqual(result, ["test"]) + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge", + return_value="test", + ) + def test_finalize_authorizations(self, mock_complete_dns_challenge): + mock_authz = [] + mock_authz_record = MagicMock() + mock_authz_record.authz = Mock() + mock_authz_record.change_id = 1 + mock_authz_record.dns_challenge.validation_domain_name = Mock() + mock_authz_record.dns_challenge.validation = Mock() + mock_authz.append(mock_authz_record) + mock_dns_provider = Mock() + mock_dns_provider.delete_txt_record = Mock() + + mock_acme_client = Mock() + result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) + self.assertEqual(result, mock_authz) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + def test_get_ordered_certificate( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificate(mock_cert) + self.assertEqual( + result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} + ) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + def test_get_ordered_certificates( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + mock_cert2 = Mock() + mock_cert2.external_id = 2 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificates([mock_cert, mock_cert2]) + self.assertEqual(len(result), 2) + self.assertEqual( + result[0]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, + ) + self.assertEqual( + result[1]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, + ) From d6719b729c93aaa1806f901a8efb8197b04f6e0b Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 10:23:49 +0200 Subject: [PATCH 120/226] Implement some test for AcmeHttpIssuerPlugin --- .../lemur_acme/tests/test_acme_http.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 lemur/plugins/lemur_acme/tests/test_acme_http.py diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py new file mode 100644 index 00000000..f6183062 --- /dev/null +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -0,0 +1,122 @@ +import unittest +from unittest.mock import patch, Mock + +from acme import challenges +from lemur.plugins.lemur_acme import plugin +from mock import MagicMock + + +class TestAcmeHttp(unittest.TestCase): + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + def setUp(self, mock_dns_provider_service, mock_destination_provider): + self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin() + self.acme = plugin.AcmeHandler() + mock_dns_provider = Mock() + mock_dns_provider.name = "cloudflare" + mock_dns_provider.credentials = "{}" + mock_dns_provider.provider_type = "cloudflare" + self.acme.dns_providers_for_domain = { + "www.test.com": [mock_dns_provider], + "test.fakedomain.net": [mock_dns_provider], + } + mock_destination_provider = Mock() + mock_destination_provider.label = "mock-sftp-destination" + mock_destination_provider.plugin_name = "sftp-destination" + self.ACMEHttpIssuerPlugin.destination_list = ["mock-sftp-destination", "mock-s3-destination"] + + @patch("acme.client.Client") + @patch("OpenSSL.crypto", return_value="mock_cert") + @patch("josepy.util.ComparableX509") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_request_certificate( + self, + mock_current_app, + mock_get_dns_challenges, + mock_jose, + mock_crypto, + mock_acme, + ): + mock_cert_response = Mock() + mock_cert_response.body = "123" + mock_cert_response_full = [mock_cert_response, True] + mock_acme.poll_and_request_issuance = Mock(return_value=mock_cert_response_full) + mock_authz = [] + mock_authz_record = MagicMock() + mock_authz_record.authz = Mock() + mock_authz.append(mock_authz_record) + mock_acme.fetch_chain = Mock(return_value="mock_chain") + mock_crypto.dump_certificate = Mock(return_value=b"chain") + mock_order = Mock() + mock_current_app.config = {} + self.acme.request_certificate(mock_acme, [], mock_order) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_create_authority(self, mock_current_app): + mock_current_app.config = Mock() + options = { + "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} + } + acme_root, b, role = self.ACMEHttpIssuerPlugin.create_authority(options) + self.assertEqual(acme_root, "123") + self.assertEqual(b, "") + self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}]) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate( + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + mock_dns_provider = Mock() + mock_dns_provider.credentials = '{"account_id": 1}' + mock_dns_provider.provider_type = "route53" + mock_dns_provider_service.get.return_value = mock_dns_provider + + mock_destination = Mock() + mock_destination.label = "mock-sftp-destination" + mock_destination.plugin_name = "SFTPDestinationPlugin" + mock_destination_service.get_by_label.return_value = mock_destination + + mock_destination_plugin = Mock() + mock_destination_plugin.upload_acme_token.return_value = True + mock_plugin_manager_get.return_value = mock_destination_plugin + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + result = provider.create_certificate(csr, issuer_options) + assert result From 76dcfbd528aa3a835e0062be0685ee5cba7edbca Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 11:35:27 +0200 Subject: [PATCH 121/226] Add more tests --- lemur/plugins/lemur_acme/plugin.py | 1 + .../lemur_acme/tests/test_acme_http.py | 132 +++++++++++++----- 2 files changed, 100 insertions(+), 33 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index d42573df..a3d15b6d 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -841,6 +841,7 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): else: # Here we probably should create a pending certificate and make use of celery, but for now # I'll ignore all of that + token_destination = None for option in json.loads(issuer_options["authority"].options): if option["name"] == "tokenDestination": token_destination = destination_service.get_by_label(option["value"]) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index f6183062..14d46344 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -3,7 +3,6 @@ from unittest.mock import patch, Mock from acme import challenges from lemur.plugins.lemur_acme import plugin -from mock import MagicMock class TestAcmeHttp(unittest.TestCase): @@ -25,33 +24,6 @@ class TestAcmeHttp(unittest.TestCase): mock_destination_provider.plugin_name = "sftp-destination" self.ACMEHttpIssuerPlugin.destination_list = ["mock-sftp-destination", "mock-s3-destination"] - @patch("acme.client.Client") - @patch("OpenSSL.crypto", return_value="mock_cert") - @patch("josepy.util.ComparableX509") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_request_certificate( - self, - mock_current_app, - mock_get_dns_challenges, - mock_jose, - mock_crypto, - mock_acme, - ): - mock_cert_response = Mock() - mock_cert_response.body = "123" - mock_cert_response_full = [mock_cert_response, True] - mock_acme.poll_and_request_issuance = Mock(return_value=mock_cert_response_full) - mock_authz = [] - mock_authz_record = MagicMock() - mock_authz_record.authz = Mock() - mock_authz.append(mock_authz_record) - mock_acme.fetch_chain = Mock(return_value="mock_chain") - mock_crypto.dump_certificate = Mock(return_value=b"chain") - mock_order = Mock() - mock_current_app.config = {} - self.acme.request_certificate(mock_acme, [], mock_order) - @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_create_authority(self, mock_current_app): mock_current_app.config = Mock() @@ -97,11 +69,6 @@ class TestAcmeHttp(unittest.TestCase): mock_client.new_order.return_value = mock_order_resource mock_acme.return_value = (mock_client, "") - mock_dns_provider = Mock() - mock_dns_provider.credentials = '{"account_id": 1}' - mock_dns_provider.provider_type = "route53" - mock_dns_provider_service.get.return_value = mock_dns_provider - mock_destination = Mock() mock_destination.label = "mock-sftp-destination" mock_destination.plugin_name = "SFTPDestinationPlugin" @@ -120,3 +87,102 @@ class TestAcmeHttp(unittest.TestCase): mock_request_certificate.return_value = ("pem_certificate", "chain") result = provider.create_certificate(csr, issuer_options) assert result + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate_missing_destination_token( + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + mock_destination = Mock() + mock_destination.label = "mock-sftp-destination" + mock_destination.plugin_name = "SFTPDestinationPlugin" + mock_destination_service.get_by_label.return_value = mock_destination + + mock_destination_plugin = Mock() + mock_destination_plugin.upload_acme_token.return_value = True + mock_plugin_manager_get.return_value = mock_destination_plugin + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + with self.assertRaisesRegexp(Exception, "No token_destination configured"): + provider.create_certificate(csr, issuer_options) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.base.manager.PluginManager.get") + @patch("lemur.plugins.lemur_acme.plugin.destination_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + def test_create_certificate_missing_http_challenge( + self, + mock_authorization_service, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_current_app, + mock_dns_provider_service, + mock_destination_service, + mock_plugin_manager_get, + mock_acme, + ): + provider = plugin.ACMEHttpIssuerPlugin() + mock_authority = Mock() + mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + + mock_order_resource = Mock() + mock_order_resource.authorizations = [Mock()] + mock_order_resource.authorizations[0].body.challenges = [Mock()] + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.DNS01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + + mock_client = Mock() + mock_client.new_order.return_value = mock_order_resource + mock_acme.return_value = (mock_client, "") + + issuer_options = { + "authority": mock_authority, + "tokenDestination": "mock-sftp-destination", + "common_name": "test.netflix.net", + } + csr = "123" + mock_request_certificate.return_value = ("pem_certificate", "chain") + with self.assertRaisesRegexp(Exception, "HTTP-01 challenge was not offered"): + provider.create_certificate(csr, issuer_options) From e3e5ef7d66a35cfcb35ddefd3dbeb0e143e43206 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 11:56:12 +0200 Subject: [PATCH 122/226] Refactor AcmeHandler, Move DNS stuff into AcmeDnsHandler --- lemur/plugins/lemur_acme/plugin.py | 264 +++++++++--------- .../plugins/lemur_acme/tests/test_acme_dns.py | 119 +++++++- .../lemur_acme/tests/test_acme_handler.py | 119 +------- .../lemur_acme/tests/test_acme_http.py | 30 +- 4 files changed, 250 insertions(+), 282 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index a3d15b6d..c82ac529 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -48,33 +48,6 @@ class AuthorizationRecord(object): class AcmeHandler(object): - def __init__(self): - self.dns_providers_for_domain = {} - try: - self.all_dns_providers = dns_provider_service.get_all_dns_providers() - except Exception as e: - metrics.send("AcmeHandler_init_error", "counter", 1) - sentry.captureException() - current_app.logger.error(f"Unable to fetch DNS Providers: {e}") - self.all_dns_providers = [] - - def get_dns_challenges(self, host, authorizations): - """Get dns challenges for provided domain""" - - domain_to_validate, is_wildcard = self.strip_wildcard(host) - dns_challenges = [] - for authz in authorizations: - if not authz.body.identifier.value.lower() == domain_to_validate.lower(): - continue - if is_wildcard and not authz.body.wildcard: - continue - if not is_wildcard and authz.body.wildcard: - continue - for combo in authz.body.challenges: - if isinstance(combo.chall, challenges.DNS01): - dns_challenges.append(combo) - - return dns_challenges def strip_wildcard(self, host): """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" @@ -90,91 +63,6 @@ class AcmeHandler(object): host = host + dns_provider_options.get("acme_challenge_extension") return host - def start_dns_challenge( - self, - acme_client, - account_number, - host, - dns_provider, - order, - dns_provider_options, - ): - current_app.logger.debug("Starting DNS challenge for {0}".format(host)) - - change_ids = [] - dns_challenges = self.get_dns_challenges(host, order.authorizations) - host_to_validate, _ = self.strip_wildcard(host) - host_to_validate = self.maybe_add_extension( - host_to_validate, dns_provider_options - ) - - if not dns_challenges: - sentry.captureException() - metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1) - raise Exception("Unable to determine DNS challenges from authorizations") - - for dns_challenge in dns_challenges: - change_id = dns_provider.create_txt_record( - dns_challenge.validation_domain_name(host_to_validate), - dns_challenge.validation(acme_client.client.net.key), - account_number, - ) - change_ids.append(change_id) - - return AuthorizationRecord( - host, order.authorizations, dns_challenges, change_ids - ) - - def complete_dns_challenge(self, acme_client, authz_record): - current_app.logger.debug( - "Finalizing DNS challenge for {0}".format( - authz_record.authz[0].body.identifier.value - ) - ) - dns_providers = self.dns_providers_for_domain.get(authz_record.host) - if not dns_providers: - metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) - raise Exception( - "No DNS providers found for domain: {}".format(authz_record.host) - ) - - for dns_provider in dns_providers: - # Grab account number (For Route53) - dns_provider_options = json.loads(dns_provider.credentials) - account_number = dns_provider_options.get("account_id") - dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) - for change_id in authz_record.change_id: - try: - dns_provider_plugin.wait_for_dns_change( - change_id, account_number=account_number - ) - except Exception: - metrics.send("complete_dns_challenge_error", "counter", 1) - sentry.captureException() - current_app.logger.debug( - f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " - f"{account_number}", - exc_info=True, - ) - raise - - for dns_challenge in authz_record.dns_challenge: - response = dns_challenge.response(acme_client.client.net.key) - - verified = response.simple_verify( - dns_challenge.chall, - authz_record.host, - acme_client.client.net.key.public_key(), - ) - - if not verified: - metrics.send("complete_dns_challenge_verification_error", "counter", 1) - raise ValueError("Failed verification") - - time.sleep(5) - res = acme_client.answer_challenge(dns_challenge, response) - current_app.logger.debug(f"answer_challenge response: {res}") - def request_certificate(self, acme_client, authorizations, order): for authorization in authorizations: for authz in authorization.authz: @@ -310,6 +198,135 @@ class AcmeHandler(object): current_app.logger.debug("Got these domains: {0}".format(domains)) return domains + +class AcmeDnsHandler(AcmeHandler): + + def __init__(self): + self.dns_providers_for_domain = {} + try: + self.all_dns_providers = dns_provider_service.get_all_dns_providers() + except Exception as e: + metrics.send("AcmeHandler_init_error", "counter", 1) + sentry.captureException() + current_app.logger.error(f"Unable to fetch DNS Providers: {e}") + self.all_dns_providers = [] + + def get_dns_challenges(self, host, authorizations): + """Get dns challenges for provided domain""" + + domain_to_validate, is_wildcard = self.strip_wildcard(host) + dns_challenges = [] + for authz in authorizations: + if not authz.body.identifier.value.lower() == domain_to_validate.lower(): + continue + if is_wildcard and not authz.body.wildcard: + continue + if not is_wildcard and authz.body.wildcard: + continue + for combo in authz.body.challenges: + if isinstance(combo.chall, challenges.DNS01): + dns_challenges.append(combo) + + return dns_challenges + + def get_dns_provider(self, type): + provider_types = { + "cloudflare": cloudflare, + "dyn": dyn, + "route53": route53, + "ultradns": ultradns, + "powerdns": powerdns + } + provider = provider_types.get(type) + if not provider: + raise UnknownProvider("No such DNS provider: {}".format(type)) + return provider + + def start_dns_challenge( + self, + acme_client, + account_number, + host, + dns_provider, + order, + dns_provider_options, + ): + current_app.logger.debug("Starting DNS challenge for {0}".format(host)) + + change_ids = [] + dns_challenges = self.get_dns_challenges(host, order.authorizations) + host_to_validate, _ = self.strip_wildcard(host) + host_to_validate = self.maybe_add_extension( + host_to_validate, dns_provider_options + ) + + if not dns_challenges: + sentry.captureException() + metrics.send("start_dns_challenge_error_no_dns_challenges", "counter", 1) + raise Exception("Unable to determine DNS challenges from authorizations") + + for dns_challenge in dns_challenges: + change_id = dns_provider.create_txt_record( + dns_challenge.validation_domain_name(host_to_validate), + dns_challenge.validation(acme_client.client.net.key), + account_number, + ) + change_ids.append(change_id) + + return AuthorizationRecord( + host, order.authorizations, dns_challenges, change_ids + ) + + def complete_dns_challenge(self, acme_client, authz_record): + current_app.logger.debug( + "Finalizing DNS challenge for {0}".format( + authz_record.authz[0].body.identifier.value + ) + ) + dns_providers = self.dns_providers_for_domain.get(authz_record.host) + if not dns_providers: + metrics.send("complete_dns_challenge_error_no_dnsproviders", "counter", 1) + raise Exception( + "No DNS providers found for domain: {}".format(authz_record.host) + ) + + for dns_provider in dns_providers: + # Grab account number (For Route53) + dns_provider_options = json.loads(dns_provider.credentials) + account_number = dns_provider_options.get("account_id") + dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) + for change_id in authz_record.change_id: + try: + dns_provider_plugin.wait_for_dns_change( + change_id, account_number=account_number + ) + except Exception: + metrics.send("complete_dns_challenge_error", "counter", 1) + sentry.captureException() + current_app.logger.debug( + f"Unable to resolve DNS challenge for change_id: {change_id}, account_id: " + f"{account_number}", + exc_info=True, + ) + raise + + for dns_challenge in authz_record.dns_challenge: + response = dns_challenge.response(acme_client.client.net.key) + + verified = response.simple_verify( + dns_challenge.chall, + authz_record.host, + acme_client.client.net.key.public_key(), + ) + + if not verified: + metrics.send("complete_dns_challenge_verification_error", "counter", 1) + raise ValueError("Failed verification") + + time.sleep(5) + res = acme_client.answer_challenge(dns_challenge, response) + current_app.logger.debug(f"answer_challenge response: {res}") + def get_authorizations(self, acme_client, order, order_info): authorizations = [] @@ -421,19 +438,6 @@ class AcmeHandler(object): sentry.captureException() pass - def get_dns_provider(self, type): - provider_types = { - "cloudflare": cloudflare, - "dyn": dyn, - "route53": route53, - "ultradns": ultradns, - "powerdns": powerdns - } - provider = provider_types.get(type) - if not provider: - raise UnknownProvider("No such DNS provider: {}".format(type)) - return provider - class ACMEIssuerPlugin(IssuerPlugin): title = "Acme" @@ -487,7 +491,7 @@ class ACMEIssuerPlugin(IssuerPlugin): super(ACMEIssuerPlugin, self).__init__(*args, **kwargs) def get_dns_provider(self, type): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() provider_types = { "cloudflare": cloudflare, @@ -502,14 +506,14 @@ class ACMEIssuerPlugin(IssuerPlugin): return provider def get_all_zones(self, dns_provider): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() dns_provider_options = json.loads(dns_provider.credentials) account_number = dns_provider_options.get("account_id") dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) return dns_provider_plugin.get_zones(account_number=account_number) def get_ordered_certificate(self, pending_cert): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() acme_client, registration = self.acme.setup_acme_client(pending_cert.authority) order_info = authorization_service.get(pending_cert.external_id) if pending_cert.dns_provider_id: @@ -555,7 +559,7 @@ class ACMEIssuerPlugin(IssuerPlugin): return cert def get_ordered_certificates(self, pending_certs): - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() pending = [] certs = [] for pending_cert in pending_certs: @@ -665,7 +669,7 @@ class ACMEIssuerPlugin(IssuerPlugin): :param issuer_options: :return: :raise Exception: """ - self.acme = AcmeHandler() + self.acme = AcmeDnsHandler() authority = issuer_options.get("authority") create_immediately = issuer_options.get("create_immediately", False) acme_client, registration = self.acme.setup_acme_client(authority) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py index 8074ca93..6b4371d6 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -12,7 +12,7 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") def setUp(self, mock_dns_provider_service): self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin() - self.acme = plugin.AcmeHandler() + self.acme = plugin.AcmeDnsHandler() mock_dns_provider = Mock() mock_dns_provider.name = "cloudflare" mock_dns_provider.credentials = "{}" @@ -42,7 +42,7 @@ class TestAcmeDns(unittest.TestCase): @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") def test_start_dns_challenge( self, mock_get_dns_challenges, mock_len, mock_app, mock_acme ): @@ -124,7 +124,7 @@ class TestAcmeDns(unittest.TestCase): @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") @patch("josepy.util.ComparableX509") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_request_certificate( self, @@ -326,9 +326,9 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate( self, @@ -360,3 +360,110 @@ class TestAcmeDns(unittest.TestCase): mock_request_certificate.return_value = ("pem_certificate", "chain") result = provider.create_certificate(csr, issuer_options) assert result + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.start_dns_challenge", + return_value="test", + ) + def test_get_authorizations(self, mock_start_dns_challenge): + mock_order = Mock() + mock_order.body.identifiers = [] + mock_domain = Mock() + mock_order.body.identifiers.append(mock_domain) + mock_order_info = Mock() + mock_order_info.account_number = 1 + mock_order_info.domains = ["test.fakedomain.net"] + result = self.acme.get_authorizations( + "acme_client", mock_order, mock_order_info + ) + self.assertEqual(result, ["test"]) + + @patch( + "lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.complete_dns_challenge", + return_value="test", + ) + def test_finalize_authorizations(self, mock_complete_dns_challenge): + mock_authz = [] + mock_authz_record = MagicMock() + mock_authz_record.authz = Mock() + mock_authz_record.change_id = 1 + mock_authz_record.dns_challenge.validation_domain_name = Mock() + mock_authz_record.dns_challenge.validation = Mock() + mock_authz.append(mock_authz_record) + mock_dns_provider = Mock() + mock_dns_provider.delete_txt_record = Mock() + + mock_acme_client = Mock() + result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) + self.assertEqual(result, mock_authz) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") + def test_get_ordered_certificate( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificate(mock_cert) + self.assertEqual( + result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} + ) + + @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") + @patch("lemur.plugins.lemur_acme.plugin.current_app") + @patch("lemur.plugins.lemur_acme.plugin.authorization_service") + @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") + @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") + def test_get_ordered_certificates( + self, + mock_request_certificate, + mock_finalize_authorizations, + mock_get_authorizations, + mock_dns_provider_service, + mock_authorization_service, + mock_current_app, + mock_acme, + ): + mock_client = Mock() + mock_acme.return_value = (mock_client, "") + mock_request_certificate.return_value = ("pem_certificate", "chain") + + mock_cert = Mock() + mock_cert.external_id = 1 + + mock_cert2 = Mock() + mock_cert2.external_id = 2 + + provider = plugin.ACMEIssuerPlugin() + provider.get_dns_provider = Mock() + result = provider.get_ordered_certificates([mock_cert, mock_cert2]) + self.assertEqual(len(result), 2) + self.assertEqual( + result[0]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, + ) + self.assertEqual( + result[1]["cert"], + {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, + ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 60ebf409..b586aa9f 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -3,21 +3,11 @@ from unittest.mock import patch, Mock from cryptography.x509 import DNSName from lemur.plugins.lemur_acme import plugin -from mock import MagicMock class TestAcmeHandler(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - def setUp(self, mock_dns_provider_service): + def setUp(self): self.acme = plugin.AcmeHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "cloudflare" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "cloudflare" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } def test_strip_wildcard(self): expected = ("example.com", False) @@ -85,110 +75,3 @@ class TestAcmeHandler(unittest.TestCase): self.assertEqual( result, [options["common_name"], "test2.netflix.net"] ) - - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.start_dns_challenge", - return_value="test", - ) - def test_get_authorizations(self, mock_start_dns_challenge): - mock_order = Mock() - mock_order.body.identifiers = [] - mock_domain = Mock() - mock_order.body.identifiers.append(mock_domain) - mock_order_info = Mock() - mock_order_info.account_number = 1 - mock_order_info.domains = ["test.fakedomain.net"] - result = self.acme.get_authorizations( - "acme_client", mock_order, mock_order_info - ) - self.assertEqual(result, ["test"]) - - @patch( - "lemur.plugins.lemur_acme.plugin.AcmeHandler.complete_dns_challenge", - return_value="test", - ) - def test_finalize_authorizations(self, mock_complete_dns_challenge): - mock_authz = [] - mock_authz_record = MagicMock() - mock_authz_record.authz = Mock() - mock_authz_record.change_id = 1 - mock_authz_record.dns_challenge.validation_domain_name = Mock() - mock_authz_record.dns_challenge.validation = Mock() - mock_authz.append(mock_authz_record) - mock_dns_provider = Mock() - mock_dns_provider.delete_txt_record = Mock() - - mock_acme_client = Mock() - result = self.acme.finalize_authorizations(mock_acme_client, mock_authz) - self.assertEqual(result, mock_authz) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificate( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificate(mock_cert) - self.assertEqual( - result, {"body": "pem_certificate", "chain": "chain", "external_id": "1"} - ) - - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.authorization_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") - def test_get_ordered_certificates( - self, - mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, - mock_dns_provider_service, - mock_authorization_service, - mock_current_app, - mock_acme, - ): - mock_client = Mock() - mock_acme.return_value = (mock_client, "") - mock_request_certificate.return_value = ("pem_certificate", "chain") - - mock_cert = Mock() - mock_cert.external_id = 1 - - mock_cert2 = Mock() - mock_cert2.external_id = 2 - - provider = plugin.ACMEIssuerPlugin() - provider.get_dns_provider = Mock() - result = provider.get_ordered_certificates([mock_cert, mock_cert2]) - self.assertEqual(len(result), 2) - self.assertEqual( - result[0]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "1"}, - ) - self.assertEqual( - result[1]["cert"], - {"body": "pem_certificate", "chain": "chain", "external_id": "2"}, - ) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index 14d46344..ea81b5c4 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -6,19 +6,11 @@ from lemur.plugins.lemur_acme import plugin class TestAcmeHttp(unittest.TestCase): - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - def setUp(self, mock_dns_provider_service, mock_destination_provider): + def setUp(self, mock_destination_provider): self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin() self.acme = plugin.AcmeHandler() - mock_dns_provider = Mock() - mock_dns_provider.name = "cloudflare" - mock_dns_provider.credentials = "{}" - mock_dns_provider.provider_type = "cloudflare" - self.acme.dns_providers_for_domain = { - "www.test.com": [mock_dns_provider], - "test.fakedomain.net": [mock_dns_provider], - } + mock_destination_provider = Mock() mock_destination_provider.label = "mock-sftp-destination" mock_destination_provider.plugin_name = "sftp-destination" @@ -38,20 +30,14 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate( self, mock_authorization_service, mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, mock_current_app, - mock_dns_provider_service, mock_destination_service, mock_plugin_manager_get, mock_acme, @@ -91,20 +77,14 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate_missing_destination_token( self, mock_authorization_service, mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, mock_current_app, - mock_dns_provider_service, mock_destination_service, mock_plugin_manager_get, mock_acme, @@ -145,20 +125,14 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.plugin.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_authorizations") - @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate_missing_http_challenge( self, mock_authorization_service, mock_request_certificate, - mock_finalize_authorizations, - mock_get_authorizations, mock_current_app, - mock_dns_provider_service, mock_destination_service, mock_plugin_manager_get, mock_acme, From 66cab6abd3e76e818e5987cacea83e6853204c78 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 17:40:15 +0200 Subject: [PATCH 123/226] Make http-01 challenge work for SAN certificates --- lemur/plugins/lemur_acme/plugin.py | 45 ++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index c82ac529..841531a5 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -830,17 +830,17 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): acme_client, registration = self.acme.setup_acme_client(authority) orderr = acme_client.new_order(csr) - challenge = None + chall = [] for authz in orderr.authorizations: # Choosing challenge. # authz.body.challenges is a set of ChallengeBody objects. for i in authz.body.challenges: # Find the supported challenge. if isinstance(i.chall, challenges.HTTP01): - challenge = i + chall.append(i) - if challenge is None: + if len(chall) == 0: raise Exception('HTTP-01 challenge was not offered by the CA server.') else: # Here we probably should create a pending certificate and make use of celery, but for now @@ -854,13 +854,42 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') destination_plugin = plugins.get(token_destination.plugin_name) - destination_plugin.upload_acme_token(challenge.chall.path, challenge.chall.token, token_destination.options) - current_app.logger.info("Uploaded HTTP-01 challenge token, trying to poll and finalize the order") + for challenge in chall: + response, validation = challenge.response_and_validation(acme_client.net.key) + + destination_plugin.upload_acme_token(challenge.chall.path, validation, token_destination.options) + + # Let the CA server know that we are ready for the challenge. + acme_client.answer_challenge(challenge, response) + + current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") + + # Wait for challenge status and then issue a certificate. + + for authz in orderr.authorizations: + authzr, resp = acme_client.poll(authz) + current_app.logger.info(authzr.body.status) + + # It is possible to set a deadline time. + finalized_orderr = acme_client.finalize_order(orderr, datetime.datetime.now() + datetime.timedelta(minutes=1)) + + pem_certificate = OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_PEM, + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, finalized_orderr.fullchain_pem + ), + ).decode() + + if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ + and datetime.datetime.now() < datetime.datetime.strptime( + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") + else: + pem_certificate_chain = finalized_orderr.fullchain_pem[ + len(pem_certificate): # noqa + ].lstrip() - pem_certificate, pem_certificate_chain = self.acme.request_certificate( - acme_client, orderr.authorizations, csr - ) # TODO add external ID (if possible) return pem_certificate, pem_certificate_chain, None From d24fae0bacf0aad20ed088097cbceab308739147 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 30 Sep 2020 17:40:51 +0200 Subject: [PATCH 124/226] Fix permissions on acme token upload, dont append well-known automatically --- lemur/plugins/lemur_acme/plugin.py | 3 +-- lemur/plugins/lemur_sftp/plugin.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 841531a5..a3d3fffe 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -866,7 +866,6 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") # Wait for challenge status and then issue a certificate. - for authz in orderr.authorizations: authzr, resp = acme_client.poll(authz) current_app.logger.info(authzr.body.status) @@ -883,7 +882,7 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): + current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: pem_certificate_chain = finalized_orderr.fullchain_pem[ diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index e44052d2..1c974a28 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -126,7 +126,6 @@ class SFTPDestinationPlugin(DestinationPlugin): current_app.logger.debug("SFTP destination plugin is started for HTTP-01 challenge") dst_path = self.get_option("destinationPath", options) - dst_path = path.join(dst_path, ".well-known/acme-challenge/") _, filename = path.split(token_path) @@ -220,8 +219,8 @@ class SFTPDestinationPlugin(DestinationPlugin): sftp.chmod(path.join(dst_path, filename), 0o600) with sftp.open(path.join(dst_path, filename), "w") as f: f.write(data) - # read only for owner, -r-------- - sftp.chmod(path.join(dst_path, filename), 0o400) + # most likely the upload user isn't the webuser, -rw-r--r-- + sftp.chmod(path.join(dst_path, filename), 0o644) ssh.close() From 41ea59d7e39119f993ca33238f7f5b0fdf0da508 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Thu, 1 Oct 2020 08:08:55 +0200 Subject: [PATCH 125/226] Remove unneeded polling --- lemur/plugins/lemur_acme/plugin.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index a3d3fffe..f02133a4 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -865,11 +865,6 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") - # Wait for challenge status and then issue a certificate. - for authz in orderr.authorizations: - authzr, resp = acme_client.poll(authz) - current_app.logger.info(authzr.body.status) - # It is possible to set a deadline time. finalized_orderr = acme_client.finalize_order(orderr, datetime.datetime.now() + datetime.timedelta(minutes=1)) From 215070b327c98def2d1e86a33d874cd010ede67f Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Thu, 1 Oct 2020 08:09:17 +0200 Subject: [PATCH 126/226] Fix create_certificate tests --- .../lemur_acme/tests/test_acme_http.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index ea81b5c4..2e2ae0fb 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -46,13 +46,23 @@ class TestAcmeHttp(unittest.TestCase): mock_authority = Mock() mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' + mock_current_app.config = {} + mock_order_resource = Mock() mock_order_resource.authorizations = [Mock()] mock_order_resource.authorizations[0].body.challenges = [Mock()] - mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01(token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') + mock_order_resource.authorizations[0].body.challenges[0].response_and_validation.return_value = (Mock(), "Anything-goes") + mock_order_resource.authorizations[0].body.challenges[0].chall = challenges.HTTP01( + token=b'\x0f\x1c\xbe#od\xd1\x9c\xa6j\\\xa4\r\xed\xe5\xbf0pz\xeaxnl)\xea[i\xbc\x95\x08\x96\x1f') mock_client = Mock() mock_client.new_order.return_value = mock_order_resource + mock_client.answer_challenge.return_value = True + + mock_finalized_order = Mock() + mock_finalized_order.fullchain_pem = "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n" + mock_client.finalize_order.return_value = mock_finalized_order + mock_acme.return_value = (mock_client, "") mock_destination = Mock() @@ -71,8 +81,11 @@ class TestAcmeHttp(unittest.TestCase): } csr = "123" mock_request_certificate.return_value = ("pem_certificate", "chain") - result = provider.create_certificate(csr, issuer_options) - assert result + pem_certificate, pem_certificate_chain, _ = provider.create_certificate(csr, issuer_options) + + self.assertEqual(pem_certificate, "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n") + self.assertEqual(pem_certificate_chain, + "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") From 81b078604c7f3d183420b2f6ce8c62aebbff7143 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 6 Oct 2020 11:04:04 +0200 Subject: [PATCH 127/226] Implement revoke certificate for ACME --- lemur/plugins/lemur_acme/plugin.py | 71 ++++++++++++++++++- .../lemur_acme/tests/test_acme_handler.py | 28 ++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f02133a4..b8cbdc55 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -49,6 +49,29 @@ class AuthorizationRecord(object): class AcmeHandler(object): + def reuse_account(self, authority): + if not authority.options: + raise InvalidAuthority("Invalid authority. Options not set") + existing_key = False + existing_regr = False + + for option in json.loads(authority.options): + if option["name"] == "acme_private_key" and option["value"]: + existing_key = True + if option["name"] == "acme_regr" and option["value"]: + existing_regr = True + + if not existing_key and current_app.config.get("ACME_PRIVATE_KEY"): + existing_key = True + + if not existing_regr and current_app.config.get("ACME_REGR"): + existing_regr = True + + if existing_key and existing_regr: + return True + else: + return False + def strip_wildcard(self, host): """Removes the leading *. and returns Host and whether it was removed or not (True/False)""" prefix = "*." @@ -755,6 +778,28 @@ class ACMEIssuerPlugin(IssuerPlugin): # Needed to override issuer function. pass + def revoke_certificate(self, certificate, comments): + self.acme = AcmeDnsHandler() + if not self.acme.reuse_account(certificate.authority): + raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") + acme_client, _ = self.acme.setup_acme_client(certificate.authority) + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, certificate.body)) + + try: + acme_client.revoke(fullchain_com, 0) # revocation reason = 0 + except (errors.ConflictError, errors.ClientError, errors.Error) as e: + # Certificate already revoked. + current_app.logger.error("Certificate revocation failed with message: " + e.detail) + metrics.send("acme_revoke_certificate_failure", "counter", 1) + return False + + current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) + metrics.send("acme_revoke_certificate_success", "counter", 1) + return True + class ACMEHttpIssuerPlugin(IssuerPlugin): title = "Acme HTTP-01" @@ -884,8 +929,8 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): len(pem_certificate): # noqa ].lstrip() - # TODO add external ID (if possible) - return pem_certificate, pem_certificate_chain, None + # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate + return pem_certificate, pem_certificate_chain, validation[0:128] @staticmethod def create_authority(options): @@ -913,3 +958,25 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): def cancel_ordered_certificate(self, pending_cert, **kwargs): # Needed to override issuer function. pass + + def revoke_certificate(self, certificate, comments): + self.acme = AcmeHandler() + if not self.acme.reuse_account(certificate.authority): + raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") + acme_client, _ = self.acme.setup_acme_client(certificate.authority) + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, certificate.body)) + + try: + acme_client.revoke(fullchain_com, 0) # revocation reason = 0 + except (errors.ConflictError, errors.ClientError, errors.Error) as e: + # Certificate already revoked. + current_app.logger.error("Certificate revocation failed with message: " + e.detail) + metrics.send("acme_revoke_certificate_failure", "counter", 1) + return False + + current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) + metrics.send("acme_revoke_certificate_success", "counter", 1) + return True diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index b586aa9f..57ddcee6 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -28,6 +28,34 @@ class TestAcmeHandler(unittest.TestCase): with self.assertRaises(Exception): self.acme.setup_acme_client(mock_authority) + def test_reuse_account_not_defined(self): + mock_authority = Mock() + mock_authority.options = [] + with self.assertRaises(Exception): + self.acme.reuse_account(mock_authority) + + def test_reuse_account_from_authority(self): + mock_authority = Mock() + mock_authority.options = '[{"name": "acme_private_key", "value": "PRIVATE_KEY"}, {"name": "acme_regr", "value": "ACME_REGR"}]' + + self.assertTrue(self.acme.reuse_account(mock_authority)) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_reuse_account_from_config(self, mock_current_app): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_current_app.config = {"ACME_PRIVATE_KEY": "PRIVATE_KEY", "ACME_REGR": "ACME_REGR"} + + self.assertTrue(self.acme.reuse_account(mock_authority)) + + @patch("lemur.plugins.lemur_acme.plugin.current_app") + def test_reuse_account_no_configuration(self, mock_current_app): + mock_authority = Mock() + mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' + mock_current_app.config = {} + + self.assertFalse(self.acme.reuse_account(mock_authority)) + @patch("lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2") @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_setup_acme_client_success(self, mock_current_app, mock_acme): From 235653b55842022588faddff10d1bd9c5d8734c4 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 7 Oct 2020 11:43:17 +0200 Subject: [PATCH 128/226] Refactor destination selection for acme-http authorities, to load destinations dynamically --- lemur/plugins/lemur_acme/plugin.py | 9 ++------- .../angular/authorities/authority/authority.js | 8 +++++++- .../authorities/authority/options.tpl.html | 17 +++++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index b8cbdc55..106103d2 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -721,7 +721,6 @@ class ACMEIssuerPlugin(IssuerPlugin): account_number = None provider_type = None - acme_client.new_order() domains = self.acme.get_domains(issuer_options) if not create_immediately: # Create pending authorizations that we'll need to do the creation @@ -844,9 +843,8 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): }, { "name": "tokenDestination", - "type": "select", + "type": "destinationSelect", "required": True, - "available": destination_list, "helpMessage": "The destination to use to deploy the token.", }, ] @@ -871,7 +869,6 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): """ self.acme = AcmeHandler() authority = issuer_options.get("authority") - create_immediately = issuer_options.get("create_immediately", False) acme_client, registration = self.acme.setup_acme_client(authority) orderr = acme_client.new_order(csr) @@ -888,12 +885,10 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): if len(chall) == 0: raise Exception('HTTP-01 challenge was not offered by the CA server.') else: - # Here we probably should create a pending certificate and make use of celery, but for now - # I'll ignore all of that token_destination = None for option in json.loads(issuer_options["authority"].options): if option["name"] == "tokenDestination": - token_destination = destination_service.get_by_label(option["value"]) + token_destination = destination_service.get(option["value"]) if token_destination is None: raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') diff --git a/lemur/static/app/angular/authorities/authority/authority.js b/lemur/static/app/angular/authorities/authority/authority.js index a449cff5..82f38a92 100644 --- a/lemur/static/app/angular/authorities/authority/authority.js +++ b/lemur/static/app/angular/authorities/authority/authority.js @@ -34,7 +34,7 @@ angular.module('lemur') }; }) - .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster) { + .controller('AuthorityCreateController', function ($scope, $uibModalInstance, AuthorityService, AuthorityApi, LemurRestangular, RoleService, PluginService, WizardHandler, toaster, DestinationService) { $scope.authority = LemurRestangular.restangularizeElement(null, {}, 'authorities'); // set the defaults AuthorityService.getDefaults($scope.authority).then(function () { @@ -52,6 +52,12 @@ angular.module('lemur') }); }); + $scope.getDestinations = function() { + return DestinationService.findDestinationsByName('').then(function(destinations) { + $scope.destinations = destinations; + }); + }; + $scope.getAuthoritiesByName = function (value) { return AuthorityService.findAuthorityByName(value).then(function (authorities) { $scope.authorities = authorities; diff --git a/lemur/static/app/angular/authorities/authority/options.tpl.html b/lemur/static/app/angular/authorities/authority/options.tpl.html index adf8eacc..e683c688 100644 --- a/lemur/static/app/angular/authorities/authority/options.tpl.html +++ b/lemur/static/app/angular/authorities/authority/options.tpl.html @@ -72,11 +72,28 @@
+ + + + + {{$select.selected.label}} + +
+ + + +
+
+ +
diff --git a/lemur/static/app/angular/certificates/certificate/options.tpl.html b/lemur/static/app/angular/certificates/certificate/options.tpl.html index 11b8fe68..2f28a4de 100644 --- a/lemur/static/app/angular/certificates/certificate/options.tpl.html +++ b/lemur/static/app/angular/certificates/certificate/options.tpl.html @@ -32,12 +32,7 @@
From a4178ca113b5a01fe25e9371a4ca89ec393244ae Mon Sep 17 00:00:00 2001 From: csine-nflx Date: Thu, 29 Oct 2020 18:52:22 -0700 Subject: [PATCH 168/226] fixing floating comma in CNAME PR --- lemur/plugins/lemur_acme/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 07324f35..e0e5b495 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -424,7 +424,7 @@ class AcmeHandler(object): dns_provider_plugin = self.get_dns_provider(dns_provider.provider_type) for dns_challenge in dns_challenges: if authz_record.domain == authz_record.target_domain: - host_to_validate = dns_challenge.validation_domain_name(host_to_validate), + host_to_validate = dns_challenge.validation_domain_name(host_to_validate) try: dns_provider_plugin.delete_txt_record( authz_record.change_id, From c5769378cf74356f094af5587b6d0861e3df6798 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 15:21:22 -0700 Subject: [PATCH 169/226] making lint happy --- lemur/plugins/lemur_aws/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 8c94dc45..489fa823 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -408,7 +408,6 @@ class S3DestinationPlugin(ExportDestinationPlugin): account_number=self.get_option("accountNumber", options), ) - def upload_acme_token(self, token_path, token, options, **kwargs): """ This is called from the acme http challenge @@ -483,4 +482,3 @@ class SNSNotificationPlugin(ExpirationNotificationPlugin): current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}") sns.publish(topic_arn, message, notification_type, region_name=self.get_option("region", options)) - From ba8eb7a3f59e8201810f63e6a174fc2e3b0c700d Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:17:02 -0700 Subject: [PATCH 170/226] better logging and metrics --- lemur/plugins/lemur_aws/plugin.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 489fa823..b54787ac 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -33,6 +33,7 @@ .. moduleauthor:: Harm Weites """ +import sys from acme.errors import ClientError from flask import current_app @@ -420,6 +421,8 @@ class S3DestinationPlugin(ExportDestinationPlugin): """ current_app.logger.debug("S3 destination plugin is started for HTTP-01 challenge") + function = f"{__name__}.{sys._getframe().f_code.co_name}" + account_number = self.get_option("accountNumber", options) bucket_name = self.get_option("bucket", options) prefix = self.get_option("prefix", options) @@ -428,12 +431,24 @@ class S3DestinationPlugin(ExportDestinationPlugin): if not prefix.endswith("/"): prefix + "/" - s3.put(bucket_name=bucket_name, - region_name=region, - prefix=prefix + filename, - data=token, - encrypt=False, - account_number=account_number) + res = s3.put(bucket_name=bucket_name, + region_name=region, + prefix=prefix + filename, + data=token, + encrypt=False, + account_number=account_number) + res = "Success" if res else "Failure" + log_data = { + "function": function, + "message": "check if any valid certificate is revoked", + "result": res, + "bucket_name": bucket_name, + "filename": filename + } + current_app.logger.info(log_data) + metrics.send(f"{function}", "counter", 1, metric_tags={"result": res, + "bucket_name": bucket_name, + "filename": filename}) class SNSNotificationPlugin(ExpirationNotificationPlugin): From cc2aa5c1de131ec389df3ffd57059d0fd9ebc134 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:17:34 -0700 Subject: [PATCH 171/226] cli for live testing --- lemur/acme_providers/cli.py | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 310efad1..56301aae 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -1,12 +1,16 @@ import time import json +import arrow from flask_script import Manager from flask import current_app from lemur.extensions import sentry from lemur.constants import SUCCESS_METRIC_STATUS +from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler +from lemur.plugins.lemur_aws import s3 +from lemur.utils import get_random_secret manager = Manager( usage="Handles all ACME related tasks" @@ -84,3 +88,96 @@ def dnstest(domain, token): status = SUCCESS_METRIC_STATUS print("[+] Done with ACME Tests.") + + +@manager.option( + "-t", + "--token", + dest="token", + default="date: " + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="Value of the Token", +) +@manager.option( + "-n", + "--token_name", + dest="token_name", + default="Token-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + required=False, + help="path", +) +@manager.option( + "-p", + "--prefix", + dest="prefix", + default="test/", + required=False, + help="S3 bucket prefix", +) +@manager.option( + "-a", + "--account_number", + dest="account_number", + required=True, + help="AWS Account", +) +@manager.option( + "-b", + "--bucket_name", + dest="bucket_name", + required=True, + help="Bucket Name", +) +def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): + + additional_options = [ + { + "name": "bucket", + "value": bucket_name, + "type": "str", + "required": True, + "validation": "[0-9a-z.-]{3,63}", + "helpMessage": "Must be a valid S3 bucket name!", + }, + { + "name": "accountNumber", + "type": "str", + "value": account_number, + "required": True, + "validation": "[0-9]{12}", + "helpMessage": "A valid AWS account number with permission to access S3", + }, + { + "name": "region", + "type": "str", + "default": "us-east-1", + "required": False, + "helpMessage": "Region bucket exists", + "available": ["us-east-1", "us-west-2", "eu-west-1"], + }, + { + "name": "encrypt", + "type": "bool", + "value": False, + "required": False, + "helpMessage": "Enable server side encryption", + "default": True, + }, + { + "name": "prefix", + "type": "str", + "value": prefix, + "required": False, + "helpMessage": "Must be a valid S3 object prefix!", + }, + ] + + p = plugins.get("aws-s3") + p.upload_acme_token(token_name, token, additional_options) + + if not prefix.endswith("/"): + prefix + "/" + + token_res = s3.get(bucket_name, prefix + token_name, account_number=account_number) + assert(token_res == token) + s3.delete(bucket_name, prefix + token_name, account_number=account_number) From e1ff89eb2d6c3e6fbc8149020c689c0a530a0675 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:18:14 -0700 Subject: [PATCH 172/226] better return arguments --- lemur/plugins/lemur_aws/s3.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index 7c4177ff..e15f6b6e 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -37,8 +37,10 @@ def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): else: try: bucket.put_object(Key=prefix, Body=data, ACL="bucket-owner-full-control") + return True except ClientError: sentry.captureException() + return False @sts_client("s3", service_type="client") @@ -55,6 +57,7 @@ def delete(bucket_name, prefix, **kwargs): return response['ResponseMetadata']['HTTPStatusCode'] < 300 except ClientError: sentry.captureException() + return False @sts_client("s3", service_type="client") @@ -69,3 +72,4 @@ def get(bucket_name, prefix, **kwargs): return response['Body'].read().decode("utf-8") except ClientError: sentry.captureException() + return None From add0960579e7e4bdfbf87a18cc0cf5b1afac4b30 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:18:37 -0700 Subject: [PATCH 173/226] more meaningful variable naming --- lemur/plugins/lemur_aws/s3.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_aws/s3.py b/lemur/plugins/lemur_aws/s3.py index e15f6b6e..1b0831b3 100644 --- a/lemur/plugins/lemur_aws/s3.py +++ b/lemur/plugins/lemur_aws/s3.py @@ -44,15 +44,15 @@ def put(bucket_name, region_name, prefix, data, encrypt, **kwargs): @sts_client("s3", service_type="client") -def delete(bucket_name, prefix, **kwargs): +def delete(bucket_name, prefixed_object_name, **kwargs): """ Use STS to delete an object """ try: - response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefix) + response = kwargs["client"].delete_object(Bucket=bucket_name, Key=prefixed_object_name) current_app.logger.debug(f"Delete data from S3." f"Bucket: {bucket_name}," - f"Prefix: {prefix}," + f"Prefix: {prefixed_object_name}," f"Status_code: {response}") return response['ResponseMetadata']['HTTPStatusCode'] < 300 except ClientError: @@ -61,14 +61,14 @@ def delete(bucket_name, prefix, **kwargs): @sts_client("s3", service_type="client") -def get(bucket_name, prefix, **kwargs): +def get(bucket_name, prefixed_object_name, **kwargs): """ Use STS to get an object """ try: - response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefix) + response = kwargs["client"].get_object(Bucket=bucket_name, Key=prefixed_object_name) current_app.logger.debug(f"Get data from S3. Bucket: {bucket_name}," - f"Prefix: {prefix}") + f"object_name: {prefixed_object_name}") return response['Body'].read().decode("utf-8") except ClientError: sentry.captureException() From 3dfafa00218b384ef420cf48cbd4bbeece8743f7 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:28:10 -0700 Subject: [PATCH 174/226] making lint happy --- lemur/acme_providers/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 56301aae..ec6326bd 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -10,7 +10,6 @@ from lemur.constants import SUCCESS_METRIC_STATUS from lemur.plugins import plugins from lemur.plugins.lemur_acme.plugin import AcmeHandler from lemur.plugins.lemur_aws import s3 -from lemur.utils import get_random_secret manager = Manager( usage="Handles all ACME related tasks" From 7bca42776b6e0d50825dd7d07222076b18ef32f6 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:28:34 -0700 Subject: [PATCH 175/226] better comments --- lemur/acme_providers/cli.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index ec6326bd..313876e6 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -128,7 +128,16 @@ def dnstest(domain, token): help="Bucket Name", ) def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name): - + """ + This method serves for testing the upload_acme_token to S3, fetching the token to verify it, and then deleting it. + It mainly serves for testing purposes. + :param token: + :param token_name: + :param prefix: + :param account_number: + :param bucket_name: + :return: + """ additional_options = [ { "name": "bucket", From 9c6856bcdd7bf93705e02a887bfef3df9cdab542 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 30 Oct 2020 18:36:32 -0700 Subject: [PATCH 176/226] adjusting the tests to the better naming --- lemur/plugins/lemur_aws/tests/test_plugin.py | 2 +- lemur/plugins/lemur_aws/tests/test_s3.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index a471f7c8..a3227296 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -74,7 +74,7 @@ def test_upload_acme_token(app): options=additional_options) response = get(bucket_name=bucket, - prefix=prefix + token_name, + prefixed_object_name=prefix + token_name, encrypt=False, account_number=account) diff --git a/lemur/plugins/lemur_aws/tests/test_s3.py b/lemur/plugins/lemur_aws/tests/test_s3.py index 88bd30d2..7d0fa843 100644 --- a/lemur/plugins/lemur_aws/tests/test_s3.py +++ b/lemur/plugins/lemur_aws/tests/test_s3.py @@ -24,18 +24,18 @@ def test_put_delete_s3_object(app): account_number=account, region=region) - response = get(bucket_name=bucket, prefix=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) # put data, and getting the same data assert (response == data) - response = get(bucket_name="wrong-bucket", prefix=path, account_number=account) + response = get(bucket_name="wrong-bucket", prefixed_object_name=path, account_number=account) # attempting to get thccle wrong data assert (response is None) - delete(bucket_name=bucket, prefix=path, account_number=account) - response = get(bucket_name=bucket, prefix=path, account_number=account) + delete(bucket_name=bucket, prefixed_object_name=path, account_number=account) + response = get(bucket_name=bucket, prefixed_object_name=path, account_number=account) # delete data, and getting the same data assert (response is None) From a4d2f79a9b965225bc8b4715dcc49d24a0662439 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 13:34:06 +0000 Subject: [PATCH 177/226] Bump cryptography from 3.2 to 3.2.1 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.2 to 3.2.1. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.2...3.2.1) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f3d169f3..20808dff 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests colorama==0.4.3 # via twine -cryptography==3.2 # via secretstorage +cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv diff --git a/requirements-docs.txt b/requirements-docs.txt index 6ae23a4c..ceb62e6c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -26,7 +26,7 @@ cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynac chardet==3.0.4 # via -r requirements.txt, requests click==7.1.1 # via -r requirements.txt, flask cloudflare==2.8.13 # via -r requirements.txt -cryptography==3.2 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests +cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt dnspython==1.15.0 # via -r requirements.txt, dnspython3 docutils==0.15.2 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 0bbee7c2..04e9af94 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -19,7 +19,7 @@ cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests click==7.1.2 # via black, flask coverage==5.3 # via -r requirements-tests.in -cryptography==3.2 # via moto, python-jose, sshpubkeys +cryptography==3.2.1 # via moto, python-jose, sshpubkeys decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys diff --git a/requirements.txt b/requirements.txt index d53241fa..165149d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.1.1 # via flask cloudflare==2.8.13 # via -r requirements.in -cryptography==3.2 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests +cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in dnspython==1.15.0 # via dnspython3 dyn==1.8.1 # via -r requirements.in From a15e1831d072ab8dce4042f561d62c995ee240e9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 2 Nov 2020 09:43:09 -0800 Subject: [PATCH 178/226] turning off Travis-ci notification on success --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 67a1d0b1..d753c241 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,3 +48,5 @@ after_success: notifications: email: lemur@netflix.com + on_success: never + on_failure: always From 771c272895395ba730c616d5b76ead08a9950d03 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 18:51:46 +0000 Subject: [PATCH 179/226] Bump pre-commit from 2.7.1 to 2.8.2 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 2.7.1 to 2.8.2. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/master/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v2.7.1...v2.8.2) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 20808dff..180be728 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,7 +24,7 @@ keyring==21.2.0 # via twine mccabe==0.6.1 # via flake8 nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine -pre-commit==2.7.1 # via -r requirements-dev.in +pre-commit==2.8.2 # via -r requirements-dev.in pycodestyle==2.3.1 # via flake8 pycparser==2.20 # via cffi pyflakes==1.6.0 # via flake8 From 2331638ed189fd5e147c204b257e0c0496459547 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:01:19 +0000 Subject: [PATCH 180/226] Bump sphinx from 3.2.1 to 3.3.0 Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 3.2.1 to 3.3.0. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/3.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v3.2.1...v3.3.0) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index ceb62e6c..8d4750e4 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -92,7 +92,7 @@ six==1.15.0 # via -r requirements.txt, acme, bcrypt, cryptography, snowballstemmer==2.0.0 # via sphinx soupsieve==2.0.1 # via -r requirements.txt, beautifulsoup4 sphinx-rtd-theme==0.5.0 # via -r requirements-docs.in -sphinx==3.2.1 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain +sphinx==3.3.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx sphinxcontrib-htmlhelp==1.0.3 # via sphinx From fc2fce6c0b6ddab4882ed1fd6b310792a544c3ac Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:10:18 +0000 Subject: [PATCH 181/226] Bump pytest from 6.1.1 to 6.1.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.1.1 to 6.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.1.1...6.1.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 04e9af94..6566a57c 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -61,7 +61,7 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema pytest-flask==1.0.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in -pytest==6.1.1 # via -r requirements-tests.in, pytest-flask, pytest-mock +pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto python-jose[cryptography]==3.1.0 # via moto pytz==2019.3 # via moto From fa620e539d2b985038821a18dccb994142351b2d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 19:21:05 +0000 Subject: [PATCH 182/226] Bump boto3 from 1.16.5 to 1.16.9 Bumps [boto3](https://github.com/boto/boto3) from 1.16.5 to 1.16.9. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.5...1.16.9) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 4 ++-- requirements-tests.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 8d4750e4..d0f5a47c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.5 # via -r requirements.txt -botocore==1.19.5 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.9 # via -r requirements.txt +botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.6.20 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index 6566a57c..fcc219e9 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.5 # via aws-sam-translator, moto +boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.5 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.6.20 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 165149d5..4e641cfd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.5 # via -r requirements.in -botocore==1.19.5 # via -r requirements.in, boto3, s3transfer +boto3==1.16.9 # via -r requirements.in +botocore==1.19.9 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 4659df42d55080fca09653c2986bb8d3a213c5db Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 2 Nov 2020 15:49:11 -0800 Subject: [PATCH 183/226] better formatting --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d753c241..f9db8d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,7 @@ after_success: notifications: email: - lemur@netflix.com + recipients: + - lemur@netflix.com on_success: never on_failure: always From 634339eac6e0ead00ad8e7bce604643e8d3d43ef Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 14:35:37 -0700 Subject: [PATCH 184/226] replacing imp (deprecated) with importlib --- lemur/factory.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lemur/factory.py b/lemur/factory.py index 0563d873..edea571a 100644 --- a/lemur/factory.py +++ b/lemur/factory.py @@ -10,7 +10,7 @@ """ import os -import imp +import importlib import errno import pkg_resources import socket @@ -73,8 +73,9 @@ def from_file(file_path, silent=False): :param file_path: :param silent: """ - d = imp.new_module("config") - d.__file__ = file_path + module_spec = importlib.util.spec_from_file_location("config", file_path) + d = importlib.util.module_from_spec(module_spec) + try: with open(file_path) as config_file: exec( # nosec: config file safe From 4ffced70f82af0ab98a079ded4f2e472e02184fc Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 14:37:09 -0700 Subject: [PATCH 185/226] backref cannot be set for viewonly relationship will be deprecated in SQLAlchemy 1.4, and will be disallowed in a future release --- lemur/certificates/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index f6562b3f..65918d73 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -184,7 +184,6 @@ class Certificate(db.Model): "PendingCertificate", secondary=pending_cert_replacement_associations, backref="pending_replace", - viewonly=True, ) logs = relationship("Log", backref="certificate") From 2dac95c6fb1737da8739efe8e2c28a746f33cb86 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 17:00:35 -0700 Subject: [PATCH 186/226] Replacing PassiveDefault (deprecated) with DefaultClause --- lemur/authorities/models.py | 4 ++-- lemur/certificates/models.py | 4 ++-- lemur/logs/models.py | 4 ++-- lemur/pending_certificates/models.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lemur/authorities/models.py b/lemur/authorities/models.py index 61ab779e..94985cc9 100644 --- a/lemur/authorities/models.py +++ b/lemur/authorities/models.py @@ -18,7 +18,7 @@ from sqlalchemy import ( func, ForeignKey, DateTime, - PassiveDefault, + DefaultClause, Boolean, ) from sqlalchemy.dialects.postgresql import JSON @@ -39,7 +39,7 @@ class Authority(db.Model): plugin_name = Column(String(64)) description = Column(Text) options = Column(JSON) - date_created = Column(DateTime, PassiveDefault(func.now()), nullable=False) + date_created = Column(DateTime, DefaultClause(func.now()), nullable=False) roles = relationship( "Role", secondary=roles_authorities, diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 65918d73..94e3a42e 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -16,7 +16,7 @@ from sqlalchemy import ( Integer, ForeignKey, String, - PassiveDefault, + DefaultClause, func, Column, Text, @@ -138,7 +138,7 @@ class Certificate(db.Model): not_after = Column(ArrowType) not_after_ix = Index("ix_certificates_not_after", not_after.desc()) - date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False) + date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False) signing_algorithm = Column(String(128)) status = Column(String(128)) diff --git a/lemur/logs/models.py b/lemur/logs/models.py index 07a2ded3..30cc204a 100644 --- a/lemur/logs/models.py +++ b/lemur/logs/models.py @@ -7,7 +7,7 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy import Column, Integer, ForeignKey, PassiveDefault, func, Enum +from sqlalchemy import Column, Integer, ForeignKey, DefaultClause, func, Enum from sqlalchemy_utils.types.arrow import ArrowType @@ -29,5 +29,5 @@ class Log(db.Model): ), nullable=False, ) - logged_at = Column(ArrowType(), PassiveDefault(func.now()), nullable=False) + logged_at = Column(ArrowType(), DefaultClause(func.now()), nullable=False) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) diff --git a/lemur/pending_certificates/models.py b/lemur/pending_certificates/models.py index fa6be073..ee3e5e97 100644 --- a/lemur/pending_certificates/models.py +++ b/lemur/pending_certificates/models.py @@ -9,7 +9,7 @@ from sqlalchemy import ( Integer, ForeignKey, String, - PassiveDefault, + DefaultClause, func, Column, Text, @@ -76,14 +76,14 @@ class PendingCertificate(db.Model): chain = Column(Text()) private_key = Column(Vault, nullable=True) - date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False) + date_created = Column(ArrowType, DefaultClause(func.now()), nullable=False) dns_provider_id = Column( Integer, ForeignKey("dns_providers.id", ondelete="CASCADE") ) status = Column(Text(), nullable=True) last_updated = Column( - ArrowType, PassiveDefault(func.now()), onupdate=func.now(), nullable=False + ArrowType, DefaultClause(func.now()), onupdate=func.now(), nullable=False ) rotation = Column(Boolean, default=False) From d821024e35cc2c7c90d1b7f74d01804725c3ef65 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 17:51:13 -0700 Subject: [PATCH 187/226] Fixing DeprecationWarning: callable is None --- lemur/plugins/lemur_acme/tests/test_acme.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 89ca6ee1..39600eb2 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -130,9 +130,8 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge = [] dns_challenge = Mock() mock_authz.dns_challenge.append(dns_challenge) - self.assertRaises( - ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz) - ) + with self.assertRaises(ValueError): + self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From d88da028b1f69bd90940f1100b1a31f755ae7fd6 Mon Sep 17 00:00:00 2001 From: sayali Date: Fri, 30 Oct 2020 18:12:22 -0700 Subject: [PATCH 188/226] Replace binary with LargeBinary https://flask-appbuilder.readthedocs.io/en/latest/_modules/sqlalchemy/sql/sqltypes.html --- lemur/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/utils.py b/lemur/utils.py index 909d959a..d49a2feb 100644 --- a/lemur/utils.py +++ b/lemur/utils.py @@ -81,7 +81,7 @@ class Vault(types.TypeDecorator): """ # required by SQLAlchemy. defines the underlying column type - impl = types.Binary + impl = types.LargeBinary def process_bind_param(self, value, dialect): """ From 825a001a8b1a49428aeb62626022453992a003cf Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 16:47:40 -0800 Subject: [PATCH 189/226] pass algorithm to jwt.decode() during login api_jwt.py : pass "algorithms" argument when calling decode(). This argument will be mandatory in a future version --- lemur/auth/service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/auth/service.py b/lemur/auth/service.py index 0e1521b3..f954ce51 100644 --- a/lemur/auth/service.py +++ b/lemur/auth/service.py @@ -101,7 +101,8 @@ def login_required(f): return dict(message="Token is invalid"), 403 try: - payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"]) + header_data = fetch_token_header(token) + payload = jwt.decode(token, current_app.config["LEMUR_TOKEN_SECRET"], algorithms=[header_data["alg"]]) except jwt.DecodeError: return dict(message="Token is invalid"), 403 except jwt.ExpiredSignatureError: From 6922d348257a2806bb10b8f45b0d4c241b922bd7 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:16:15 -0800 Subject: [PATCH 190/226] invalid escape sequence \ --- lemur/notifications/service.py | 4 ++-- lemur/plugins/bases/notification.py | 2 +- lemur/plugins/lemur_email/plugin.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 34edccc0..5bc5f3e1 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -43,7 +43,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): "name": "recipients", "type": "str", "required": True, - "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", + "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", "helpMessage": "Comma delimited list of email addresses", "value": ",".join(recipients), }, @@ -63,7 +63,7 @@ def create_default_expiration_notifications(name, recipients, intervals=None): "name": "interval", "type": "int", "required": True, - "validation": "^\d+$", + "validation": r"^\d+$", "helpMessage": "Number of days to be alert before expiration.", "value": i, } diff --git a/lemur/plugins/bases/notification.py b/lemur/plugins/bases/notification.py index 0da0dad2..76aa33de 100644 --- a/lemur/plugins/bases/notification.py +++ b/lemur/plugins/bases/notification.py @@ -42,7 +42,7 @@ class ExpirationNotificationPlugin(NotificationPlugin): "name": "interval", "type": "int", "required": True, - "validation": "^\d+$", + "validation": r"^\d+$", "helpMessage": "Number of days to be alert before expiration.", }, { diff --git a/lemur/plugins/lemur_email/plugin.py b/lemur/plugins/lemur_email/plugin.py index f380c82e..041b27ec 100644 --- a/lemur/plugins/lemur_email/plugin.py +++ b/lemur/plugins/lemur_email/plugin.py @@ -91,7 +91,7 @@ class EmailNotificationPlugin(ExpirationNotificationPlugin): "name": "recipients", "type": "str", "required": True, - "validation": "^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", + "validation": r"^([\w+-.%]+@[\w-.]+\.[A-Za-z]{2,4},?)+$", "helpMessage": "Comma delimited list of email addresses", } ] From b75bd56546b9d2c525a3e6f337ca9f54969045db Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:29:22 -0800 Subject: [PATCH 191/226] Check if ValueError assert works old way --- lemur/plugins/lemur_acme/tests/test_acme.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 39600eb2..6a81d85d 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -130,8 +130,9 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge = [] dns_challenge = Mock() mock_authz.dns_challenge.append(dns_challenge) - with self.assertRaises(ValueError): - self.acme.complete_dns_challenge(mock_acme, mock_authz) + # with self.assertRaises(ValueError): + # self.acme.complete_dns_challenge(mock_acme, mock_authz) + self.assertRaises(ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz)) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From 86b2cfbe4add04692b7d24e2068acfea9f4382b2 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:45:38 -0800 Subject: [PATCH 192/226] invalid escape sequence \ --- lemur/plugins/lemur_acme/plugin.py | 4 ++-- lemur/plugins/lemur_sftp/plugin.py | 2 +- lemur/plugins/lemur_slack/plugin.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e0e5b495..1835971b 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -481,7 +481,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "name": "acme_url", "type": "str", "required": True, - "validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", + "validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", "helpMessage": "Must be a valid web url starting with http[s]://", }, { @@ -494,7 +494,7 @@ class ACMEIssuerPlugin(IssuerPlugin): "name": "email", "type": "str", "default": "", - "validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", + "validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", "helpMessage": "Email to use", }, { diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 66784048..2447cc4e 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -47,7 +47,7 @@ class SFTPDestinationPlugin(DestinationPlugin): "type": "int", "required": True, "helpMessage": "The SFTP port, default is 22.", - "validation": "^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})", + "validation": r"^(6553[0-5]|655[0-2][0-9]\d|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|[1-9](\d){0,3})", "default": "22", }, { diff --git a/lemur/plugins/lemur_slack/plugin.py b/lemur/plugins/lemur_slack/plugin.py index 70d97aa5..3ad22bca 100644 --- a/lemur/plugins/lemur_slack/plugin.py +++ b/lemur/plugins/lemur_slack/plugin.py @@ -89,7 +89,7 @@ class SlackNotificationPlugin(ExpirationNotificationPlugin): "name": "webhook", "type": "str", "required": True, - "validation": "^https:\/\/hooks\.slack\.com\/services\/.+$", + "validation": r"^https:\/\/hooks\.slack\.com\/services\/.+$", "helpMessage": "The url Slack told you to use for this integration", }, { From 3d64aa8d114b81ab933f9c67a161114795c3e1f2 Mon Sep 17 00:00:00 2001 From: sayali Date: Mon, 2 Nov 2020 18:58:38 -0800 Subject: [PATCH 193/226] Fixing DeprecationWarning: callable is None: another syntax --- lemur/plugins/lemur_acme/tests/test_acme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 6a81d85d..cbb4d314 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -132,7 +132,7 @@ class TestAcme(unittest.TestCase): mock_authz.dns_challenge.append(dns_challenge) # with self.assertRaises(ValueError): # self.acme.complete_dns_challenge(mock_acme, mock_authz) - self.assertRaises(ValueError, self.acme.complete_dns_challenge(mock_acme, mock_authz)) + self.assertRaises(ValueError, self.acme.complete_dns_challenge, mock_acme, mock_authz) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From c71dbcb0a06f5940a866738b3ab01e990f7947aa Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 3 Nov 2020 09:48:25 +0100 Subject: [PATCH 194/226] Fix duplicate tests --- lemur/plugins/lemur_acme/tests/test_acme_dns.py | 13 ------------- lemur/plugins/lemur_acme/tests/test_acme_handler.py | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py index ef676e20..3efc3934 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -40,19 +40,6 @@ class TestAcmeDns(unittest.TestCase): result = yield self.acme.get_dns_challenges(host, mock_authz) self.assertEqual(result, mock_entry) - def test_strip_wildcard(self): - expected = ("example.com", False) - result = self.acme.strip_wildcard("example.com") - self.assertEqual(expected, result) - - expected = ("example.com", True) - result = self.acme.strip_wildcard("*.example.com") - self.assertEqual(expected, result) - - def test_authz_record(self): - a = AuthorizationRecord("domain", "host", "authz", "challenge", "id") - self.assertEqual(type(a), AuthorizationRecord) - @patch("acme.client.Client") @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 1456f802..d9f4313d 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -19,7 +19,7 @@ class TestAcmeHandler(unittest.TestCase): self.assertEqual(expected, result) def test_authz_record(self): - a = acme_handlers.AuthorizationRecord("host", "authz", "challenge", "id") + a = acme_handlers.AuthorizationRecord("domain", "host", "authz", "challenge", "id") self.assertEqual(type(a), acme_handlers.AuthorizationRecord) def test_setup_acme_client_fail(self): From dc7497e29d66214d88b358ae6781bf67fe4aae64 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 3 Nov 2020 19:05:18 -0800 Subject: [PATCH 195/226] Fix Working outside of application context Test Failures in dev --- lemur/plugins/lemur_acme/tests/test_acme.py | 10 ++++++++++ lemur/plugins/lemur_acme/tests/test_powerdns.py | 11 +++++++++++ lemur/plugins/lemur_acme/tests/test_ultradns.py | 10 ++++++++++ 3 files changed, 31 insertions(+) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index cbb4d314..0e7ed9e3 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -3,6 +3,7 @@ from unittest.mock import patch, Mock import josepy as jose from cryptography.x509 import DNSName +from flask import Flask from lemur.plugins.lemur_acme import plugin from lemur.common.utils import generate_private_key from mock import MagicMock @@ -22,6 +23,15 @@ class TestAcme(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) def test_get_dns_challenges(self, mock_len): assert mock_len diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index 37e4968e..ca344d8d 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -1,5 +1,7 @@ import unittest from unittest.mock import patch, Mock + +from flask import Flask from lemur.plugins.lemur_acme import plugin, powerdns @@ -17,6 +19,15 @@ class TestPowerdns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.powerdns.current_app") def test_get_zones(self, mock_current_app): account_number = "1234567890" diff --git a/lemur/plugins/lemur_acme/tests/test_ultradns.py b/lemur/plugins/lemur_acme/tests/test_ultradns.py index f1d61e68..fa695a95 100644 --- a/lemur/plugins/lemur_acme/tests/test_ultradns.py +++ b/lemur/plugins/lemur_acme/tests/test_ultradns.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, Mock +from flask import Flask from lemur.plugins.lemur_acme import plugin, ultradns from requests.models import Response @@ -19,6 +20,15 @@ class TestUltradns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } + # Creates a new Flask application for a test duration. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.ultradns.requests") @patch("lemur.plugins.lemur_acme.ultradns.current_app") def test_ultradns_get_token(self, mock_current_app, mock_requests): From 003779a1126f6fff16812ed9f306573fb89c9cdf Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 3 Nov 2020 19:27:41 -0800 Subject: [PATCH 196/226] Mock fix for DeprecationWarning: callable is None --- lemur/plugins/lemur_acme/tests/test_acme.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 0e7ed9e3..29148002 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -127,22 +127,24 @@ class TestAcme(unittest.TestCase): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) + mock_dns_challenge = Mock() + response = Mock() + response.simple_verify = Mock(return_value=False) + mock_dns_challenge.response = Mock(return_value=response) + mock_authz = Mock() - mock_authz.dns_challenge.response = Mock() - mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False) - mock_authz.authz = [] + mock_authz.dns_challenge = [] + mock_authz.dns_challenge.append(mock_dns_challenge) + mock_authz.target_domain = "www.test.com" mock_authz_record = Mock() mock_authz_record.body.identifier.value = "test" + mock_authz.authz = [] mock_authz.authz.append(mock_authz_record) mock_authz.change_id = [] mock_authz.change_id.append("123") - mock_authz.dns_challenge = [] - dns_challenge = Mock() - mock_authz.dns_challenge.append(dns_challenge) - # with self.assertRaises(ValueError): - # self.acme.complete_dns_challenge(mock_acme, mock_authz) - self.assertRaises(ValueError, self.acme.complete_dns_challenge, mock_acme, mock_authz) + with self.assertRaises(ValueError): + self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch("acme.client.Client") @patch("OpenSSL.crypto", return_value="mock_cert") From ab014873d0a53a8d4fc88c578c1a4f32ac67dd51 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 3 Nov 2020 19:33:13 -0800 Subject: [PATCH 197/226] invalid escape sequence warning for not an escape char --- lemur/tests/test_dns_providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/tests/test_dns_providers.py b/lemur/tests/test_dns_providers.py index 83315be5..9b8fdb5a 100644 --- a/lemur/tests/test_dns_providers.py +++ b/lemur/tests/test_dns_providers.py @@ -13,7 +13,7 @@ class TestDNSProvider(unittest.TestCase): self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com')) self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com')) self.assertFalse(dnsutil.is_valid_domain('e/xample.com')) - self.assertFalse(dnsutil.is_valid_domain('exam\ple.com')) + self.assertFalse(dnsutil.is_valid_domain('exam\\ple.com')) self.assertFalse(dnsutil.is_valid_domain(' Date: Wed, 4 Nov 2020 10:53:27 -0800 Subject: [PATCH 198/226] Stop repeating certs when sending expiration notifications to security team email --- lemur/notifications/messaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 3928689e..3fa339d2 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -137,11 +137,11 @@ def send_expiration_notifications(exclude): # security team gets all security_email = current_app.config.get("LEMUR_SECURITY_TEAM_EMAIL") - security_data = [] for owner, notification_group in get_eligible_certificates(exclude=exclude).items(): for notification_label, certificates in notification_group.items(): notification_data = [] + security_data = [] notification = certificates[0][0] From 7d2ce61303a9b6e77f6012908706cb79f1a6c278 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 4 Nov 2020 18:03:43 -0800 Subject: [PATCH 199/226] Updating comment for application context --- lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++- lemur/plugins/lemur_acme/tests/test_powerdns.py | 3 ++- lemur/plugins/lemur_acme/tests/test_ultradns.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 29148002..4ee56396 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -23,7 +23,8 @@ class TestAcme(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } - # Creates a new Flask application for a test duration. + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. _app = Flask('lemur_test_acme') self.ctx = _app.app_context() assert self.ctx diff --git a/lemur/plugins/lemur_acme/tests/test_powerdns.py b/lemur/plugins/lemur_acme/tests/test_powerdns.py index ca344d8d..cf850970 100644 --- a/lemur/plugins/lemur_acme/tests/test_powerdns.py +++ b/lemur/plugins/lemur_acme/tests/test_powerdns.py @@ -19,7 +19,8 @@ class TestPowerdns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } - # Creates a new Flask application for a test duration. + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. _app = Flask('lemur_test_acme') self.ctx = _app.app_context() assert self.ctx diff --git a/lemur/plugins/lemur_acme/tests/test_ultradns.py b/lemur/plugins/lemur_acme/tests/test_ultradns.py index fa695a95..7616459e 100644 --- a/lemur/plugins/lemur_acme/tests/test_ultradns.py +++ b/lemur/plugins/lemur_acme/tests/test_ultradns.py @@ -20,7 +20,8 @@ class TestUltradns(unittest.TestCase): "test.fakedomain.net": [mock_dns_provider], } - # Creates a new Flask application for a test duration. + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. _app = Flask('lemur_test_acme') self.ctx = _app.app_context() assert self.ctx From 206d010c9a01e48894e22d31d83cc208bdacc83d Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 4 Nov 2020 18:23:39 -0800 Subject: [PATCH 200/226] Version updates and making lint happy --- lemur/certificates/service.py | 9 +++++---- lemur/certificates/utils.py | 2 +- lemur/notifications/messaging.py | 9 +++++---- requirements-dev.txt | 6 +++--- requirements.txt | 6 +++--- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 167425cc..ac844120 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -12,6 +12,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from flask import current_app from sqlalchemy import func, or_, not_, cast, Integer +from sqlalchemy.sql.expression import false, true from lemur import database from lemur.authorities.models import Authority @@ -150,7 +151,7 @@ def get_all_certs_attached_to_endpoint_without_autorotate(): """ return ( Certificate.query.filter(Certificate.endpoints.any()) - .filter(Certificate.rotation == False) + .filter(Certificate.rotation == false()) .filter(Certificate.not_after >= arrow.now()) .filter(not_(Certificate.replaced.any())) .all() # noqa @@ -205,9 +206,9 @@ def get_all_pending_reissue(): :return: """ return ( - Certificate.query.filter(Certificate.rotation == True) + Certificate.query.filter(Certificate.rotation == true()) .filter(not_(Certificate.replaced.any())) - .filter(Certificate.in_rotation_window == True) + .filter(Certificate.in_rotation_window == true()) .all() ) # noqa @@ -525,7 +526,7 @@ def render(args): ) if current_app.config.get("ALLOW_CERT_DELETION", False): - query = query.filter(Certificate.deleted == False) # noqa + query = query.filter(Certificate.deleted == false()) result = database.sort_and_page(query, Certificate, args) return result diff --git a/lemur/certificates/utils.py b/lemur/certificates/utils.py index e642e058..18f1c967 100644 --- a/lemur/certificates/utils.py +++ b/lemur/certificates/utils.py @@ -82,4 +82,4 @@ def get_key_type_from_csr(data): raise Exception("Unsupported key type") except NotImplemented: - raise NotImplemented() + raise NotImplementedError diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 3fa339d2..75d227b1 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -16,6 +16,7 @@ from itertools import groupby import arrow from flask import current_app from sqlalchemy import and_ +from sqlalchemy.sql.expression import false, true from lemur import database from lemur.certificates.models import Certificate @@ -40,10 +41,10 @@ def get_certificates(exclude=None): q = ( database.db.session.query(Certificate) .filter(Certificate.not_after <= max) - .filter(Certificate.notify == True) - .filter(Certificate.expired == False) - .filter(Certificate.revoked == False) - ) # noqa + .filter(Certificate.notify == true()) + .filter(Certificate.expired == false()) + .filter(Certificate.revoked == false()) + ) exclude_conditions = [] if exclude: diff --git a/requirements-dev.txt b/requirements-dev.txt index 180be728..5b4c4fe4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,7 +15,7 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv -flake8==3.5.0 # via -r requirements-dev.in +flake8==3.8.0 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests invoke==1.4.1 # via -r requirements-dev.in @@ -25,9 +25,9 @@ mccabe==0.6.1 # via flake8 nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit pkginfo==1.5.0.1 # via twine pre-commit==2.8.2 # via -r requirements-dev.in -pycodestyle==2.3.1 # via flake8 +pycodestyle==2.6.0 # via flake8 pycparser==2.20 # via cffi -pyflakes==1.6.0 # via flake8 +pyflakes==2.2.0 # via flake8 pygments==2.6.1 # via readme-renderer pyyaml==5.3.1 # via -r requirements-dev.in, pre-commit readme-renderer==25.0 # via twine diff --git a/requirements.txt b/requirements.txt index 4e641cfd..ceaafc85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,14 +15,14 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.9 # via -r requirements.in -botocore==1.19.9 # via -r requirements.in, boto3, s3transfer +boto3==1.16.10 # via -r requirements.in +botocore==1.19.10 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.6.20 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.1.1 # via flask +click==7.1.2 # black 20.8b1 has requirement click>=7.1.2 cloudflare==2.8.13 # via -r requirements.in cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in From 320667935df6fe74b09e6bf04f3d5ac3ecebc451 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 4 Nov 2020 19:09:34 -0800 Subject: [PATCH 201/226] flake8 version 3.8.4 --- requirements-dev.in | 2 +- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index 2ffc5488..53a7586c 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,6 @@ # Run `make up-reqs` to update pinned dependencies in requirement text files -flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors. +flake8==3.8.4 # flake8 latest version pre-commit invoke twine diff --git a/requirements-dev.txt b/requirements-dev.txt index 5b4c4fe4..1a9980b7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv -flake8==3.8.0 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests +flake8==3.8.4 # via -r requirements-dev.in invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine From 519411b309dd239b3adbe59988cbcf8a5141e851 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 6 Nov 2020 22:40:55 -0800 Subject: [PATCH 202/226] regex --- lemur/acme_providers/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/acme_providers/cli.py b/lemur/acme_providers/cli.py index 313876e6..7efa196e 100644 --- a/lemur/acme_providers/cli.py +++ b/lemur/acme_providers/cli.py @@ -144,7 +144,7 @@ def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name) "value": bucket_name, "type": "str", "required": True, - "validation": "[0-9a-z.-]{3,63}", + "validation": r"[0-9a-z.-]{3,63}", "helpMessage": "Must be a valid S3 bucket name!", }, { @@ -152,7 +152,7 @@ def upload_acme_token_s3(token, token_name, prefix, account_number, bucket_name) "type": "str", "value": account_number, "required": True, - "validation": "[0-9]{12}", + "validation": r"[0-9]{12}", "helpMessage": "A valid AWS account number with permission to access S3", }, { From 7c779d6283be0cfe7e2571ee3264df2989fbdee9 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 6 Nov 2020 22:41:48 -0800 Subject: [PATCH 203/226] regex --- lemur/plugins/lemur_aws/tests/test_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lemur/plugins/lemur_aws/tests/test_plugin.py b/lemur/plugins/lemur_aws/tests/test_plugin.py index a3227296..be9b14fd 100644 --- a/lemur/plugins/lemur_aws/tests/test_plugin.py +++ b/lemur/plugins/lemur_aws/tests/test_plugin.py @@ -28,7 +28,7 @@ def test_upload_acme_token(app): "value": bucket, "type": "str", "required": True, - "validation": "[0-9a-z.-]{3,63}", + "validation": r"[0-9a-z.-]{3,63}", "helpMessage": "Must be a valid S3 bucket name!", }, { @@ -36,7 +36,7 @@ def test_upload_acme_token(app): "type": "str", "value": account, "required": True, - "validation": "[0-9]{12}", + "validation": r"[0-9]{12}", "helpMessage": "A valid AWS account number with permission to access S3", }, { From 57208fe198e9ba17661bbf4c40ccec937623daeb Mon Sep 17 00:00:00 2001 From: Frederic Brin Date: Mon, 9 Nov 2020 09:40:28 +0100 Subject: [PATCH 204/226] Fix group lookup when AD DNS Referal is in lookup path Fix an issue when the DNS AD referal is in the path. An Exception is raised, with the following stacktrace:: Traceback (most recent call last): File "/www/lemur/lemur/auth/views.py", line 317, in post user = ldap_principal.authenticate() File "/www/lemur/lemur/auth/ldap.py", line 147, in authenticate self._bind() File "/www/lemur/lemur/auth/ldap.py", line 216, in _bind self.ldap_groups.append(values["cn"][0].decode("ascii")) TypeError: list indices must be integers or slices, not str This is issue is trigerred by some extra rows that referrences the DNS subtree:: ['ldaps://DomainDnsZones.xxxx'] Limiting the extraction to the expected dicts fix this issue. --- lemur/auth/ldap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lemur/auth/ldap.py b/lemur/auth/ldap.py index ed87b76c..030c7c78 100644 --- a/lemur/auth/ldap.py +++ b/lemur/auth/ldap.py @@ -210,7 +210,8 @@ class LdapPrincipal: self.ldap_groups = [] for group in lgroups: (dn, values) = group - self.ldap_groups.append(values["cn"][0].decode("ascii")) + if type(values) == dict: + self.ldap_groups.append(values["cn"][0].decode("ascii")) else: lgroups = self.ldap_client.search_s( self.ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, self.ldap_attrs From d3e8921731c51a7701344f343b2748d869eda0fb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:20:48 +0000 Subject: [PATCH 205/226] Bump pytest-flask from 1.0.0 to 1.1.0 Bumps [pytest-flask](https://github.com/pytest-dev/pytest-flask) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/pytest-dev/pytest-flask/releases) - [Changelog](https://github.com/pytest-dev/pytest-flask/blob/master/docs/changelog.rst) - [Commits](https://github.com/pytest-dev/pytest-flask/compare/1.0.0...1.1.0) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index fcc219e9..a7c746dc 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -59,7 +59,7 @@ pycparser==2.20 # via cffi pyflakes==2.2.0 # via -r requirements-tests.in pyparsing==2.4.7 # via packaging pyrsistent==0.16.0 # via jsonschema -pytest-flask==1.0.0 # via -r requirements-tests.in +pytest-flask==1.1.0 # via -r requirements-tests.in pytest-mock==3.3.1 # via -r requirements-tests.in pytest==6.1.2 # via -r requirements-tests.in, pytest-flask, pytest-mock python-dateutil==2.8.1 # via botocore, faker, freezegun, moto From a74b8aed152b1b22398621484d5a46084dff5e75 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:29:51 +0000 Subject: [PATCH 206/226] Bump faker from 4.14.0 to 4.14.2 Bumps [faker](https://github.com/joke2k/faker) from 4.14.0 to 4.14.2. - [Release notes](https://github.com/joke2k/faker/releases) - [Changelog](https://github.com/joke2k/faker/blob/master/CHANGELOG.rst) - [Commits](https://github.com/joke2k/faker/compare/v4.14.0...v4.14.2) Signed-off-by: dependabot-preview[bot] --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index a7c746dc..a7e5a479 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -24,7 +24,7 @@ decorator==4.4.2 # via networkx docker==4.2.0 # via moto ecdsa==0.14.1 # via moto, python-jose, sshpubkeys factory-boy==3.1.0 # via -r requirements-tests.in -faker==4.14.0 # via -r requirements-tests.in, factory-boy +faker==4.14.2 # via -r requirements-tests.in, factory-boy fakeredis==1.4.4 # via -r requirements-tests.in flask==1.1.2 # via pytest-flask freezegun==1.0.0 # via -r requirements-tests.in From 4c6645ca04fa43b7ac4c92a7b0fb030842e94897 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:40:24 +0000 Subject: [PATCH 207/226] Bump certifi from 2020.6.20 to 2020.11.8 Bumps [certifi](https://github.com/certifi/python-certifi) from 2020.6.20 to 2020.11.8. - [Release notes](https://github.com/certifi/python-certifi/releases) - [Commits](https://github.com/certifi/python-certifi/compare/2020.06.20...2020.11.08) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 8 ++++---- requirements-tests.txt | 2 +- requirements.txt | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1a9980b7..e2eb7051 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # appdirs==1.4.3 # via virtualenv bleach==3.1.4 # via readme-renderer -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfgv==3.1.0 # via pre-commit chardet==3.0.4 # via requests @@ -15,9 +15,9 @@ cryptography==3.2.1 # via secretstorage distlib==0.3.0 # via virtualenv docutils==0.16 # via readme-renderer filelock==3.0.12 # via virtualenv +flake8==3.8.4 # via -r requirements-dev.in identify==1.4.14 # via pre-commit idna==2.9 # via requests -flake8==3.8.4 # via -r requirements-dev.in invoke==1.4.1 # via -r requirements-dev.in jeepney==0.4.3 # via keyring, secretstorage keyring==21.2.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index d0f5a47c..f9d4cbfa 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,14 +17,14 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.9 # via -r requirements.txt -botocore==1.19.9 # via -r requirements.txt, boto3, s3transfer +boto3==1.16.10 # via -r requirements.txt +botocore==1.19.10 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt -certifi==2020.6.20 # via -r requirements.txt, requests +certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl chardet==3.0.4 # via -r requirements.txt, requests -click==7.1.1 # via -r requirements.txt, flask +click==7.1.2 # via -r requirements.txt, flask cloudflare==2.8.13 # via -r requirements.txt cryptography==3.2.1 # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index a7e5a479..a91aaa30 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -13,7 +13,7 @@ black==20.8b1 # via -r requirements-tests.in boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2020.6.20 # via requests +certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto chardet==3.0.4 # via requests diff --git a/requirements.txt b/requirements.txt index ceaafc85..3c1a8aba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,11 +18,11 @@ blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.in botocore==1.19.10 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in -certifi==2020.6.20 # via -r requirements.in, requests +certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in cffi==1.14.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==7.1.2 # black 20.8b1 has requirement click>=7.1.2 +click==7.1.2 # via flask cloudflare==2.8.13 # via -r requirements.in cryptography==3.2.1 # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests dnspython3==1.15.0 # via -r requirements.in From 7ec2860f886be6986c3f1823d883f13250d8c6dd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 21:49:57 +0000 Subject: [PATCH 208/226] Bump botocore from 1.19.9 to 1.19.14 Bumps [botocore](https://github.com/boto/botocore) from 1.19.9 to 1.19.14. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.19.9...1.19.14) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index f9d4cbfa..e9fd1566 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -18,7 +18,7 @@ beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.txt -botocore==1.19.10 # via -r requirements.txt, boto3, s3transfer +botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests certsrv==2.1.1 # via -r requirements.txt diff --git a/requirements-tests.txt b/requirements-tests.txt index a91aaa30..d055da83 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -12,7 +12,7 @@ bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in boto3==1.16.9 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.19.9 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests cffi==1.14.0 # via cryptography cfn-lint==0.29.5 # via moto diff --git a/requirements.txt b/requirements.txt index 3c1a8aba..73bc348e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven boto3==1.16.10 # via -r requirements.in -botocore==1.19.10 # via -r requirements.in, boto3, s3transfer +botocore==1.19.14 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests certsrv==2.1.1 # via -r requirements.in From 65d9ac6a0f83c1bc447a6335d64ee0c0fa6ee016 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 22:22:52 +0000 Subject: [PATCH 209/226] Bump boto3 from 1.16.9 to 1.16.14 Bumps [boto3](https://github.com/boto/boto3) from 1.16.9 to 1.16.14. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.16.9...1.16.14) Signed-off-by: dependabot-preview[bot] --- requirements-docs.txt | 2 +- requirements-tests.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index e9fd1566..1fcf06ab 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -17,7 +17,7 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare billiard==3.6.3.0 # via -r requirements.txt, celery blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven -boto3==1.16.10 # via -r requirements.txt +boto3==1.16.14 # via -r requirements.txt botocore==1.19.14 # via -r requirements.txt, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.txt certifi==2020.11.8 # via -r requirements.txt, requests diff --git a/requirements-tests.txt b/requirements-tests.txt index d055da83..b82e2ac8 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ aws-sam-translator==1.22.0 # via cfn-lint aws-xray-sdk==2.5.0 # via moto bandit==1.6.2 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in -boto3==1.16.9 # via aws-sam-translator, moto +boto3==1.16.14 # via aws-sam-translator, moto boto==2.49.0 # via moto botocore==1.19.14 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2020.11.8 # via requests diff --git a/requirements.txt b/requirements.txt index 73bc348e..d7b56f2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko beautifulsoup4==4.9.1 # via cloudflare billiard==3.6.3.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.16.10 # via -r requirements.in +boto3==1.16.14 # via -r requirements.in botocore==1.19.14 # via -r requirements.in, boto3, s3transfer celery[redis]==4.4.2 # via -r requirements.in certifi==2020.11.8 # via -r requirements.in, requests From 99ca0ac78df2ecc084d6da9d78ee02a284e4c2a3 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 10 Nov 2020 15:32:04 +0100 Subject: [PATCH 210/226] Add context fix to tests, Add regex, Flake8 --- lemur/plugins/lemur_acme/acme_handlers.py | 4 +--- lemur/plugins/lemur_acme/challenge_types.py | 4 +--- lemur/plugins/lemur_acme/plugin.py | 4 ++-- lemur/plugins/lemur_acme/tests/test_acme_handler.py | 11 +++++++++++ lemur/plugins/lemur_acme/tests/test_acme_http.py | 11 +++++++++++ 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index 40de0f77..ebc2b2b4 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -123,9 +123,7 @@ class AcmeHandler(object): current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: - pem_certificate_chain = orderr.fullchain_pem[ - len(pem_certificate): # noqa - ].lstrip() + pem_certificate_chain = orderr.fullchain_pem[len(pem_certificate):].lstrip() current_app.logger.debug( "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index f87acc41..fd779742 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -121,9 +121,7 @@ class AcmeHttpChallenge(AcmeChallenge): current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: - pem_certificate_chain = finalized_orderr.fullchain_pem[ - len(pem_certificate): # noqa - ].lstrip() + pem_certificate_chain = finalized_orderr.fullchain_pem[len(pem_certificate):].lstrip() # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate return pem_certificate, pem_certificate_chain, None diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 51f6aa3a..7dc7af25 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -309,7 +309,7 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): "name": "acme_url", "type": "str", "required": True, - "validation": "/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", + "validation": r"/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/", "helpMessage": "Must be a valid web url starting with http[s]://", }, { @@ -322,7 +322,7 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): "name": "email", "type": "str", "default": "", - "validation": "/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", + "validation": r"/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/", "helpMessage": "Email to use", }, { diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index d9f4313d..650bf513 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, Mock +from flask import Flask from cryptography.x509 import DNSName from lemur.plugins.lemur_acme import acme_handlers @@ -9,6 +10,16 @@ class TestAcmeHandler(unittest.TestCase): def setUp(self): self.acme = acme_handlers.AcmeHandler() + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + def test_strip_wildcard(self): expected = ("example.com", False) result = self.acme.strip_wildcard("example.com") diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index 365bcb5c..786e0eba 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch, Mock +from flask import Flask from acme import challenges from lemur.plugins.lemur_acme import plugin @@ -11,6 +12,16 @@ class TestAcmeHttp(unittest.TestCase): self.ACMEHttpIssuerPlugin = plugin.ACMEHttpIssuerPlugin() self.acme = plugin.AcmeHandler() + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_acme') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + @patch("lemur.plugins.lemur_acme.plugin.current_app") def test_create_authority(self, mock_current_app): mock_current_app.config = Mock() From 960b8e78e3a7cefff74fc86e44f242c1c8fffc7f Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 10 Nov 2020 16:22:25 +0100 Subject: [PATCH 211/226] Implement cleanup_acme_token for http challenge --- lemur/plugins/lemur_acme/challenge_types.py | 23 ++++-- lemur/plugins/lemur_sftp/plugin.py | 78 ++++++++++++++++++++- 2 files changed, 95 insertions(+), 6 deletions(-) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index fd779742..f7310dbf 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -100,9 +100,10 @@ class AcmeHttpChallenge(AcmeChallenge): if validation_target is None: raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') + validations = {} for challenge in chall: - response = self.deploy(challenge, acme_client, validation_target) - + response, validation = self.deploy(challenge, acme_client, validation_target) + validations[challenge.chall.path] = validation acme_client.answer_challenge(challenge, response) current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") @@ -123,6 +124,9 @@ class AcmeHttpChallenge(AcmeChallenge): else: pem_certificate_chain = finalized_orderr.fullchain_pem[len(pem_certificate):].lstrip() + for token_path, token in validations.items(): + self.cleanup(token_path, token, validation_target) + # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate return pem_certificate, pem_certificate_chain, None @@ -146,10 +150,19 @@ class AcmeHttpChallenge(AcmeChallenge): destination_plugin.upload_acme_token(challenge.chall.path, validation, destination.options) current_app.logger.info("Uploaded HTTP-01 challenge token.") - return response + return response, validation - def cleanup(self, challenge, acme_client, validation_target): - pass + def cleanup(self, token_path, token, validation_target): + destination = destination_service.get(validation_target) + + if destination is None: + current_app.logger.info( + 'Couldn\'t find the destination with name {}, won\'t cleanup the challenge'.format(validation_target)) + + destination_plugin = plugins.get(destination.plugin_name) + + destination_plugin.delete_acme_token(token_path, token, destination.options) + current_app.logger.info("Cleaned up HTTP-01 challenge token.") class AcmeDnsChallenge(AcmeChallenge): diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 7f3ccc4f..8d76b879 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -134,6 +134,81 @@ class SFTPDestinationPlugin(DestinationPlugin): self.upload_file(dst_path, files, options) + # this is called from the acme http challenge + def delete_acme_token(self, token_path, token, options, **kwargs): + dst_path = self.get_option("destinationPath", options) + + _, filename = path.split(token_path) + + # prepare files for upload + files = {filename: token} + + self.delete_file(dst_path, files, options) + + # here the file is uploaded for real, this helps to keep this class DRY + def delete_file(self, dst_path, files, options): + + host = self.get_option("host", options) + port = self.get_option("port", options) + user = self.get_option("user", options) + password = self.get_option("password", options) + ssh_priv_key = self.get_option("privateKeyPath", options) + ssh_priv_key_pass = self.get_option("privateKeyPass", options) + + # upload files + try: + current_app.logger.debug( + "Connecting to {0}@{1}:{2}".format(user, host, port) + ) + ssh = paramiko.SSHClient() + + # allow connection to the new unknown host + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # open the ssh connection + if password: + current_app.logger.debug("Using password") + ssh.connect(host, username=user, port=port, password=password) + elif ssh_priv_key: + current_app.logger.debug("Using RSA private key") + pkey = paramiko.RSAKey.from_private_key_file( + ssh_priv_key, ssh_priv_key_pass + ) + ssh.connect(host, username=user, port=port, pkey=pkey) + else: + current_app.logger.error( + "No password or private key provided. Can't proceed" + ) + raise paramiko.ssh_exception.AuthenticationException + + # open the sftp session inside the ssh connection + sftp = ssh.open_sftp() + + # upload certificate files to the sftp destination + for filename, data in files.items(): + current_app.logger.debug( + "Deleting {0} from {1}".format(filename, dst_path) + ) + try: + sftp.remove(path.join(dst_path, filename)) + except PermissionError as permerror: + if permerror.errno == 13: + current_app.logger.debug( + "Deleting {0} from {1} returned Permission Denied Error, making file writable and retrying".format( + filename, dst_path) + ) + sftp.chmod(path.join(dst_path, filename), 0o600) + sftp.remove(path.join(dst_path, filename)) + + ssh.close() + + except Exception as e: + current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + try: + ssh.close() + except BaseException: + pass + # here the file is uploaded for real, this helps to keep this class DRY def upload_file(self, dst_path, files, options): @@ -200,7 +275,8 @@ class SFTPDestinationPlugin(DestinationPlugin): try: sftp.mkdir(remote_path) except IOError as ioerror: - current_app.logger.debug("Couldn't create {0}, error message: {1}".format(remote_path, ioerror)) + current_app.logger.debug( + "Couldn't create {0}, error message: {1}".format(remote_path, ioerror)) # upload certificate files to the sftp destination for filename, data in files.items(): From 6ffe7bc5267e5cb8c9e2563014ff257da3740a44 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 10 Nov 2020 16:47:56 +0100 Subject: [PATCH 212/226] Check if challenges are already validated, and skip them if possible --- lemur/plugins/lemur_acme/challenge_types.py | 31 +++++++++++++-------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index f7310dbf..480923d5 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -12,6 +12,7 @@ import json import OpenSSL from acme import challenges +from acme.messages import STATUS_VALID from flask import current_app from lemur.authorizations import service as authorization_service @@ -81,17 +82,23 @@ class AcmeHttpChallenge(AcmeChallenge): orderr = acme_client.new_order(csr) chall = [] + validations = {} + all_pre_validated = True for authz in orderr.authorizations: # Choosing challenge. # authz.body.challenges is a set of ChallengeBody objects. for i in authz.body.challenges: # Find the supported challenge. - if isinstance(i.chall, challenges.HTTP01): - chall.append(i) + if i.status != STATUS_VALID: + if isinstance(i.chall, challenges.HTTP01): + chall.append(i) + all_pre_validated = False + else: + current_app.logger.info("{} already validated, skipping".format(authz.body.identifier.value)) - if len(chall) == 0: + if len(chall) == 0 and not all_pre_validated: raise Exception('HTTP-01 challenge was not offered by the CA server.') - else: + elif not all_pre_validated: validation_target = None for option in json.loads(issuer_options["authority"].options): if option["name"] == "tokenDestination": @@ -100,13 +107,12 @@ class AcmeHttpChallenge(AcmeChallenge): if validation_target is None: raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') - validations = {} - for challenge in chall: - response, validation = self.deploy(challenge, acme_client, validation_target) - validations[challenge.chall.path] = validation - acme_client.answer_challenge(challenge, response) + for challenge in chall: + response, validation = self.deploy(challenge, acme_client, validation_target) + validations[challenge.chall.path] = validation + acme_client.answer_challenge(challenge, response) - current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") + current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") finalized_orderr = acme_client.finalize_order(orderr, datetime.datetime.now() + datetime.timedelta(seconds=90)) @@ -124,8 +130,9 @@ class AcmeHttpChallenge(AcmeChallenge): else: pem_certificate_chain = finalized_orderr.fullchain_pem[len(pem_certificate):].lstrip() - for token_path, token in validations.items(): - self.cleanup(token_path, token, validation_target) + if len(validations) != 0: + for token_path, token in validations.items(): + self.cleanup(token_path, token, validation_target) # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate return pem_certificate, pem_certificate_chain, None From 9ebcdfc189fd494ae9dc1b81434f6026ae1e7da8 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 10 Nov 2020 17:00:47 +0100 Subject: [PATCH 213/226] Check authorization state and skip already validated challenges --- lemur/plugins/lemur_acme/challenge_types.py | 17 +++++++++-------- .../plugins/lemur_acme/tests/test_acme_http.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index 480923d5..e774a02d 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -86,15 +86,16 @@ class AcmeHttpChallenge(AcmeChallenge): all_pre_validated = True for authz in orderr.authorizations: # Choosing challenge. - # authz.body.challenges is a set of ChallengeBody objects. - for i in authz.body.challenges: - # Find the supported challenge. - if i.status != STATUS_VALID: + # check if authorizations is already in a valid state + if authz.body.status != STATUS_VALID: + all_pre_validated = False + # authz.body.challenges is a set of ChallengeBody objects. + for i in authz.body.challenges: + # Find the supported challenge. if isinstance(i.chall, challenges.HTTP01): chall.append(i) - all_pre_validated = False - else: - current_app.logger.info("{} already validated, skipping".format(authz.body.identifier.value)) + else: + current_app.logger.info("{} already validated, skipping".format(authz.body.identifier.value)) if len(chall) == 0 and not all_pre_validated: raise Exception('HTTP-01 challenge was not offered by the CA server.') @@ -114,7 +115,7 @@ class AcmeHttpChallenge(AcmeChallenge): current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") - finalized_orderr = acme_client.finalize_order(orderr, datetime.datetime.now() + datetime.timedelta(seconds=90)) + finalized_orderr = acme_client.poll_and_finalize(orderr, datetime.datetime.now() + datetime.timedelta(seconds=90)) pem_certificate = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index 786e0eba..7ed75ad2 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -67,7 +67,7 @@ class TestAcmeHttp(unittest.TestCase): mock_finalized_order = Mock() mock_finalized_order.fullchain_pem = "-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n-----BEGIN CERTIFICATE-----\nMIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2\nMDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw\nggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0\n8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym\noLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0\nZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN\nxDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56\ndhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9\nAgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw\nHQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0\nBggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu\nb3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu\nY3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq\nhkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF\nUGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9\nAFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp\nDQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7\nIkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf\nzWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI\nPTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w\nSVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em\n2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0\nWzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt\nn5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=\n-----END CERTIFICATE-----\n" - mock_client.finalize_order.return_value = mock_finalized_order + mock_client.poll_and_finalize.return_value = mock_finalized_order mock_acme.return_value = (mock_client, "") From fba1fdcc34c6634c291719f38f1ab2bad0c00d9a Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 10 Nov 2020 18:05:06 +0100 Subject: [PATCH 214/226] Improve exception handling during http challenge --- lemur/plugins/lemur_acme/challenge_types.py | 24 ++++++++++++++++----- lemur/plugins/lemur_sftp/plugin.py | 4 ++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index e774a02d..e95e10d1 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -12,7 +12,7 @@ import json import OpenSSL from acme import challenges -from acme.messages import STATUS_VALID +from acme.messages import errors, STATUS_VALID, ERROR_CODES from flask import current_app from lemur.authorizations import service as authorization_service @@ -109,13 +109,27 @@ class AcmeHttpChallenge(AcmeChallenge): raise Exception('No token_destination configured for this authority. Cant complete HTTP-01 challenge') for challenge in chall: - response, validation = self.deploy(challenge, acme_client, validation_target) - validations[challenge.chall.path] = validation - acme_client.answer_challenge(challenge, response) + try: + response, validation = self.deploy(challenge, acme_client, validation_target) + validations[challenge.chall.path] = validation + acme_client.answer_challenge(challenge, response) + except Exception as e: + current_app.logger.error(e) + raise Exception('Failure while trying to deploy token to configure destination. See logs for more information') current_app.logger.info("Uploaded HTTP-01 challenge tokens, trying to poll and finalize the order") - finalized_orderr = acme_client.poll_and_finalize(orderr, datetime.datetime.now() + datetime.timedelta(seconds=90)) + try: + finalized_orderr = acme_client.poll_and_finalize(orderr, + datetime.datetime.now() + datetime.timedelta(seconds=90)) + except errors.ValidationError as validationError: + for authz in validationError.failed_authzrs: + for chall in authz.body.challenges: + if chall.error: + current_app.logger.error( + "ValidationError occured of type {}, with message {}".format(chall.error.typ, + ERROR_CODES[chall.error.code])) + raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.') pem_certificate = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 8d76b879..f0a63b70 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -306,3 +306,7 @@ class SFTPDestinationPlugin(DestinationPlugin): ssh.close() except BaseException: pass + message = '' + for _, error in e.errors.items(): + message = error.strerror + raise Exception('Couldn\'t upload file to {}, error message: {}'.format(host, message)) From 31b5f3df86b93940d6b61bc611bccd75d266724a Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Tue, 10 Nov 2020 18:18:45 +0100 Subject: [PATCH 215/226] Remove duplicate code for revoke_certificate --- lemur/plugins/lemur_acme/acme_handlers.py | 23 +++++++++- lemur/plugins/lemur_acme/plugin.py | 43 +------------------ .../plugins/lemur_acme/tests/test_acme_dns.py | 4 +- 3 files changed, 26 insertions(+), 44 deletions(-) diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index ebc2b2b4..6d0ac5f4 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -27,7 +27,7 @@ from flask import current_app from lemur.common.utils import generate_private_key from lemur.dns_providers import service as dns_provider_service -from lemur.exceptions import InvalidAuthority, UnknownProvider +from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration from lemur.extensions import metrics, sentry from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns @@ -216,6 +216,27 @@ class AcmeHandler(object): current_app.logger.debug("Got these domains: {0}".format(domains)) return domains + def revoke_certificate(self, certificate): + if not self.reuse_account(certificate.authority): + raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") + acme_client, _ = self.acme.setup_acme_client(certificate.authority) + + fullchain_com = jose.ComparableX509( + OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, certificate.body)) + + try: + acme_client.revoke(fullchain_com, 0) # revocation reason = 0 + except (errors.ConflictError, errors.ClientError, errors.Error) as e: + # Certificate already revoked. + current_app.logger.error("Certificate revocation failed with message: " + e.detail) + metrics.send("acme_revoke_certificate_failure", "counter", 1) + return False + + current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) + metrics.send("acme_revoke_certificate_success", "counter", 1) + return True + class AcmeDnsHandler(AcmeHandler): diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 7dc7af25..4763a2fa 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -12,9 +12,6 @@ .. moduleauthor:: Curtis Castrapel """ -import OpenSSL.crypto -import josepy as jose -from acme import errors from acme.errors import PollError, WildcardUnsupportedError from acme.messages import Error as AcmeError from botocore.exceptions import ClientError @@ -272,25 +269,7 @@ class ACMEIssuerPlugin(IssuerPlugin): def revoke_certificate(self, certificate, comments): self.acme = AcmeDnsHandler() - if not self.acme.reuse_account(certificate.authority): - raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") - acme_client, _ = self.acme.setup_acme_client(certificate.authority) - - fullchain_com = jose.ComparableX509( - OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, certificate.body)) - - try: - acme_client.revoke(fullchain_com, 0) # revocation reason = 0 - except (errors.ConflictError, errors.ClientError, errors.Error) as e: - # Certificate already revoked. - current_app.logger.error("Certificate revocation failed with message: " + e.detail) - metrics.send("acme_revoke_certificate_failure", "counter", 1) - return False - - current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) - metrics.send("acme_revoke_certificate_success", "counter", 1) - return True + return self.acme.revoke_certificate(certificate) class ACMEHttpIssuerPlugin(IssuerPlugin): @@ -391,22 +370,4 @@ class ACMEHttpIssuerPlugin(IssuerPlugin): def revoke_certificate(self, certificate, comments): self.acme = AcmeHandler() - if not self.acme.reuse_account(certificate.authority): - raise InvalidConfiguration("There is no ACME account saved, unable to revoke the certificate.") - acme_client, _ = self.acme.setup_acme_client(certificate.authority) - - fullchain_com = jose.ComparableX509( - OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, certificate.body)) - - try: - acme_client.revoke(fullchain_com, 0) # revocation reason = 0 - except (errors.ConflictError, errors.ClientError, errors.Error) as e: - # Certificate already revoked. - current_app.logger.error("Certificate revocation failed with message: " + e.detail) - metrics.send("acme_revoke_certificate_failure", "counter", 1) - return False - - current_app.logger.warning("Certificate succesfully revoked: " + certificate.name) - metrics.send("acme_revoke_certificate_success", "counter", 1) - return True + return self.acme.revoke_certificate(certificate) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py index acedf977..0d0feb35 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -168,7 +168,7 @@ class TestAcmeDns(unittest.TestCase): with self.assertRaises(Exception): self.acme.setup_acme_client(mock_authority) - @patch("lemur.plugins.lemur_acme.plugin.jose.JWK.json_loads") + @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads") @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") def test_setup_acme_client_success_load_account_from_authority(self, mock_current_app, mock_acme, mock_key_json_load): @@ -190,7 +190,7 @@ class TestAcmeDns(unittest.TestCase): assert result_client assert not result_registration - @patch("lemur.plugins.lemur_acme.plugin.jose.JWKRSA.fields_to_partial_json") + @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWKRSA.fields_to_partial_json") @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") From 7a7f05ec9edcbb0abc58bda587726600667130f0 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 08:05:37 +0100 Subject: [PATCH 216/226] Fix comments in sftp delete_files --- lemur/plugins/lemur_sftp/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index f0a63b70..304271ab 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -145,7 +145,7 @@ class SFTPDestinationPlugin(DestinationPlugin): self.delete_file(dst_path, files, options) - # here the file is uploaded for real, this helps to keep this class DRY + # here the file is deleted def delete_file(self, dst_path, files, options): host = self.get_option("host", options) @@ -155,7 +155,7 @@ class SFTPDestinationPlugin(DestinationPlugin): ssh_priv_key = self.get_option("privateKeyPath", options) ssh_priv_key_pass = self.get_option("privateKeyPass", options) - # upload files + # delete files try: current_app.logger.debug( "Connecting to {0}@{1}:{2}".format(user, host, port) @@ -184,8 +184,8 @@ class SFTPDestinationPlugin(DestinationPlugin): # open the sftp session inside the ssh connection sftp = ssh.open_sftp() - # upload certificate files to the sftp destination - for filename, data in files.items(): + # delete files + for filename, _ in files.items(): current_app.logger.debug( "Deleting {0} from {1}".format(filename, dst_path) ) From 7b1beb62b64099a858d87045fcd6fb9b2994eede Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 08:05:59 +0100 Subject: [PATCH 217/226] Add directory uri, to exception message --- lemur/plugins/lemur_acme/challenge_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index e95e10d1..c530d7c5 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -98,7 +98,7 @@ class AcmeHttpChallenge(AcmeChallenge): current_app.logger.info("{} already validated, skipping".format(authz.body.identifier.value)) if len(chall) == 0 and not all_pre_validated: - raise Exception('HTTP-01 challenge was not offered by the CA server.') + raise Exception('HTTP-01 challenge was not offered by the CA server at {}'.format(orderr.uri)) elif not all_pre_validated: validation_target = None for option in json.loads(issuer_options["authority"].options): From 5cdd88e03351393bf88e662e1559f2ae8eb57dc9 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 08:34:40 +0100 Subject: [PATCH 218/226] Remove unnecessary token from delete_acme_token --- lemur/plugins/lemur_acme/challenge_types.py | 18 +++++++++--------- lemur/plugins/lemur_sftp/plugin.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index c530d7c5..36eddf8f 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -82,7 +82,7 @@ class AcmeHttpChallenge(AcmeChallenge): orderr = acme_client.new_order(csr) chall = [] - validations = {} + deployed_challenges = [] all_pre_validated = True for authz in orderr.authorizations: # Choosing challenge. @@ -110,8 +110,8 @@ class AcmeHttpChallenge(AcmeChallenge): for challenge in chall: try: - response, validation = self.deploy(challenge, acme_client, validation_target) - validations[challenge.chall.path] = validation + response = self.deploy(challenge, acme_client, validation_target) + deployed_challenges.append(challenge.chall.path) acme_client.answer_challenge(challenge, response) except Exception as e: current_app.logger.error(e) @@ -145,9 +145,9 @@ class AcmeHttpChallenge(AcmeChallenge): else: pem_certificate_chain = finalized_orderr.fullchain_pem[len(pem_certificate):].lstrip() - if len(validations) != 0: - for token_path, token in validations.items(): - self.cleanup(token_path, token, validation_target) + if len(deployed_challenges) != 0: + for token_path in deployed_challenges: + self.cleanup(token_path, validation_target) # validation is a random string, we use it as external id, to make it possible to implement revoke_certificate return pem_certificate, pem_certificate_chain, None @@ -172,9 +172,9 @@ class AcmeHttpChallenge(AcmeChallenge): destination_plugin.upload_acme_token(challenge.chall.path, validation, destination.options) current_app.logger.info("Uploaded HTTP-01 challenge token.") - return response, validation + return response - def cleanup(self, token_path, token, validation_target): + def cleanup(self, token_path, validation_target): destination = destination_service.get(validation_target) if destination is None: @@ -183,7 +183,7 @@ class AcmeHttpChallenge(AcmeChallenge): destination_plugin = plugins.get(destination.plugin_name) - destination_plugin.delete_acme_token(token_path, token, destination.options) + destination_plugin.delete_acme_token(token_path, destination.options) current_app.logger.info("Cleaned up HTTP-01 challenge token.") diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 304271ab..4e3cefd9 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -135,13 +135,13 @@ class SFTPDestinationPlugin(DestinationPlugin): self.upload_file(dst_path, files, options) # this is called from the acme http challenge - def delete_acme_token(self, token_path, token, options, **kwargs): + def delete_acme_token(self, token_path, options, **kwargs): dst_path = self.get_option("destinationPath", options) _, filename = path.split(token_path) # prepare files for upload - files = {filename: token} + files = {filename: None} self.delete_file(dst_path, files, options) From 6e5aa4e979f7579718790c82695b5455323d88d1 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 08:46:55 +0100 Subject: [PATCH 219/226] Deduplicate chain/certificate extraction --- lemur/plugins/lemur_acme/acme_handlers.py | 15 ++++++++++----- lemur/plugins/lemur_acme/challenge_types.py | 15 +-------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/lemur/plugins/lemur_acme/acme_handlers.py b/lemur/plugins/lemur_acme/acme_handlers.py index 6d0ac5f4..c1ab5281 100644 --- a/lemur/plugins/lemur_acme/acme_handlers.py +++ b/lemur/plugins/lemur_acme/acme_handlers.py @@ -111,10 +111,18 @@ class AcmeHandler(object): f"Successfully resolved Acme order: {order.uri}", exc_info=True ) + pem_certificate, pem_certificate_chain = self.extract_cert_and_chain(orderr.fullchain_pem) + + current_app.logger.debug( + "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) + ) + return pem_certificate, pem_certificate_chain + + def extract_cert_and_chain(self, fullchain_pem): pem_certificate = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, orderr.fullchain_pem + OpenSSL.crypto.FILETYPE_PEM, fullchain_pem ), ).decode() @@ -123,11 +131,8 @@ class AcmeHandler(object): current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") else: - pem_certificate_chain = orderr.fullchain_pem[len(pem_certificate):].lstrip() + pem_certificate_chain = fullchain_pem[len(pem_certificate):].lstrip() - current_app.logger.debug( - "{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)) - ) return pem_certificate, pem_certificate_chain @retry(stop_max_attempt_number=5, wait_fixed=5000) diff --git a/lemur/plugins/lemur_acme/challenge_types.py b/lemur/plugins/lemur_acme/challenge_types.py index 36eddf8f..538ec236 100644 --- a/lemur/plugins/lemur_acme/challenge_types.py +++ b/lemur/plugins/lemur_acme/challenge_types.py @@ -10,7 +10,6 @@ import datetime import json -import OpenSSL from acme import challenges from acme.messages import errors, STATUS_VALID, ERROR_CODES from flask import current_app @@ -131,19 +130,7 @@ class AcmeHttpChallenge(AcmeChallenge): ERROR_CODES[chall.error.code])) raise Exception('Validation error occured, can\'t complete challenges. See logs for more information.') - pem_certificate = OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_PEM, - OpenSSL.crypto.load_certificate( - OpenSSL.crypto.FILETYPE_PEM, finalized_orderr.fullchain_pem - ), - ).decode() - - if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \ - and datetime.datetime.now() < datetime.datetime.strptime( - current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'): - pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA") - else: - pem_certificate_chain = finalized_orderr.fullchain_pem[len(pem_certificate):].lstrip() + pem_certificate, pem_certificate_chain = self.acme.extract_cert_and_chain(finalized_orderr.fullchain_pem) if len(deployed_challenges) != 0: for token_path in deployed_challenges: From df11a03bdebbed94c8acc23098888b19302d551b Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 11:02:15 +0100 Subject: [PATCH 220/226] Implement sftp upload tests --- lemur/plugins/lemur_sftp/plugin.py | 7 ++- lemur/plugins/lemur_sftp/tests/test_sftp.py | 70 +++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 lemur/plugins/lemur_sftp/tests/test_sftp.py diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 4e3cefd9..8992d39b 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -307,6 +307,7 @@ class SFTPDestinationPlugin(DestinationPlugin): except BaseException: pass message = '' - for _, error in e.errors.items(): - message = error.strerror - raise Exception('Couldn\'t upload file to {}, error message: {}'.format(host, message)) + if e.errors: + for _, error in e.errors.items(): + message = error.strerror + raise Exception('Couldn\'t upload file to {}, error message: {}'.format(host, message)) diff --git a/lemur/plugins/lemur_sftp/tests/test_sftp.py b/lemur/plugins/lemur_sftp/tests/test_sftp.py new file mode 100644 index 00000000..d58a8984 --- /dev/null +++ b/lemur/plugins/lemur_sftp/tests/test_sftp.py @@ -0,0 +1,70 @@ +import unittest +from unittest.mock import patch, Mock, MagicMock, mock_open + +from flask import Flask +from lemur.plugins.lemur_sftp import plugin + + +class TestSftp(unittest.TestCase): + def setUp(self): + self.sftp_destination = plugin.SFTPDestinationPlugin() + # Creates a new Flask application for a test duration. In python 3.8, manual push of application context is + # needed to run tests in dev environment without getting error 'Working outside of application context'. + _app = Flask('lemur_test_sftp') + self.ctx = _app.app_context() + assert self.ctx + self.ctx.push() + + def tearDown(self): + self.ctx.pop() + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_upload_file_single_with_password(self, mock_paramiko): + dst_path = '/tmp/non-existant' + files = {'first-file': 'data'} + options = [{'name': 'host', 'value': 'non-existant'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}] + + mock_sftp = Mock() + mock_sftp.open = mock_open() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.upload_file(dst_path, files, options) + + mock_sftp.open.assert_called_once() + handle = mock_sftp.open() + handle.write.assert_called_once_with('data') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existant', username='test_acme', port='22', + password='test_password') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_upload_file_multiple_with_key(self, mock_paramiko): + dst_path = '/tmp/non-existant' + files = {'first-file': 'data', 'second-file': 'data2'} + options = [{'name': 'host', 'value': 'non-existant'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'privateKeyPath', 'value': '/tmp/id_rsa'}, + {'name': 'privateKeyPass', 'value': 'ssh-key-password'}] + + mock_sftp = Mock() + mock_sftp.open = mock_open() + + mock_paramiko.RSAKey.from_private_key_file.return_value = 'ssh-rsa test-key' + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.upload_file(dst_path, files, options) + + mock_sftp.open.assert_called() + handle = mock_sftp.open() + handle.write.assert_called_with('data2') + mock_ssh.close.assert_called_once() + + mock_paramiko.RSAKey.from_private_key_file.assert_called_with('/tmp/id_rsa', 'ssh-key-password') + mock_ssh.connect.assert_called_with('non-existant', username='test_acme', port='22', + pkey='ssh-rsa test-key') From ae7a044b9c38e1763e0e425f968ee5c486075fb6 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 11:13:09 +0100 Subject: [PATCH 221/226] Add test for upload_acme_token --- lemur/plugins/lemur_sftp/tests/test_sftp.py | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lemur/plugins/lemur_sftp/tests/test_sftp.py b/lemur/plugins/lemur_sftp/tests/test_sftp.py index d58a8984..09ad4e80 100644 --- a/lemur/plugins/lemur_sftp/tests/test_sftp.py +++ b/lemur/plugins/lemur_sftp/tests/test_sftp.py @@ -68,3 +68,27 @@ class TestSftp(unittest.TestCase): mock_paramiko.RSAKey.from_private_key_file.assert_called_with('/tmp/id_rsa', 'ssh-key-password') mock_ssh.connect.assert_called_with('non-existant', username='test_acme', port='22', pkey='ssh-rsa test-key') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_upload_acme_token(self, mock_paramiko): + token_path = './well-known/acme-challenge/some-token-path' + token = 'token-data' + options = [{'name': 'host', 'value': 'non-existant'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}, + {'name': 'destinationPath', 'value': '/tmp/destination-path'}] + + mock_sftp = Mock() + mock_sftp.open = mock_open() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.upload_acme_token(token_path, token, options) + + mock_sftp.open.assert_called_once() + handle = mock_sftp.open() + handle.write.assert_called_once_with('token-data') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existant', username='test_acme', port='22', + password='test_password') \ No newline at end of file From e12ee1d89cc673753942e4b3af7fd5a04e1dd65a Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 11:23:24 +0100 Subject: [PATCH 222/226] Implement delete file and delete token tests --- lemur/plugins/lemur_sftp/tests/test_sftp.py | 68 ++++++++++++++++----- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/lemur/plugins/lemur_sftp/tests/test_sftp.py b/lemur/plugins/lemur_sftp/tests/test_sftp.py index 09ad4e80..52fa6a14 100644 --- a/lemur/plugins/lemur_sftp/tests/test_sftp.py +++ b/lemur/plugins/lemur_sftp/tests/test_sftp.py @@ -20,9 +20,9 @@ class TestSftp(unittest.TestCase): @patch("lemur.plugins.lemur_sftp.plugin.paramiko") def test_upload_file_single_with_password(self, mock_paramiko): - dst_path = '/tmp/non-existant' + dst_path = '/var/non-existent' files = {'first-file': 'data'} - options = [{'name': 'host', 'value': 'non-existant'}, {'name': 'port', 'value': '22'}, + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}] mock_sftp = Mock() @@ -34,19 +34,19 @@ class TestSftp(unittest.TestCase): self.sftp_destination.upload_file(dst_path, files, options) - mock_sftp.open.assert_called_once() + mock_sftp.open.assert_called_once_with('/var/non-existent/first-file', 'w') handle = mock_sftp.open() handle.write.assert_called_once_with('data') mock_ssh.close.assert_called_once() - mock_ssh.connect.assert_called_with('non-existant', username='test_acme', port='22', + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', password='test_password') @patch("lemur.plugins.lemur_sftp.plugin.paramiko") def test_upload_file_multiple_with_key(self, mock_paramiko): - dst_path = '/tmp/non-existant' + dst_path = '/var/non-existent' files = {'first-file': 'data', 'second-file': 'data2'} - options = [{'name': 'host', 'value': 'non-existant'}, {'name': 'port', 'value': '22'}, - {'name': 'user', 'value': 'test_acme'}, {'name': 'privateKeyPath', 'value': '/tmp/id_rsa'}, + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'privateKeyPath', 'value': '/var/id_rsa'}, {'name': 'privateKeyPass', 'value': 'ssh-key-password'}] mock_sftp = Mock() @@ -60,22 +60,22 @@ class TestSftp(unittest.TestCase): self.sftp_destination.upload_file(dst_path, files, options) - mock_sftp.open.assert_called() + mock_sftp.open.assert_called_with('/var/non-existent/second-file', 'w') handle = mock_sftp.open() handle.write.assert_called_with('data2') mock_ssh.close.assert_called_once() - mock_paramiko.RSAKey.from_private_key_file.assert_called_with('/tmp/id_rsa', 'ssh-key-password') - mock_ssh.connect.assert_called_with('non-existant', username='test_acme', port='22', + mock_paramiko.RSAKey.from_private_key_file.assert_called_with('/var/id_rsa', 'ssh-key-password') + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', pkey='ssh-rsa test-key') @patch("lemur.plugins.lemur_sftp.plugin.paramiko") def test_upload_acme_token(self, mock_paramiko): token_path = './well-known/acme-challenge/some-token-path' token = 'token-data' - options = [{'name': 'host', 'value': 'non-existant'}, {'name': 'port', 'value': '22'}, + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}, - {'name': 'destinationPath', 'value': '/tmp/destination-path'}] + {'name': 'destinationPath', 'value': '/var/destination-path'}] mock_sftp = Mock() mock_sftp.open = mock_open() @@ -86,9 +86,49 @@ class TestSftp(unittest.TestCase): self.sftp_destination.upload_acme_token(token_path, token, options) - mock_sftp.open.assert_called_once() + mock_sftp.open.assert_called_once_with('/var/destination-path/some-token-path', 'w') handle = mock_sftp.open() handle.write.assert_called_once_with('token-data') mock_ssh.close.assert_called_once() - mock_ssh.connect.assert_called_with('non-existant', username='test_acme', port='22', + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', + password='test_password') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_delete_file_with_password(self, mock_paramiko): + dst_path = '/var/non-existent' + files = {'first-file': None} + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}] + + mock_sftp = Mock() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.delete_file(dst_path, files, options) + + mock_sftp.remove.assert_called_once_with('/var/non-existent/first-file') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', + password='test_password') + + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") + def test_delete_acme_token(self, mock_paramiko): + token_path = './well-known/acme-challenge/some-token-path' + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}, {'name': 'password', 'value': 'test_password'}, + {'name': 'destinationPath', 'value': '/var/destination-path'}] + + mock_sftp = Mock() + + mock_ssh = mock_paramiko.SSHClient.return_value + mock_ssh.connect = MagicMock() + mock_ssh.open_sftp.return_value = mock_sftp + + self.sftp_destination.delete_acme_token(token_path, options) + + mock_sftp.remove.assert_called_once_with('/var/destination-path/some-token-path') + mock_ssh.close.assert_called_once() + mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', password='test_password') \ No newline at end of file From 648565d3e91bd9ed572012b552be1440ae989034 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 11:45:57 +0100 Subject: [PATCH 223/226] Improve exception handling in lemur_sftp, Add Authentication failure test --- lemur/plugins/lemur_sftp/plugin.py | 10 +++++++--- lemur/plugins/lemur_sftp/tests/test_sftp.py | 12 +++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 8992d39b..8698bdd9 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -19,6 +19,7 @@ from os import path import paramiko +from paramiko.ssh_exception import AuthenticationException from flask import current_app from lemur.plugins import lemur_sftp @@ -179,7 +180,7 @@ class SFTPDestinationPlugin(DestinationPlugin): current_app.logger.error( "No password or private key provided. Can't proceed" ) - raise paramiko.ssh_exception.AuthenticationException + raise AuthenticationException # open the sftp session inside the ssh connection sftp = ssh.open_sftp() @@ -243,7 +244,7 @@ class SFTPDestinationPlugin(DestinationPlugin): current_app.logger.error( "No password or private key provided. Can't proceed" ) - raise paramiko.ssh_exception.AuthenticationException + raise AuthenticationException # split the path into it's segments, so we can create it recursively allparts = [] @@ -300,6 +301,9 @@ class SFTPDestinationPlugin(DestinationPlugin): ssh.close() + except AuthenticationException as e: + current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + raise AuthenticationException("Couldn't connect to {0}, due to an Authentication exception.") except Exception as e: current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) try: @@ -307,7 +311,7 @@ class SFTPDestinationPlugin(DestinationPlugin): except BaseException: pass message = '' - if e.errors: + if hasattr(e, 'errors'): for _, error in e.errors.items(): message = error.strerror raise Exception('Couldn\'t upload file to {}, error message: {}'.format(host, message)) diff --git a/lemur/plugins/lemur_sftp/tests/test_sftp.py b/lemur/plugins/lemur_sftp/tests/test_sftp.py index 52fa6a14..e30a1ac9 100644 --- a/lemur/plugins/lemur_sftp/tests/test_sftp.py +++ b/lemur/plugins/lemur_sftp/tests/test_sftp.py @@ -3,6 +3,7 @@ from unittest.mock import patch, Mock, MagicMock, mock_open from flask import Flask from lemur.plugins.lemur_sftp import plugin +from paramiko.ssh_exception import AuthenticationException class TestSftp(unittest.TestCase): @@ -18,6 +19,15 @@ class TestSftp(unittest.TestCase): def tearDown(self): self.ctx.pop() + def test_failing_ssh_connection(self): + dst_path = '/var/non-existent' + files = {'first-file': 'data'} + options = [{'name': 'host', 'value': 'non-existent'}, {'name': 'port', 'value': '22'}, + {'name': 'user', 'value': 'test_acme'}] + + with self.assertRaises(AuthenticationException): + self.sftp_destination.upload_file(dst_path, files, options) + @patch("lemur.plugins.lemur_sftp.plugin.paramiko") def test_upload_file_single_with_password(self, mock_paramiko): dst_path = '/var/non-existent' @@ -131,4 +141,4 @@ class TestSftp(unittest.TestCase): mock_sftp.remove.assert_called_once_with('/var/destination-path/some-token-path') mock_ssh.close.assert_called_once() mock_ssh.connect.assert_called_with('non-existent', username='test_acme', port='22', - password='test_password') \ No newline at end of file + password='test_password') From 2b01bdb471dd40390984f54df66895b4b08397de Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 11:58:36 +0100 Subject: [PATCH 224/226] Refactor sftp plugin, to avoid duplicate code --- lemur/plugins/lemur_sftp/plugin.py | 129 ++++++++++++----------------- 1 file changed, 55 insertions(+), 74 deletions(-) diff --git a/lemur/plugins/lemur_sftp/plugin.py b/lemur/plugins/lemur_sftp/plugin.py index 8698bdd9..75d49e2b 100644 --- a/lemur/plugins/lemur_sftp/plugin.py +++ b/lemur/plugins/lemur_sftp/plugin.py @@ -19,7 +19,7 @@ from os import path import paramiko -from paramiko.ssh_exception import AuthenticationException +from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError from flask import current_app from lemur.plugins import lemur_sftp @@ -97,6 +97,50 @@ class SFTPDestinationPlugin(DestinationPlugin): }, ] + def open_sftp_connection(self, options): + host = self.get_option("host", options) + port = self.get_option("port", options) + user = self.get_option("user", options) + password = self.get_option("password", options) + ssh_priv_key = self.get_option("privateKeyPath", options) + ssh_priv_key_pass = self.get_option("privateKeyPass", options) + + # delete files + try: + current_app.logger.debug( + "Connecting to {0}@{1}:{2}".format(user, host, port) + ) + ssh = paramiko.SSHClient() + + # allow connection to the new unknown host + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # open the ssh connection + if password: + current_app.logger.debug("Using password") + ssh.connect(host, username=user, port=port, password=password) + elif ssh_priv_key: + current_app.logger.debug("Using RSA private key") + pkey = paramiko.RSAKey.from_private_key_file( + ssh_priv_key, ssh_priv_key_pass + ) + ssh.connect(host, username=user, port=port, pkey=pkey) + else: + current_app.logger.error( + "No password or private key provided. Can't proceed" + ) + raise AuthenticationException + + # open the sftp session inside the ssh connection + return ssh.open_sftp(), ssh + + except AuthenticationException as e: + current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + raise AuthenticationException("Couldn't connect to {0}, due to an Authentication exception.") + except NoValidConnectionsError as e: + current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) + raise NoValidConnectionsError("Couldn't connect to {0}, possible timeout or invalid hostname") + # this is called when using this as a default destination plugin def upload(self, name, body, private_key, cert_chain, options, **kwargs): @@ -149,41 +193,9 @@ class SFTPDestinationPlugin(DestinationPlugin): # here the file is deleted def delete_file(self, dst_path, files, options): - host = self.get_option("host", options) - port = self.get_option("port", options) - user = self.get_option("user", options) - password = self.get_option("password", options) - ssh_priv_key = self.get_option("privateKeyPath", options) - ssh_priv_key_pass = self.get_option("privateKeyPass", options) - - # delete files try: - current_app.logger.debug( - "Connecting to {0}@{1}:{2}".format(user, host, port) - ) - ssh = paramiko.SSHClient() - - # allow connection to the new unknown host - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # open the ssh connection - if password: - current_app.logger.debug("Using password") - ssh.connect(host, username=user, port=port, password=password) - elif ssh_priv_key: - current_app.logger.debug("Using RSA private key") - pkey = paramiko.RSAKey.from_private_key_file( - ssh_priv_key, ssh_priv_key_pass - ) - ssh.connect(host, username=user, port=port, pkey=pkey) - else: - current_app.logger.error( - "No password or private key provided. Can't proceed" - ) - raise AuthenticationException - - # open the sftp session inside the ssh connection - sftp = ssh.open_sftp() + # open the ssh and sftp sessions + sftp, ssh = self.open_sftp_connection(options) # delete files for filename, _ in files.items(): @@ -202,7 +214,8 @@ class SFTPDestinationPlugin(DestinationPlugin): sftp.remove(path.join(dst_path, filename)) ssh.close() - + except (AuthenticationException, NoValidConnectionsError) as e: + raise e except Exception as e: current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) try: @@ -213,38 +226,9 @@ class SFTPDestinationPlugin(DestinationPlugin): # here the file is uploaded for real, this helps to keep this class DRY def upload_file(self, dst_path, files, options): - host = self.get_option("host", options) - port = self.get_option("port", options) - user = self.get_option("user", options) - password = self.get_option("password", options) - ssh_priv_key = self.get_option("privateKeyPath", options) - ssh_priv_key_pass = self.get_option("privateKeyPass", options) - - # upload files try: - current_app.logger.debug( - "Connecting to {0}@{1}:{2}".format(user, host, port) - ) - ssh = paramiko.SSHClient() - - # allow connection to the new unknown host - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # open the ssh connection - if password: - current_app.logger.debug("Using password") - ssh.connect(host, username=user, port=port, password=password) - elif ssh_priv_key: - current_app.logger.debug("Using RSA private key") - pkey = paramiko.RSAKey.from_private_key_file( - ssh_priv_key, ssh_priv_key_pass - ) - ssh.connect(host, username=user, port=port, pkey=pkey) - else: - current_app.logger.error( - "No password or private key provided. Can't proceed" - ) - raise AuthenticationException + # open the ssh and sftp sessions + sftp, ssh = self.open_sftp_connection(options) # split the path into it's segments, so we can create it recursively allparts = [] @@ -261,9 +245,6 @@ class SFTPDestinationPlugin(DestinationPlugin): path_copy = parts[0] allparts.insert(0, parts[1]) - # open the sftp session inside the ssh connection - sftp = ssh.open_sftp() - # make sure that the destination path exists, recursively remote_path = allparts[0] for part in allparts: @@ -301,9 +282,8 @@ class SFTPDestinationPlugin(DestinationPlugin): ssh.close() - except AuthenticationException as e: - current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) - raise AuthenticationException("Couldn't connect to {0}, due to an Authentication exception.") + except (AuthenticationException, NoValidConnectionsError) as e: + raise e except Exception as e: current_app.logger.error("ERROR in {0}: {1}".format(e.__class__, e)) try: @@ -314,4 +294,5 @@ class SFTPDestinationPlugin(DestinationPlugin): if hasattr(e, 'errors'): for _, error in e.errors.items(): message = error.strerror - raise Exception('Couldn\'t upload file to {}, error message: {}'.format(host, message)) + raise Exception( + 'Couldn\'t upload file to {}, error message: {}'.format(self.get_option("host", options), message)) From 453826c59c6bb70ec8649c1477825667005aae25 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 12:11:13 +0100 Subject: [PATCH 225/226] Get rid of unnecessary current_app patches --- .../plugins/lemur_acme/tests/test_acme_dns.py | 63 ++++--------------- .../lemur_acme/tests/test_acme_handler.py | 13 ++-- .../lemur_acme/tests/test_acme_http.py | 12 +--- 3 files changed, 17 insertions(+), 71 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py index 0d0feb35..b6a5550f 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -52,21 +52,18 @@ class TestAcmeDns(unittest.TestCase): self.assertEqual(result, mock_entry) @patch("acme.client.Client") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1) @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") def test_start_dns_challenge( - self, mock_get_dns_challenges, mock_len, mock_app, mock_acme + self, mock_get_dns_challenges, mock_len, mock_acme ): assert mock_len mock_order = Mock() - mock_app.logger.debug = Mock() mock_authz = Mock() mock_authz.body.resolved_combinations = [] mock_entry = MagicMock() from acme import challenges - c = challenges.DNS01() mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail mock_authz.body.resolved_combinations.append(mock_entry) mock_acme.request_domain_challenges = Mock(return_value=mock_authz) @@ -83,11 +80,10 @@ class TestAcmeDns(unittest.TestCase): self.assertEqual(type(result), AuthorizationRecord) @patch("acme.client.Client") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") @patch("time.sleep") def test_complete_dns_challenge_success( - self, mock_sleep, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_sleep, mock_wait_for_dns_change, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -108,10 +104,9 @@ class TestAcmeDns(unittest.TestCase): self.acme.complete_dns_challenge(mock_acme, mock_authz) @patch("acme.client.Client") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") @patch("lemur.plugins.lemur_acme.cloudflare.wait_for_dns_change") def test_complete_dns_challenge_fail( - self, mock_wait_for_dns_change, mock_current_app, mock_acme + self, mock_wait_for_dns_change, mock_acme ): mock_dns_provider = Mock() mock_dns_provider.wait_for_dns_change = Mock(return_value=True) @@ -139,10 +134,8 @@ class TestAcmeDns(unittest.TestCase): @patch("OpenSSL.crypto", return_value="mock_cert") @patch("josepy.util.ComparableX509") @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_dns_challenges") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") def test_request_certificate( self, - mock_current_app, mock_get_dns_challenges, mock_jose, mock_crypto, @@ -159,7 +152,6 @@ class TestAcmeDns(unittest.TestCase): mock_acme.fetch_chain = Mock(return_value="mock_chain") mock_crypto.dump_certificate = Mock(return_value=b"chain") mock_order = Mock() - mock_current_app.config = {} self.acme.request_certificate(mock_acme, [], mock_order) def test_setup_acme_client_fail(self): @@ -170,8 +162,7 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads") @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_setup_acme_client_success_load_account_from_authority(self, mock_current_app, mock_acme, mock_key_json_load): + def test_setup_acme_client_success_load_account_from_authority(self, mock_acme, mock_key_json_load): mock_authority = Mock() mock_authority.id = 2 mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ @@ -180,7 +171,6 @@ class TestAcmeDns(unittest.TestCase): '{"name": "acme_regr", "value": "{\\"body\\": {}, \\"uri\\": \\"http://test.com\\"}"}]' mock_client = Mock() mock_acme.return_value = mock_client - mock_current_app.config = {} mock_key_json_load.return_value = jose.JWKRSA(key=generate_private_key("RSA2048")) @@ -193,8 +183,7 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWKRSA.fields_to_partial_json") @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_setup_acme_client_success_store_new_account(self, mock_current_app, mock_acme, mock_authorities_service, + def test_setup_acme_client_success_store_new_account(self, mock_acme, mock_authorities_service, mock_key_generation): mock_authority = Mock() mock_authority.id = 2 @@ -207,7 +196,6 @@ class TestAcmeDns(unittest.TestCase): mock_client.agree_to_tos = Mock(return_value=True) mock_client.new_account_and_tos.return_value = mock_registration mock_acme.return_value = mock_client - mock_current_app.config = {} mock_key_generation.return_value = {"n": "PwIOkViO"} @@ -222,8 +210,7 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_setup_acme_client_success(self, mock_current_app, mock_acme, mock_authorities_service): + def test_setup_acme_client_success(self, mock_acme, mock_authorities_service): mock_authority = Mock() mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ '{"name": "store_account", "value": false}]' @@ -233,20 +220,17 @@ class TestAcmeDns(unittest.TestCase): mock_client.register = mock_registration mock_client.agree_to_tos = Mock(return_value=True) mock_acme.return_value = mock_client - mock_current_app.config = {} result_client, result_registration = self.acme.setup_acme_client(mock_authority) mock_authorities_service.update_options.assert_not_called() assert result_client assert result_registration - @patch('lemur.plugins.lemur_acme.acme_handlers.current_app') - def test_get_domains_single(self, mock_current_app): + def test_get_domains_single(self): options = {"common_name": "test.netflix.net"} result = self.acme.get_domains(options) self.assertEqual(result, [options["common_name"]]) - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_get_domains_multiple(self, mock_current_app): + def test_get_domains_multiple(self): options = { "common_name": "test.netflix.net", "extensions": { @@ -258,8 +242,7 @@ class TestAcmeDns(unittest.TestCase): result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] ) - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_get_domains_san(self, mock_current_app): + def test_get_domains_san(self): options = { "common_name": "test.netflix.net", "extensions": { @@ -271,9 +254,7 @@ class TestAcmeDns(unittest.TestCase): result, [options["common_name"], "test2.netflix.net"] ) - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_create_authority(self, mock_current_app): - mock_current_app.config = Mock() + def test_create_authority(self): options = { "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} } @@ -282,17 +263,8 @@ class TestAcmeDns(unittest.TestCase): self.assertEqual(b, "") self.assertEqual(role, [{"username": "", "password": "", "name": "acme"}]) - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.dyn.current_app") - @patch("lemur.plugins.lemur_acme.cloudflare.current_app") @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") - def test_get_dns_provider( - self, - mock_dns_provider_service, - mock_current_app_cloudflare, - mock_current_app_dyn, - mock_current_app, - ): + def test_get_dns_provider(self, mock_dns_provider_service): provider = plugin.AcmeDnsHandler() route53 = provider.get_dns_provider("route53") assert route53 @@ -303,9 +275,6 @@ class TestAcmeDns(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") - @patch("lemur.plugins.lemur_acme.plugin.current_app") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - @patch("lemur.plugins.lemur_acme.challenge_types.current_app") @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.get_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.finalize_authorizations") @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.request_certificate") @@ -316,9 +285,6 @@ class TestAcmeDns(unittest.TestCase): mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations, - mock_current_app_c, - mock_current_app_a, - mock_current_app_p, mock_dns_provider_service, mock_acme, ): @@ -344,8 +310,7 @@ class TestAcmeDns(unittest.TestCase): assert result @patch("lemur.plugins.lemur_acme.plugin.AcmeDnsHandler.start_dns_challenge", return_value="test") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app", return_value=False) - def test_get_authorizations(self, mock_current_app, mock_start_dns_challenge): + def test_get_authorizations(self, mock_start_dns_challenge): mock_order = Mock() mock_order.body.identifiers = [] mock_domain = Mock() @@ -378,7 +343,6 @@ class TestAcmeDns(unittest.TestCase): self.assertEqual(result, mock_authz) @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @@ -393,7 +357,6 @@ class TestAcmeDns(unittest.TestCase): mock_dns_provider_service_p, mock_dns_provider_service, mock_authorization_service, - mock_current_app, mock_acme, ): mock_client = Mock() @@ -411,7 +374,6 @@ class TestAcmeDns(unittest.TestCase): ) @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") @patch("lemur.plugins.lemur_acme.acme_handlers.dns_provider_service") @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service") @@ -426,7 +388,6 @@ class TestAcmeDns(unittest.TestCase): mock_dns_provider_service, mock_dns_provider_service_p, mock_authorization_service, - mock_current_app, mock_acme, ): mock_client = Mock() diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 650bf513..0ff8c2ff 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -69,8 +69,7 @@ class TestAcmeHandler(unittest.TestCase): @patch("lemur.plugins.lemur_acme.acme_handlers.authorities_service") @patch("lemur.plugins.lemur_acme.acme_handlers.BackwardsCompatibleClientV2") - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_setup_acme_client_success(self, mock_current_app, mock_acme, mock_authorities_service): + def test_setup_acme_client_success(self, mock_acme, mock_authorities_service): mock_authority = Mock() mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}, ' \ '{"name": "store_account", "value": false}]' @@ -80,20 +79,17 @@ class TestAcmeHandler(unittest.TestCase): mock_client.register = mock_registration mock_client.agree_to_tos = Mock(return_value=True) mock_acme.return_value = mock_client - mock_current_app.config = {} result_client, result_registration = self.acme.setup_acme_client(mock_authority) mock_authorities_service.update_options.assert_not_called() assert result_client assert result_registration - @patch('lemur.plugins.lemur_acme.acme_handlers.current_app') - def test_get_domains_single(self, mock_current_app): + def test_get_domains_single(self): options = {"common_name": "test.netflix.net"} result = self.acme.get_domains(options) self.assertEqual(result, [options["common_name"]]) - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_get_domains_multiple(self, mock_current_app): + def test_get_domains_multiple(self): options = { "common_name": "test.netflix.net", "extensions": { @@ -105,8 +101,7 @@ class TestAcmeHandler(unittest.TestCase): result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"] ) - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_get_domains_san(self, mock_current_app): + def test_get_domains_san(self): options = { "common_name": "test.netflix.net", "extensions": { diff --git a/lemur/plugins/lemur_acme/tests/test_acme_http.py b/lemur/plugins/lemur_acme/tests/test_acme_http.py index 7ed75ad2..5a546165 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_http.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_http.py @@ -22,9 +22,7 @@ class TestAcmeHttp(unittest.TestCase): def tearDown(self): self.ctx.pop() - @patch("lemur.plugins.lemur_acme.plugin.current_app") - def test_create_authority(self, mock_current_app): - mock_current_app.config = Mock() + def test_create_authority(self): options = { "plugin": {"plugin_options": [{"name": "certificate", "value": "123"}]} } @@ -36,14 +34,12 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") - @patch("lemur.plugins.lemur_acme.challenge_types.current_app") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate( self, mock_authorization_service, mock_request_certificate, - mock_current_app, mock_destination_service, mock_plugin_manager_get, mock_acme, @@ -52,8 +48,6 @@ class TestAcmeHttp(unittest.TestCase): mock_authority = Mock() mock_authority.options = '[{"name": "tokenDestination", "value": "mock-sftp-destination"}]' - mock_current_app.config = {} - mock_order_resource = Mock() mock_order_resource.authorizations = [Mock()] mock_order_resource.authorizations[0].body.challenges = [Mock()] @@ -96,14 +90,12 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate_missing_destination_token( self, mock_authorization_service, mock_request_certificate, - mock_current_app, mock_destination_service, mock_plugin_manager_get, mock_acme, @@ -144,14 +136,12 @@ class TestAcmeHttp(unittest.TestCase): @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.setup_acme_client") @patch("lemur.plugins.base.manager.PluginManager.get") @patch("lemur.plugins.lemur_acme.challenge_types.destination_service") - @patch("lemur.plugins.lemur_acme.plugin.current_app") @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.request_certificate") @patch("lemur.plugins.lemur_acme.plugin.authorization_service") def test_create_certificate_missing_http_challenge( self, mock_authorization_service, mock_request_certificate, - mock_current_app, mock_destination_service, mock_plugin_manager_get, mock_acme, From 9fd3440cf6b18ae9adff519babffeadebd5d0274 Mon Sep 17 00:00:00 2001 From: Mathias Petermann Date: Wed, 11 Nov 2020 12:21:06 +0100 Subject: [PATCH 226/226] Cleanup tests --- lemur/plugins/lemur_acme/tests/test_acme_dns.py | 1 - lemur/plugins/lemur_acme/tests/test_acme_handler.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_dns.py b/lemur/plugins/lemur_acme/tests/test_acme_dns.py index b6a5550f..f0e7dbfa 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_dns.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_dns.py @@ -62,7 +62,6 @@ class TestAcmeDns(unittest.TestCase): mock_authz = Mock() mock_authz.body.resolved_combinations = [] mock_entry = MagicMock() - from acme import challenges mock_entry.chall = TestAcmeDns.test_complete_dns_challenge_fail mock_authz.body.resolved_combinations.append(mock_entry) diff --git a/lemur/plugins/lemur_acme/tests/test_acme_handler.py b/lemur/plugins/lemur_acme/tests/test_acme_handler.py index 0ff8c2ff..cfc18c83 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme_handler.py +++ b/lemur/plugins/lemur_acme/tests/test_acme_handler.py @@ -59,11 +59,9 @@ class TestAcmeHandler(unittest.TestCase): self.assertTrue(self.acme.reuse_account(mock_authority)) - @patch("lemur.plugins.lemur_acme.acme_handlers.current_app") - def test_reuse_account_no_configuration(self, mock_current_app): + def test_reuse_account_no_configuration(self): mock_authority = Mock() mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]' - mock_current_app.config = {} self.assertFalse(self.acme.reuse_account(mock_authority))