From 8de9842092d073852907a0242503e941e2d7c0e3 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:22:45 -0700 Subject: [PATCH 1/7] Backfill the key_type column: DB Upgrade --- lemur/common/utils.py | 17 ++++ lemur/migrations/versions/c301c59688d2_.py | 108 +++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 lemur/migrations/versions/c301c59688d2_.py diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 01cc64ae..283d1eec 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -71,6 +71,23 @@ def parse_private_key(private_key): ) +def get_key_type_from_certificate(body): + """ + + Helper function to determine key type by pasrding given PEM certificate + + :param body: PEM string + :return: Key type string + """ + parsed_cert = parse_certificate(body) + if isinstance(parsed_cert.public_key(), rsa.RSAPublicKey): + return "RSA{key_size}".format( + key_size=parsed_cert.public_key().key_size + ) + elif isinstance(parsed_cert.public_key(), ec.EllipticCurvePublicKey): + return get_key_type_from_ec_curve(parsed_cert.public_key().curve.name) + + def split_pem(data): """ Split a string of several PEM payloads to a list of strings. diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py new file mode 100644 index 00000000..6bd94cfb --- /dev/null +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -0,0 +1,108 @@ +""" + +This upgrade of database updates the key_type information for certificates +that are either still valid or have expired in last 30 days. For RSA keys, +the algorithm is determined based on the key length. For rest of the keys, +the certificate body is parsed to determine the exact key type information. + +Each individual change is explicitly committed. The logs are added to file +named upgrade_logs in current working directory. If faced any issue while +running this upgrade, there is no harm in re-running the upgrade. Each run +processes only the keys for which key type information is not yet determined. +A successful end to end run will end up updating the Alembic Version to new +Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported +by Lemur. This could be a long running job depending upon the number of +keys it may process. + +Revision ID: c301c59688d2 +Revises: 434c29e40511 +Create Date: 2020-09-21 14:28:50.757998 + +""" + +# revision identifiers, used by Alembic. +revision = 'c301c59688d2' +down_revision = '434c29e40511' + +from alembic import op +from sqlalchemy.sql import text +from lemur.common import utils +import time + +log_file = open('upgrade_logs', 'a') + + +def upgrade(): + log_file.write("\n*** Starting new run ***\n") + start_time = time.time() + + # Update RSA keys using the key length information + update_key_type_rsa(1024) + update_key_type_rsa(2048) + update_key_type_rsa(4096) + + # Process remaining certificates. Though below method does not make any assumptions, most of the remaining ones should be ECC certs. + update_key_type() + + log_file.write("--- Total %s seconds ---\n" % (time.time() - start_time)) + log_file.close() + + +def downgrade(): + # Change key type column back to null + # Going back 32 days instead of 31 to make sure no certificates are skipped + stmt = text( + "update certificates set key_type=null where not_after > CURRENT_DATE - 32" + ) + op.execute(stmt) + + +""" + Helper methods performing updates for RSA and rest of the keys +""" + + +def update_key_type_rsa(bits): + log_file.write("Processing certificate with key type RSA %s\n" % bits) + + stmt = text( + "update certificates set key_type='RSA{0}' where bits={0} and not_after > CURRENT_DATE - 31 and key_type is null".format(bits) + ) + log_file.write("Query: %s\n" % stmt) + + start_time = time.time() + op.execute(stmt) + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def update_key_type(): + conn = op.get_bind() + start_time = time.time() + + # Loop through all certificates are valid today or expired in last 30 days + for cert_id, body in conn.execute( + text( + "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null") + ): + try: + cert_key_type = utils.get_key_type_from_certificate(body) + except ValueError: + log_file.write("Error in processing certificate. ID: %s\n" % cert_id) + else: + log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) + stmt = text( + "update certificates set key_type=:key_type where id=:id" + ) + stmt = stmt.bindparams(key_type=cert_key_type, id=cert_id) + op.execute(stmt) + + commit() + + log_file.write("--- %s seconds ---\n" % (time.time() - start_time)) + + +def commit(): + stmt = text("commit") + op.execute(stmt) From 9211178e77b1c78a17775c27e1caaee8d0e0756f Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:31:38 -0700 Subject: [PATCH 2/7] Added date-time and modified log file name --- lemur/migrations/versions/c301c59688d2_.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 6bd94cfb..2c91783f 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -28,12 +28,13 @@ from alembic import op from sqlalchemy.sql import text from lemur.common import utils import time +import datetime -log_file = open('upgrade_logs', 'a') +log_file = open('db_upgrade.log', 'a') def upgrade(): - log_file.write("\n*** Starting new run ***\n") + log_file.write("\n*** Starting new run(%s) ***\n" % datetime.datetime.now()) start_time = time.time() # Update RSA keys using the key length information From 921e8d8236d3be4b1bf9cfa044fe581f0744a2f2 Mon Sep 17 00:00:00 2001 From: sayali Date: Tue, 22 Sep 2020 18:46:15 -0700 Subject: [PATCH 3/7] Add error message to the logs --- lemur/migrations/versions/c301c59688d2_.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 2c91783f..0dd46315 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -6,9 +6,12 @@ the algorithm is determined based on the key length. For rest of the keys, the certificate body is parsed to determine the exact key type information. Each individual change is explicitly committed. The logs are added to file -named upgrade_logs in current working directory. If faced any issue while -running this upgrade, there is no harm in re-running the upgrade. Each run -processes only the keys for which key type information is not yet determined. +named db_upgrade.log in current working directory. Any error encountered +while parsing a certificate will also be logged along with the certificate +ID. If faced any issue while running this upgrade, there is no harm in +re-running the upgrade. Each run processes only the keys for which key type +information is not yet determined. + A successful end to end run will end up updating the Alembic Version to new Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported by Lemur. This could be a long running job depending upon the number of @@ -89,8 +92,8 @@ def update_key_type(): ): try: cert_key_type = utils.get_key_type_from_certificate(body) - except ValueError: - log_file.write("Error in processing certificate. ID: %s\n" % cert_id) + except ValueError as e: + log_file.write("Error in processing certificate - ID: %s Error: %s \n" % (cert_id, str(e))) else: log_file.write("Processing certificate - ID: %s key_type: %s\n" % (cert_id, cert_key_type)) stmt = text( From e3fa0726080b6a10434ff1a68871fe14998a133a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 10:17:30 -0700 Subject: [PATCH 4/7] Update c301c59688d2_.py language --- lemur/migrations/versions/c301c59688d2_.py | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 0dd46315..8f1941b2 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -1,21 +1,23 @@ """ -This upgrade of database updates the key_type information for certificates -that are either still valid or have expired in last 30 days. For RSA keys, -the algorithm is determined based on the key length. For rest of the keys, -the certificate body is parsed to determine the exact key type information. +This database upgrade updates the key_type information for either +still valid or expired certificates in the last 30 days. For RSA +keys, the algorithm is determined based on the key length. For +the rest of the keys, the certificate body is parsed to determine +the exact key_type information. -Each individual change is explicitly committed. The logs are added to file -named db_upgrade.log in current working directory. Any error encountered -while parsing a certificate will also be logged along with the certificate -ID. If faced any issue while running this upgrade, there is no harm in -re-running the upgrade. Each run processes only the keys for which key type -information is not yet determined. +Each individual DB change is explicitly committed, and the +respective log is added to a file named db_upgrade.log in the current +working directory. Any error encountered while parsing a certificate +will also be logged along with the certificate ID. If faced with +any issue while running this upgrade, there is no harm in +re-running the upgrade. Each run processes only rows for which +key_type information is not yet determined. -A successful end to end run will end up updating the Alembic Version to new -Revision ID c301c59688d2. Currently only RSA and ECC certificates are supported -by Lemur. This could be a long running job depending upon the number of -keys it may process. +A successful complete run will end up updating the Alembic Version +to the new Revision ID c301c59688d2. Currently, only RSA and ECC +certificates are supported by Lemur. This could be a long-running +job depending upon the number of DB entries it may process. Revision ID: c301c59688d2 Revises: 434c29e40511 From 19b693f636a77b7a460e2a24f9d557aa410f1a7a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Wed, 23 Sep 2020 10:21:23 -0700 Subject: [PATCH 5/7] Update c301c59688d2_.py language --- lemur/migrations/versions/c301c59688d2_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index 8f1941b2..c96272ff 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -87,7 +87,7 @@ def update_key_type(): conn = op.get_bind() start_time = time.time() - # Loop through all certificates are valid today or expired in last 30 days + # Loop through all certificates that are valid today or expired in the last 30 days. for cert_id, body in conn.execute( text( "select id, body from certificates where bits < 1024 and not_after > CURRENT_DATE - 31 and key_type is null") From 710290f590fca0b523b73a9f35d213fc6cad6879 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 23 Sep 2020 11:45:36 -0700 Subject: [PATCH 6/7] Formatting changes --- lemur/migrations/versions/c301c59688d2_.py | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lemur/migrations/versions/c301c59688d2_.py b/lemur/migrations/versions/c301c59688d2_.py index c96272ff..3b0a86f7 100644 --- a/lemur/migrations/versions/c301c59688d2_.py +++ b/lemur/migrations/versions/c301c59688d2_.py @@ -6,18 +6,18 @@ keys, the algorithm is determined based on the key length. For the rest of the keys, the certificate body is parsed to determine the exact key_type information. -Each individual DB change is explicitly committed, and the -respective log is added to a file named db_upgrade.log in the current -working directory. Any error encountered while parsing a certificate -will also be logged along with the certificate ID. If faced with -any issue while running this upgrade, there is no harm in -re-running the upgrade. Each run processes only rows for which -key_type information is not yet determined. +Each individual DB change is explicitly committed, and the respective +log is added to a file named db_upgrade.log in the current working +directory. Any error encountered while parsing a certificate will +also be logged along with the certificate ID. If faced with any issue +while running this upgrade, there is no harm in re-running the upgrade. +Each run processes only rows for which key_type information is not yet +determined. -A successful complete run will end up updating the Alembic Version -to the new Revision ID c301c59688d2. Currently, only RSA and ECC -certificates are supported by Lemur. This could be a long-running -job depending upon the number of DB entries it may process. +A successful complete run will end up updating the Alembic Version to +the new Revision ID c301c59688d2. Currently, Lemur supports only RSA +and ECC certificates. This could be a long-running job depending upon +the number of DB entries it may process. Revision ID: c301c59688d2 Revises: 434c29e40511 @@ -72,7 +72,7 @@ def update_key_type_rsa(bits): log_file.write("Processing certificate with key type RSA %s\n" % bits) stmt = text( - "update certificates set key_type='RSA{0}' where bits={0} and not_after > CURRENT_DATE - 31 and key_type is null".format(bits) + f"update certificates set key_type='RSA{bits}' where bits={bits} and not_after > CURRENT_DATE - 31 and key_type is null" ) log_file.write("Query: %s\n" % stmt) From 12af0ecb457263f9dad344c4af458de473ff6a59 Mon Sep 17 00:00:00 2001 From: sayali Date: Wed, 23 Sep 2020 11:46:38 -0700 Subject: [PATCH 7/7] UT get_key_type_from_certificate --- lemur/tests/test_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lemur/tests/test_utils.py b/lemur/tests/test_utils.py index 1dac39bb..162e53b0 100644 --- a/lemur/tests/test_utils.py +++ b/lemur/tests/test_utils.py @@ -2,11 +2,13 @@ import pytest from lemur.tests.vectors import ( SAN_CERT, + SAN_CERT_STR, INTERMEDIATE_CERT, ROOTCA_CERT, EC_CERT_EXAMPLE, ECDSA_PRIME256V1_CERT, ECDSA_SECP384r1_CERT, + ECDSA_SECP384r1_CERT_STR, DSA_CERT, ) @@ -106,3 +108,9 @@ def test_is_selfsigned(selfsigned_cert): # unsupported algorithm (DSA) with pytest.raises(Exception): is_selfsigned(DSA_CERT) + + +def test_get_key_type_from_certificate(): + from lemur.common.utils import get_key_type_from_certificate + assert (get_key_type_from_certificate(SAN_CERT_STR) == "RSA2048") + assert (get_key_type_from_certificate(ECDSA_SECP384r1_CERT_STR) == "ECCSECP384R1")