From 31a86687e72e02883a9d80abe345b5b5b64d2667 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 14 Jan 2019 09:20:02 -0800 Subject: [PATCH 1/3] Reduce the expense of joins --- lemur/certificates/service.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index e4503324..1b203260 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -20,7 +20,6 @@ from lemur.common.utils import generate_private_key, truthiness from lemur.destinations.models import Destination from lemur.domains.models import Domain from lemur.extensions import metrics, sentry, signals -from lemur.models import certificate_associations from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate from lemur.plugins.base import plugins @@ -341,13 +340,13 @@ def render(args): elif 'id' in terms: query = query.filter(Certificate.id == cast(terms[1], Integer)) elif 'name' in terms: - query = query.outerjoin(certificate_associations).outerjoin(Domain).filter( + query = query.filter( or_( Certificate.name.ilike(term), - Domain.name.ilike(term), + Certificate.domains.any(Domain.name.ilike(term)), Certificate.cn.ilike(term), ) - ).group_by(Certificate.id) + ) else: query = database.filter(query, Certificate, terms) From 3567a768d5a0e281d41d7c3f8bdb73bf3c6a7728 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 14 Jan 2019 13:35:55 -0800 Subject: [PATCH 2/3] Compare certificate hashes to determine if Lemur already has a synced certificate --- lemur/common/utils.py | 11 +++++++++++ lemur/sources/service.py | 5 +++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 62e59d69..0504c958 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -12,6 +12,7 @@ import string import sqlalchemy from cryptography import x509 from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import rsa, ec from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask_restful.reqparse import RequestParser @@ -226,3 +227,13 @@ def truthiness(s): """If input string resembles something truthy then return True, else False.""" return s.lower() in ('true', 'yes', 'on', 't', '1') + + +def find_matching_certificates_by_hash(cert, matching_certs): + """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs), + determine if any of the certificate hashes match and return the matches.""" + matching = [] + for c in matching_certs: + if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(hashes.SHA256()): + matching.append(c) + return matching diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 227f1bce..55d2ee62 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.utils import parse_certificate +from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate from lemur.common.defaults import serial from lemur.plugins.base import plugins @@ -126,7 +126,8 @@ def sync_certificates(source, user): if not exists: cert = parse_certificate(certificate['body']) - exists = certificate_service.get_by_serial(serial(cert)) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) if not certificate.get('owner'): certificate['owner'] = user.email From d3284a4006a87940ca485adea33957c116176c02 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Mon, 14 Jan 2019 17:52:06 -0800 Subject: [PATCH 3/3] adjusting the query to filter authorities based on matching CN --- lemur/authorities/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lemur/authorities/service.py b/lemur/authorities/service.py index 024cb42a..41c381e3 100644 --- a/lemur/authorities/service.py +++ b/lemur/authorities/service.py @@ -15,6 +15,7 @@ from lemur import database from lemur.common.utils import truthiness from lemur.extensions import metrics from lemur.authorities.models import Authority +from lemur.certificates.models import Certificate from lemur.roles import service as role_service from lemur.certificates.service import upload @@ -179,7 +180,12 @@ def render(args): if 'active' in filt: query = query.filter(Authority.active == truthiness(terms[1])) elif 'cn' in filt: - query = query.join(Authority.active == truthiness(terms[1])) + term = '%{0}%'.format(terms[1]) + sub_query = database.session_query(Certificate.root_authority_id) \ + .filter(Certificate.cn.ilike(term)) \ + .subquery() + + query = query.filter(Authority.id.in_(sub_query)) else: query = database.filter(query, Authority, terms)