From cc0b2d5439c26822deded52cc68f0364dd52db64 Mon Sep 17 00:00:00 2001 From: Ilya Labun Date: Mon, 13 Jan 2020 14:31:14 +0100 Subject: [PATCH 1/2] Added new lowercase indexes for certificates cn, name and domains name Co-authored-by: Javier Ramos --- lemur/migrations/versions/8323a5ea723a_.py | 50 ++++++++++++++++++++++ lemur/migrations/versions/ee827d1e1974_.py | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 lemur/migrations/versions/8323a5ea723a_.py diff --git a/lemur/migrations/versions/8323a5ea723a_.py b/lemur/migrations/versions/8323a5ea723a_.py new file mode 100644 index 00000000..9505cdb1 --- /dev/null +++ b/lemur/migrations/versions/8323a5ea723a_.py @@ -0,0 +1,50 @@ +"""Add lowercase index for certificate name and cn and also for domain name + +Revision ID: 8323a5ea723a +Revises: b33c838cb669 +Create Date: 2020-01-10 10:51:44.776052 + +""" + +# revision identifiers, used by Alembic. +revision = '8323a5ea723a' +down_revision = 'b33c838cb669' + +from alembic import op +from sqlalchemy import text + +import sqlalchemy as sa + + +def upgrade(): + op.create_index( + "ix_certificates_cn_lower", + "certificates", + [text("lower(cn)")], + unique=False, + postgresql_ops={"lower(cn)": "gin_trgm_ops"}, + postgresql_using="gin", + ) + op.create_index( + "ix_certificates_name_lower", + "certificates", + [text("lower(name)")], + unique=False, + postgresql_ops={"lower(name)": "gin_trgm_ops"}, + postgresql_using="gin", + ) + op.create_index( + "ix_domains_name_lower", + "domains", + [text("lower(name)")], + unique=False, + postgresql_ops={"lower(name)": "gin_trgm_ops"}, + postgresql_using="gin", + ) + + + +def downgrade(): + op.drop_index("ix_certificates_cn_lower", table_name="certificates") + op.drop_index("ix_certificates_name_lower", table_name="certificates") + op.drop_index("ix_domains_name_lower", table_name="domains") diff --git a/lemur/migrations/versions/ee827d1e1974_.py b/lemur/migrations/versions/ee827d1e1974_.py index 56696fe3..649f1ed7 100644 --- a/lemur/migrations/versions/ee827d1e1974_.py +++ b/lemur/migrations/versions/ee827d1e1974_.py @@ -45,6 +45,6 @@ def upgrade(): def downgrade(): - op.drop_index("ix_domains_name", table_name="domains") + op.drop_index("ix_domains_name_gin", table_name="domains") op.drop_index("ix_certificates_name", table_name="certificates") op.drop_index("ix_certificates_cn", table_name="certificates") From bc1a2cf69ccc41dade086202591eaef72e6a5283 Mon Sep 17 00:00:00 2001 From: Ilya Labun Date: Mon, 13 Jan 2020 14:43:41 +0100 Subject: [PATCH 2/2] Optimize certificates SQL query Co-authored-by: Javier Ramos --- lemur/certificates/service.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 0e91b563..c8d732bb 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -20,6 +20,7 @@ from lemur.common.utils import generate_private_key, truthiness from lemur.destinations.models import Destination from lemur.domains.models import Domain from lemur.extensions import metrics, sentry, signals +from lemur.models import certificate_associations from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate from lemur.plugins.base import plugins @@ -379,8 +380,8 @@ def render(args): elif "cn" in terms: query = query.filter( or_( - Certificate.cn.ilike(term), - Certificate.domains.any(Domain.name.ilike(term)), + func.lower(Certificate.cn).like(term.lower()), + Certificate.id.in_(like_domain_query(term)), ) ) elif "id" in terms: @@ -388,9 +389,9 @@ def render(args): elif "name" in terms: query = query.filter( or_( - Certificate.name.ilike(term), - Certificate.domains.any(Domain.name.ilike(term)), - Certificate.cn.ilike(term), + func.lower(Certificate.name).like(term.lower()), + Certificate.id.in_(like_domain_query(term)), + func.lower(Certificate.cn).like(term.lower()), ) ) elif "fixedName" in terms: @@ -435,6 +436,14 @@ def render(args): return result +def like_domain_query(term): + domain_query = database.session_query(Domain.id) + domain_query = domain_query.filter(func.lower(Domain.name).like(term.lower())) + assoc_query = database.session_query(certificate_associations.c.certificate_id) + assoc_query = assoc_query.filter(certificate_associations.c.domain_id.in_(domain_query)) + return assoc_query + + def query_name(certificate_name, args): """ Helper function that queries for a certificate by name