From 52e773230d3fcc5c01fe36b644835381f733b339 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 5 Nov 2018 10:29:11 -0800 Subject: [PATCH] Add new gin index to optimize ILIKE queries --- lemur/certificates/models.py | 8 +++++ lemur/certificates/service.py | 3 +- lemur/common/celery.py | 4 +-- lemur/domains/models.py | 7 ++++- lemur/migrations/versions/ee827d1e1974_.py | 35 ++++++++++++++++++++++ requirements-dev.txt | 20 ++++++------- requirements-docs.txt | 2 +- requirements-tests.txt | 25 ++++++++-------- requirements.txt | 22 +++++++------- 9 files changed, 86 insertions(+), 40 deletions(-) create mode 100644 lemur/migrations/versions/ee827d1e1974_.py diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 6b438c06..7a1706f4 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -77,6 +77,14 @@ def get_or_increase_name(name, serial): class Certificate(db.Model): __tablename__ = 'certificates' + __table_args__ = ( + Index('ix_certificates_cn', "cn", + postgresql_ops={"cn": "gin_trgm_ops"}, + postgresql_using='gin'), + Index('ix_certificates_name', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True) external_id = Column(String(128)) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index fd6142ee..8fc031c4 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -362,7 +362,8 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now) - return database.sort_and_page(query, Certificate, args) + result = database.sort_and_page(query, Certificate, args) + return result def create_csr(**csr_config): diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 1711b452..69bd9ce1 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -193,8 +193,8 @@ def clean_source(source): @celery.task() def sync_all_sources(): """ - This function will sync certificates from all sources. This function triggers one celery task per source. - """ + This function will sync certificates from all sources. This function triggers one celery task per source. + """ sources = validate_sources("all") for source in sources: current_app.logger.debug("Creating celery task to sync source {}".format(source.label)) diff --git a/lemur/domains/models.py b/lemur/domains/models.py index d0c7fef4..05fccd9c 100644 --- a/lemur/domains/models.py +++ b/lemur/domains/models.py @@ -7,13 +7,18 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy import Column, Integer, String, Boolean +from sqlalchemy import Column, Integer, String, Boolean, Index from lemur.database import db class Domain(db.Model): __tablename__ = 'domains' + __table_args__ = ( + Index('ix_domains_name_gin', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) name = Column(String(256), index=True) sensitive = Column(Boolean, default=False) diff --git a/lemur/migrations/versions/ee827d1e1974_.py b/lemur/migrations/versions/ee827d1e1974_.py new file mode 100644 index 00000000..26f7cea4 --- /dev/null +++ b/lemur/migrations/versions/ee827d1e1974_.py @@ -0,0 +1,35 @@ +"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries. + +Revision ID: ee827d1e1974 +Revises: 7ead443ba911 +Create Date: 2018-11-05 09:49:40.226368 + +""" + +# revision identifiers, used by Alembic. +revision = 'ee827d1e1974' +down_revision = '7ead443ba911' + +from alembic import op +from sqlalchemy.exc import ProgrammingError + +def upgrade(): + try: + connection = op.get_bind() + connection.execute("CREATE EXTENSION pg_trgm") + except ProgrammingError as e: + # Extension is most likely already enabled + connection.execute("ROLLBACK") + + op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'}, + postgresql_using='gin') + op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False, + postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin') + op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'}, + postgresql_using='gin') + + +def downgrade(): + op.drop_index('ix_domains_name', table_name='domains') + op.drop_index('ix_certificates_name', table_name='certificates') + op.drop_index('ix_certificates_cn', table_name='certificates') diff --git a/requirements-dev.txt b/requirements-dev.txt index c473aa56..428d2082 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,32 +8,30 @@ aspy.yaml==1.1.1 # via pre-commit bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.10.15 # via requests -cffi==1.11.5 # via cmarkgfm cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests -cmarkgfm==0.4.2 # via readme-renderer docutils==0.14 # via readme-renderer -flake8==3.5.0 -future==0.16.0 # via readme-renderer +flake8==3.6.0 identify==1.1.7 # via pre-commit idna==2.7 # via requests +importlib-metadata==0.6 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine -pre-commit==1.11.2 -pycodestyle==2.3.1 # via flake8 -pycparser==2.19 # via cffi -pyflakes==1.6.0 # via flake8 +pre-commit==1.12.0 +pycodestyle==2.4.0 # via flake8 +pyflakes==2.0.0 # via flake8 pygments==2.2.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit -readme-renderer==22.0 # via twine +readme-renderer==24.0 # via twine requests-toolbelt==0.8.0 # via twine requests==2.20.0 # via requests-toolbelt, twine six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.28.1 # via twine twine==1.12.1 -urllib3==1.24 # via requests -virtualenv==16.0.0 # via pre-commit +urllib3==1.24.1 # via requests +virtualenv==16.1.0 # via pre-commit webencodings==0.5.1 # via bleach diff --git a/requirements-docs.txt b/requirements-docs.txt index 812a947f..c89ca75b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -69,7 +69,7 @@ pygments==2.2.0 # via sphinx pyjwt==1.6.4 pynacl==1.3.0 pyopenssl==18.0.0 -pyparsing==2.2.2 # via packaging +pyparsing==2.3.0 # via packaging pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 diff --git a/requirements-tests.txt b/requirements-tests.txt index 2ad412db..64875903 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,10 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -biscuits==0.1.1 # via responses -boto3==1.9.28 # via moto +boto3==1.9.37 # via moto boto==2.49.0 # via moto -botocore==1.12.28 # via boto3, moto, s3transfer +botocore==1.12.37 # via boto3, moto, s3transfer certifi==2018.10.15 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -27,40 +26,40 @@ factory-boy==2.11.1 faker==0.9.2 flask==1.0.2 # via pytest-flask freezegun==0.3.11 -future==0.16.0 # via python-jose +future==0.17.1 # via python-jose idna==2.7 # via cryptography, requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.0 # via aws-xray-sdk -markupsafe==1.0 # via jinja2 +markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.3.0 # via pytest moto==1.3.4 nose==1.3.7 -pbr==5.0.0 # via mock +pbr==5.1.0 # via mock pluggy==0.8.0 # via pytest py==1.7.0 # via pytest pyaml==17.12.1 # via moto pycparser==2.19 # via cffi -pycryptodome==3.6.6 # via python-jose +pycryptodome==3.7.0 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==3.9.1 -python-dateutil==2.7.3 # via botocore, faker, freezegun, moto +pytest==3.10.0 +python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.5 # via moto +pytz==2018.7 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.1 # via moto +responses==0.10.2 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.23 # via botocore, requests -websocket-client==0.53.0 # via docker +urllib3==1.24.1 # via botocore, requests +websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.10.11 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.txt b/requirements.txt index ffe6372f..51d93b09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,9 @@ # acme==0.27.1 alembic-autogenerate-enums==0.0.2 -alembic==1.0.1 # via flask-migrate +alembic==1.0.2 # via flask-migrate amqp==2.3.2 # via kombu -aniso8601==3.0.2 # via flask-restful +aniso8601==4.0.1 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 @@ -37,7 +37,7 @@ flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 -future==0.16.0 +future==0.17.1 gunicorn==19.9.0 idna==2.7 # via cryptography, requests inflection==0.3.1 @@ -49,13 +49,13 @@ jsonlines==1.2.0 # via cloudflare kombu==4.2.1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic -markupsafe==1.0 # via jinja2, mako -marshmallow-sqlalchemy==0.14.1 -marshmallow==2.16.0 +markupsafe==1.1.0 # via jinja2, mako +marshmallow-sqlalchemy==0.15.0 +marshmallow==2.16.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.0.0 # via mock +pbr==5.1.0 # via mock pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap @@ -65,10 +65,10 @@ pyjwt==1.6.4 pynacl==1.3.0 # via paramiko pyopenssl==18.0.0 pyrfc3339==1.1 # via acme -python-dateutil==2.7.3 # via alembic, arrow, botocore +python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339 +pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 redis==2.10.6 # via celery @@ -78,9 +78,9 @@ retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.13 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 -urllib3==1.24 # via requests +urllib3==1.24.1 # via requests vine==1.1.4 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0