Merge pull request #2037 from castrapel/db_optimize_select

Add new gin index to optimize ILIKE queries

.travis.yml

@@ -28,6 +28,7 @@ env:
 before_script:
   - psql -c "create database lemur;" -U postgres
   - psql -c "create user lemur with password 'lemur;'" -U postgres
+  - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
   - npm config set registry https://registry.npmjs.org
   - npm install -g bower
   - pip install --upgrade setuptools
@@ -45,4 +46,4 @@ after_success:
 
 notifications:
   email:
-    kglisson@netflix.com
+    ccastrapel@netflix.com

lemur/certificates/models.py

@@ -77,6 +77,14 @@ def get_or_increase_name(name, serial):
 
 class Certificate(db.Model):
     __tablename__ = 'certificates'
+    __table_args__ = (
+        Index('ix_certificates_cn', "cn",
+              postgresql_ops={"cn": "gin_trgm_ops"},
+              postgresql_using='gin'),
+        Index('ix_certificates_name', "name",
+              postgresql_ops={"name": "gin_trgm_ops"},
+              postgresql_using='gin'),
+    )
     id = Column(Integer, primary_key=True)
     ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
     external_id = Column(String(128))

lemur/certificates/service.py

@@ -362,7 +362,8 @@ def render(args):
         now = arrow.now().format('YYYY-MM-DD')
         query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
 
-    return database.sort_and_page(query, Certificate, args)
+    result = database.sort_and_page(query, Certificate, args)
+    return result
 
 
 def create_csr(**csr_config):

lemur/common/celery.py

@@ -193,8 +193,8 @@ def clean_source(source):
 @celery.task()
 def sync_all_sources():
     """
-        This function will sync certificates from all sources. This function triggers one celery task per source.
+    This function will sync certificates from all sources. This function triggers one celery task per source.
-        """
+    """
     sources = validate_sources("all")
     for source in sources:
         current_app.logger.debug("Creating celery task to sync source {}".format(source.label))

lemur/domains/models.py

@@ -7,13 +7,18 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 
 """
-from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy import Column, Integer, String, Boolean, Index
 
 from lemur.database import db
 
 
 class Domain(db.Model):
     __tablename__ = 'domains'
+    __table_args__ = (
+        Index('ix_domains_name_gin', "name",
+              postgresql_ops={"name": "gin_trgm_ops"},
+              postgresql_using='gin'),
+    )
     id = Column(Integer, primary_key=True)
     name = Column(String(256), index=True)
     sensitive = Column(Boolean, default=False)

lemur/migrations/versions/ee827d1e1974_.py

@@ -0,0 +1,31 @@
+"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries.
+
+Revision ID: ee827d1e1974
+Revises: 7ead443ba911
+Create Date: 2018-11-05 09:49:40.226368
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ee827d1e1974'
+down_revision = '7ead443ba911'
+
+from alembic import op
+from sqlalchemy.exc import ProgrammingError
+
+def upgrade():
+    connection = op.get_bind()
+    connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+    op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'},
+                    postgresql_using='gin')
+    op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False,
+                    postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin')
+    op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'},
+                    postgresql_using='gin')
+
+
+def downgrade():
+    op.drop_index('ix_domains_name', table_name='domains')
+    op.drop_index('ix_certificates_name', table_name='certificates')
+    op.drop_index('ix_certificates_cn', table_name='certificates')

requirements-dev.in

@@ -1,6 +1,6 @@
 # Run `make up-reqs` to update pinned dependencies in requirement text files
 
-flake8>=3.2,<4.0
+flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors.
 pre-commit
 invoke
 twine

requirements-dev.txt

@@ -8,32 +8,30 @@ aspy.yaml==1.1.1          # via pre-commit
 bleach==3.0.2             # via readme-renderer
 cached-property==1.5.1    # via pre-commit
 certifi==2018.10.15       # via requests
-cffi==1.11.5              # via cmarkgfm
 cfgv==1.1.0               # via pre-commit
 chardet==3.0.4            # via requests
-cmarkgfm==0.4.2           # via readme-renderer
 docutils==0.14            # via readme-renderer
 flake8==3.5.0
-future==0.16.0            # via readme-renderer
 identify==1.1.7           # via pre-commit
 idna==2.7                 # via requests
+importlib-metadata==0.6   # via pre-commit
+importlib-resources==1.0.2  # via pre-commit
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.2
 pkginfo==1.4.2            # via twine
-pre-commit==1.11.2
+pre-commit==1.12.0
 pycodestyle==2.3.1        # via flake8
-pycparser==2.19           # via cffi
 pyflakes==1.6.0           # via flake8
 pygments==2.2.0           # via readme-renderer
 pyyaml==3.13              # via aspy.yaml, pre-commit
-readme-renderer==22.0     # via twine
+readme-renderer==24.0     # via twine
 requests-toolbelt==0.8.0  # via twine
 requests==2.20.0          # via requests-toolbelt, twine
 six==1.11.0               # via bleach, cfgv, pre-commit, readme-renderer
 toml==0.10.0              # via pre-commit
 tqdm==4.28.1              # via twine
 twine==1.12.1
-urllib3==1.24             # via requests
+urllib3==1.24.1           # via requests
-virtualenv==16.0.0        # via pre-commit
+virtualenv==16.1.0        # via pre-commit
 webencodings==0.5.1       # via bleach

requirements-docs.txt

@@ -7,9 +7,9 @@
 acme==0.27.1
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.1
+alembic==1.0.2
 amqp==2.3.2
-aniso8601==3.0.2
+aniso8601==4.0.1
 arrow==0.12.1
 asn1crypto==0.24.0
 asyncpool==1.0
@@ -39,7 +39,7 @@ flask-restful==0.3.6
 flask-script==2.0.6
 flask-sqlalchemy==2.3.2
 flask==0.12
-future==0.16.0
+future==0.17.1
 gunicorn==19.9.0
 idna==2.7
 imagesize==1.1.0          # via sphinx
@@ -52,14 +52,14 @@ jsonlines==1.2.0
 kombu==4.2.1
 lockfile==0.12.2
 mako==1.0.7
-markupsafe==1.0
+markupsafe==1.1.0
-marshmallow-sqlalchemy==0.14.1
+marshmallow-sqlalchemy==0.15.0
-marshmallow==2.16.0
+marshmallow==2.16.3
 mock==2.0.0
 ndg-httpsclient==0.5.1
 packaging==18.0           # via sphinx
 paramiko==2.4.2
-pbr==5.0.0
+pbr==5.1.0
 pem==18.2.0
 psycopg2==2.7.5
 pyasn1-modules==0.2.2
@@ -69,11 +69,11 @@ pygments==2.2.0           # via sphinx
 pyjwt==1.6.4
 pynacl==1.3.0
 pyopenssl==18.0.0
-pyparsing==2.2.2          # via packaging
+pyparsing==2.3.0          # via packaging
 pyrfc3339==1.1
-python-dateutil==2.7.3
+python-dateutil==2.7.5
 python-editor==1.0.3
-pytz==2018.5
+pytz==2018.7
 pyyaml==3.13
 raven[flask]==6.9.0
 redis==2.10.6
@@ -88,9 +88,9 @@ sphinx==1.8.1
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx
 sqlalchemy-utils==0.33.6
-sqlalchemy==1.2.12
+sqlalchemy==1.2.13
 tabulate==0.8.2
-urllib3==1.24
+urllib3==1.24.1
 vine==1.1.4
 werkzeug==0.14.1
 xmltodict==0.11.0

requirements-tests.txt

@@ -8,10 +8,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-biscuits==0.1.1           # via responses
+boto3==1.9.37             # via moto
-boto3==1.9.28             # via moto
 boto==2.49.0              # via moto
-botocore==1.12.28         # via boto3, moto, s3transfer
+botocore==1.12.37         # via boto3, moto, s3transfer
 certifi==2018.10.15       # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -27,40 +26,40 @@ factory-boy==2.11.1
 faker==0.9.2
 flask==1.0.2              # via pytest-flask
 freezegun==0.3.11
-future==0.16.0            # via python-jose
+future==0.17.1            # via python-jose
 idna==2.7                 # via cryptography, requests
 itsdangerous==1.1.0       # via flask
 jinja2==2.10              # via flask, moto
 jmespath==0.9.3           # via boto3, botocore
 jsondiff==1.1.1           # via moto
 jsonpickle==1.0           # via aws-xray-sdk
-markupsafe==1.0           # via jinja2
+markupsafe==1.1.0         # via jinja2
 mock==2.0.0               # via moto
 more-itertools==4.3.0     # via pytest
 moto==1.3.4
 nose==1.3.7
-pbr==5.0.0                # via mock
+pbr==5.1.0                # via mock
 pluggy==0.8.0             # via pytest
 py==1.7.0                 # via pytest
 pyaml==17.12.1            # via moto
 pycparser==2.19           # via cffi
-pycryptodome==3.6.6       # via python-jose
+pycryptodome==3.7.0       # via python-jose
 pyflakes==2.0.0
 pytest-flask==0.14.0
 pytest-mock==1.10.0
-pytest==3.9.1
+pytest==3.10.0
-python-dateutil==2.7.3    # via botocore, faker, freezegun, moto
+python-dateutil==2.7.5    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
-pytz==2018.5              # via moto
+pytz==2018.7              # via moto
 pyyaml==3.13              # via pyaml
 requests-mock==1.5.2
 requests==2.20.0          # via aws-xray-sdk, docker, moto, requests-mock, responses
-responses==0.10.1         # via moto
+responses==0.10.2         # via moto
 s3transfer==0.1.13        # via boto3
 six==1.11.0               # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
 text-unidecode==1.2       # via faker
-urllib3==1.23             # via botocore, requests
+urllib3==1.24.1           # via botocore, requests
-websocket-client==0.53.0  # via docker
+websocket-client==0.54.0  # via docker
 werkzeug==0.14.1          # via flask, moto, pytest-flask
 wrapt==1.10.11            # via aws-xray-sdk
 xmltodict==0.11.0         # via moto

requirements.txt

@@ -6,9 +6,9 @@
 #
 acme==0.27.1
 alembic-autogenerate-enums==0.0.2
-alembic==1.0.1            # via flask-migrate
+alembic==1.0.2            # via flask-migrate
 amqp==2.3.2               # via kombu
-aniso8601==3.0.2          # via flask-restful
+aniso8601==4.0.1          # via flask-restful
 arrow==0.12.1
 asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
@@ -37,7 +37,7 @@ flask-restful==0.3.6
 flask-script==2.0.6
 flask-sqlalchemy==2.3.2
 flask==0.12
-future==0.16.0
+future==0.17.1
 gunicorn==19.9.0
 idna==2.7                 # via cryptography, requests
 inflection==0.3.1
@@ -49,13 +49,13 @@ jsonlines==1.2.0          # via cloudflare
 kombu==4.2.1              # via celery
 lockfile==0.12.2
 mako==1.0.7               # via alembic
-markupsafe==1.0           # via jinja2, mako
+markupsafe==1.1.0         # via jinja2, mako
-marshmallow-sqlalchemy==0.14.1
+marshmallow-sqlalchemy==0.15.0
-marshmallow==2.16.0
+marshmallow==2.16.3
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
 paramiko==2.4.2
-pbr==5.0.0                # via mock
+pbr==5.1.0                # via mock
 pem==18.2.0
 psycopg2==2.7.5
 pyasn1-modules==0.2.2     # via python-ldap
@@ -65,10 +65,10 @@ pyjwt==1.6.4
 pynacl==1.3.0             # via paramiko
 pyopenssl==18.0.0
 pyrfc3339==1.1            # via acme
-python-dateutil==2.7.3    # via alembic, arrow, botocore
+python-dateutil==2.7.5    # via alembic, arrow, botocore
 python-editor==1.0.3      # via alembic
 python-ldap==3.1.0
-pytz==2018.5              # via acme, celery, flask-restful, pyrfc3339
+pytz==2018.7              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==3.13              # via cloudflare
 raven[flask]==6.9.0
 redis==2.10.6             # via celery
@@ -78,9 +78,9 @@ retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.11.0
 sqlalchemy-utils==0.33.6
-sqlalchemy==1.2.12        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.2.13        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
-urllib3==1.24             # via requests
+urllib3==1.24.1           # via requests
 vine==1.1.4               # via amqp
 werkzeug==0.14.1          # via flask
 xmltodict==0.11.0