Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add new gin index to optimize ILIKE queries

This commit is contained in:
Curtis Castrapel 2018-11-05 10:29:11 -08:00
parent baa73c7f3e
commit 52e773230d
9 changed files with 86 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lemur/certificates/models.py
View File

@ -77,6 +77,14 @@ def get_or_increase_name(name, serial):
class Certificate(db.Model):
__tablename__ = 'certificates'
__table_args__ = (
Index('ix_certificates_cn', "cn",
postgresql_ops={"cn": "gin_trgm_ops"},
postgresql_using='gin'),
Index('ix_certificates_name', "name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
)
id = Column(Integer, primary_key=True)
ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
external_id = Column(String(128))

3
lemur/certificates/service.py
View File

@ -362,7 +362,8 @@ def render(args):
now = arrow.now().format('YYYY-MM-DD')
query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
return database.sort_and_page(query, Certificate, args)
result = database.sort_and_page(query, Certificate, args)
return result
def create_csr(**csr_config):

7
lemur/domains/models.py
View File

@ -7,13 +7,18 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy import Column, Integer, String, Boolean
from sqlalchemy import Column, Integer, String, Boolean, Index
from lemur.database import db
class Domain(db.Model):
__tablename__ = 'domains'
__table_args__ = (
Index('ix_domains_name_gin', "name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
)
id = Column(Integer, primary_key=True)
name = Column(String(256), index=True)
sensitive = Column(Boolean, default=False)

35
lemur/migrations/versions/ee827d1e1974_.py Normal file
View File

@ -0,0 +1,35 @@
"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries.
Revision ID: ee827d1e1974
Revises: 7ead443ba911
Create Date: 2018-11-05 09:49:40.226368
"""
# revision identifiers, used by Alembic.
revision = 'ee827d1e1974'
down_revision = '7ead443ba911'
from alembic import op
from sqlalchemy.exc import ProgrammingError
def upgrade():
try:
connection = op.get_bind()
connection.execute("CREATE EXTENSION pg_trgm")
except ProgrammingError as e:
# Extension is most likely already enabled
connection.execute("ROLLBACK")
op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'},
postgresql_using='gin')
op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False,
postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin')
op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'},
postgresql_using='gin')
def downgrade():
op.drop_index('ix_domains_name', table_name='domains')
op.drop_index('ix_certificates_name', table_name='certificates')
op.drop_index('ix_certificates_cn', table_name='certificates')

20
requirements-dev.txt
View File

@ -8,32 +8,30 @@ aspy.yaml==1.1.1 # via pre-commit
bleach==3.0.2 # via readme-renderer
cached-property==1.5.1 # via pre-commit
certifi==2018.10.15 # via requests
cffi==1.11.5 # via cmarkgfm
cfgv==1.1.0 # via pre-commit
chardet==3.0.4 # via requests
cmarkgfm==0.4.2 # via readme-renderer
docutils==0.14 # via readme-renderer
flake8==3.5.0
future==0.16.0 # via readme-renderer
flake8==3.6.0
identify==1.1.7 # via pre-commit
idna==2.7 # via requests
importlib-metadata==0.6 # via pre-commit
importlib-resources==1.0.2 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.2
pkginfo==1.4.2 # via twine
pre-commit==1.11.2
pycodestyle==2.3.1 # via flake8
pycparser==2.19 # via cffi
pyflakes==1.6.0 # via flake8
pre-commit==1.12.0
pycodestyle==2.4.0 # via flake8
pyflakes==2.0.0 # via flake8
pygments==2.2.0 # via readme-renderer
pyyaml==3.13 # via aspy.yaml, pre-commit
readme-renderer==22.0 # via twine
readme-renderer==24.0 # via twine
requests-toolbelt==0.8.0 # via twine
requests==2.20.0 # via requests-toolbelt, twine
six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.28.1 # via twine
twine==1.12.1
urllib3==1.24 # via requests
virtualenv==16.0.0 # via pre-commit
urllib3==1.24.1 # via requests
virtualenv==16.1.0 # via pre-commit
webencodings==0.5.1 # via bleach

2
requirements-docs.txt
View File

@ -69,7 +69,7 @@ pygments==2.2.0 # via sphinx
pyjwt==1.6.4
pynacl==1.3.0
pyopenssl==18.0.0
pyparsing==2.2.2 # via packaging
pyparsing==2.3.0 # via packaging
pyrfc3339==1.1
python-dateutil==2.7.3
python-editor==1.0.3

25
requirements-tests.txt
View File

@ -8,10 +8,9 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.2.1 # via pytest
attrs==18.2.0 # via pytest
aws-xray-sdk==0.95 # via moto
biscuits==0.1.1 # via responses
boto3==1.9.28 # via moto
boto3==1.9.37 # via moto
boto==2.49.0 # via moto
botocore==1.12.28 # via boto3, moto, s3transfer
botocore==1.12.37 # via boto3, moto, s3transfer
certifi==2018.10.15 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
@ -27,40 +26,40 @@ factory-boy==2.11.1
faker==0.9.2
flask==1.0.2 # via pytest-flask
freezegun==0.3.11
future==0.16.0 # via python-jose
future==0.17.1 # via python-jose
idna==2.7 # via cryptography, requests
itsdangerous==1.1.0 # via flask
jinja2==2.10 # via flask, moto
jmespath==0.9.3 # via boto3, botocore
jsondiff==1.1.1 # via moto
jsonpickle==1.0 # via aws-xray-sdk
markupsafe==1.0 # via jinja2
markupsafe==1.1.0 # via jinja2
mock==2.0.0 # via moto
more-itertools==4.3.0 # via pytest
moto==1.3.4
nose==1.3.7
pbr==5.0.0 # via mock
pbr==5.1.0 # via mock
pluggy==0.8.0 # via pytest
py==1.7.0 # via pytest
pyaml==17.12.1 # via moto
pycparser==2.19 # via cffi
pycryptodome==3.6.6 # via python-jose
pycryptodome==3.7.0 # via python-jose
pyflakes==2.0.0
pytest-flask==0.14.0
pytest-mock==1.10.0
pytest==3.9.1
python-dateutil==2.7.3 # via botocore, faker, freezegun, moto
pytest==3.10.0
python-dateutil==2.7.5 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.5 # via moto
pytz==2018.7 # via moto
pyyaml==3.13 # via pyaml
requests-mock==1.5.2
requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses
responses==0.10.1 # via moto
responses==0.10.2 # via moto
s3transfer==0.1.13 # via boto3
six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker
urllib3==1.23 # via botocore, requests
websocket-client==0.53.0 # via docker
urllib3==1.24.1 # via botocore, requests
websocket-client==0.54.0 # via docker
werkzeug==0.14.1 # via flask, moto, pytest-flask
wrapt==1.10.11 # via aws-xray-sdk
xmltodict==0.11.0 # via moto

22
requirements.txt
View File

@ -6,9 +6,9 @@
#
acme==0.27.1
alembic-autogenerate-enums==0.0.2
alembic==1.0.1 # via flask-migrate
alembic==1.0.2 # via flask-migrate
amqp==2.3.2 # via kombu
aniso8601==3.0.2 # via flask-restful
aniso8601==4.0.1 # via flask-restful
arrow==0.12.1
asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
@ -37,7 +37,7 @@ flask-restful==0.3.6
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask==0.12
future==0.16.0
future==0.17.1
gunicorn==19.9.0
idna==2.7 # via cryptography, requests
inflection==0.3.1
@ -49,13 +49,13 @@ jsonlines==1.2.0 # via cloudflare
kombu==4.2.1 # via celery
lockfile==0.12.2
mako==1.0.7 # via alembic
markupsafe==1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.14.1
marshmallow==2.16.0
markupsafe==1.1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.15.0
marshmallow==2.16.3
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.0.0 # via mock
pbr==5.1.0 # via mock
pem==18.2.0
psycopg2==2.7.5
pyasn1-modules==0.2.2 # via python-ldap
@ -65,10 +65,10 @@ pyjwt==1.6.4
pynacl==1.3.0 # via paramiko
pyopenssl==18.0.0
pyrfc3339==1.1 # via acme
python-dateutil==2.7.3 # via alembic, arrow, botocore
python-dateutil==2.7.5 # via alembic, arrow, botocore
python-editor==1.0.3 # via alembic
python-ldap==3.1.0
pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339
pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339
pyyaml==3.13 # via cloudflare
raven[flask]==6.9.0
redis==2.10.6 # via celery
@ -78,9 +78,9 @@ retrying==1.3.3
s3transfer==0.1.13 # via boto3
six==1.11.0
sqlalchemy-utils==0.33.6
sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlalchemy==1.2.13 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2
urllib3==1.24 # via requests
urllib3==1.24.1 # via requests
vine==1.1.4 # via amqp
werkzeug==0.14.1 # via flask
xmltodict==0.11.0