Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2037 from castrapel/db_optimize_select 

Add new gin index to optimize ILIKE queries
This commit is contained in:
Curtis 2018-11-05 13:32:37 -08:00 committed by GitHub
parent baa73c7f3e b9f511ed02
commit 8e3f9a3c5a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 93 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -28,6 +28,7 @@ env:
before_script:
- psql -c "create database lemur;" -U postgres
- psql -c "create user lemur with password 'lemur;'" -U postgres
- psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
- npm config set registry https://registry.npmjs.org
- npm install -g bower
- pip install --upgrade setuptools
@ -45,4 +46,4 @@ after_success:
notifications:
email:
kglisson@netflix.com
ccastrapel@netflix.com

8
lemur/certificates/models.py
View File

@ -77,6 +77,14 @@ def get_or_increase_name(name, serial):
class Certificate(db.Model):
__tablename__ = 'certificates'
__table_args__ = (
Index('ix_certificates_cn', "cn",
postgresql_ops={"cn": "gin_trgm_ops"},
postgresql_using='gin'),
Index('ix_certificates_name', "name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
)
id = Column(Integer, primary_key=True)
ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
external_id = Column(String(128))

3
lemur/certificates/service.py
View File

@ -362,7 +362,8 @@ def render(args):
now = arrow.now().format('YYYY-MM-DD')
query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now)
return database.sort_and_page(query, Certificate, args)
result = database.sort_and_page(query, Certificate, args)
return result
def create_csr(**csr_config):

4
lemur/common/celery.py
View File

@ -193,8 +193,8 @@ def clean_source(source):
@celery.task()
def sync_all_sources():
"""
This function will sync certificates from all sources. This function triggers one celery task per source.
"""
This function will sync certificates from all sources. This function triggers one celery task per source.
"""
sources = validate_sources("all")
for source in sources:
current_app.logger.debug("Creating celery task to sync source {}".format(source.label))

7
lemur/domains/models.py
View File

@ -7,13 +7,18 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy import Column, Integer, String, Boolean
from sqlalchemy import Column, Integer, String, Boolean, Index
from lemur.database import db
class Domain(db.Model):
__tablename__ = 'domains'
__table_args__ = (
Index('ix_domains_name_gin', "name",
postgresql_ops={"name": "gin_trgm_ops"},
postgresql_using='gin'),
)
id = Column(Integer, primary_key=True)
name = Column(String(256), index=True)
sensitive = Column(Boolean, default=False)

31
lemur/migrations/versions/ee827d1e1974_.py Normal file
View File

@ -0,0 +1,31 @@
"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries.
Revision ID: ee827d1e1974
Revises: 7ead443ba911
Create Date: 2018-11-05 09:49:40.226368
"""
# revision identifiers, used by Alembic.
revision = 'ee827d1e1974'
down_revision = '7ead443ba911'
from alembic import op
from sqlalchemy.exc import ProgrammingError
def upgrade():
connection = op.get_bind()
connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'},
postgresql_using='gin')
op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False,
postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin')
op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'},
postgresql_using='gin')
def downgrade():
op.drop_index('ix_domains_name', table_name='domains')
op.drop_index('ix_certificates_name', table_name='certificates')
op.drop_index('ix_certificates_cn', table_name='certificates')

2
requirements-dev.in
View File

@ -1,6 +1,6 @@
# Run `make up-reqs` to update pinned dependencies in requirement text files
flake8>=3.2,<4.0
flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors.
pre-commit
invoke
twine

14
requirements-dev.txt
View File

@ -8,32 +8,30 @@ aspy.yaml==1.1.1 # via pre-commit
bleach==3.0.2 # via readme-renderer
cached-property==1.5.1 # via pre-commit
certifi==2018.10.15 # via requests
cffi==1.11.5 # via cmarkgfm
cfgv==1.1.0 # via pre-commit
chardet==3.0.4 # via requests
cmarkgfm==0.4.2 # via readme-renderer
docutils==0.14 # via readme-renderer
flake8==3.5.0
future==0.16.0 # via readme-renderer
identify==1.1.7 # via pre-commit
idna==2.7 # via requests
importlib-metadata==0.6 # via pre-commit
importlib-resources==1.0.2 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.2
pkginfo==1.4.2 # via twine
pre-commit==1.11.2
pre-commit==1.12.0
pycodestyle==2.3.1 # via flake8
pycparser==2.19 # via cffi
pyflakes==1.6.0 # via flake8
pygments==2.2.0 # via readme-renderer
pyyaml==3.13 # via aspy.yaml, pre-commit
readme-renderer==22.0 # via twine
readme-renderer==24.0 # via twine
requests-toolbelt==0.8.0 # via twine
requests==2.20.0 # via requests-toolbelt, twine
six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.28.1 # via twine
twine==1.12.1
urllib3==1.24 # via requests
virtualenv==16.0.0 # via pre-commit
urllib3==1.24.1 # via requests
virtualenv==16.1.0 # via pre-commit
webencodings==0.5.1 # via bleach

24
requirements-docs.txt
View File

@ -7,9 +7,9 @@
acme==0.27.1
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.1
alembic==1.0.2
amqp==2.3.2
aniso8601==3.0.2
aniso8601==4.0.1
arrow==0.12.1
asn1crypto==0.24.0
asyncpool==1.0
@ -39,7 +39,7 @@ flask-restful==0.3.6
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask==0.12
future==0.16.0
future==0.17.1
gunicorn==19.9.0
idna==2.7
imagesize==1.1.0 # via sphinx
@ -52,14 +52,14 @@ jsonlines==1.2.0
kombu==4.2.1
lockfile==0.12.2
mako==1.0.7
markupsafe==1.0
marshmallow-sqlalchemy==0.14.1
marshmallow==2.16.0
markupsafe==1.1.0
marshmallow-sqlalchemy==0.15.0
marshmallow==2.16.3
mock==2.0.0
ndg-httpsclient==0.5.1
packaging==18.0 # via sphinx
paramiko==2.4.2
pbr==5.0.0
pbr==5.1.0
pem==18.2.0
psycopg2==2.7.5
pyasn1-modules==0.2.2
@ -69,11 +69,11 @@ pygments==2.2.0 # via sphinx
pyjwt==1.6.4
pynacl==1.3.0
pyopenssl==18.0.0
pyparsing==2.2.2 # via packaging
pyparsing==2.3.0 # via packaging
pyrfc3339==1.1
python-dateutil==2.7.3
python-dateutil==2.7.5
python-editor==1.0.3
pytz==2018.5
pytz==2018.7
pyyaml==3.13
raven[flask]==6.9.0
redis==2.10.6
@ -88,9 +88,9 @@ sphinx==1.8.1
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx
sqlalchemy-utils==0.33.6
sqlalchemy==1.2.12
sqlalchemy==1.2.13
tabulate==0.8.2
urllib3==1.24
urllib3==1.24.1
vine==1.1.4
werkzeug==0.14.1
xmltodict==0.11.0

25
requirements-tests.txt
View File

@ -8,10 +8,9 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.2.1 # via pytest
attrs==18.2.0 # via pytest
aws-xray-sdk==0.95 # via moto
biscuits==0.1.1 # via responses
boto3==1.9.28 # via moto
boto3==1.9.37 # via moto
boto==2.49.0 # via moto
botocore==1.12.28 # via boto3, moto, s3transfer
botocore==1.12.37 # via boto3, moto, s3transfer
certifi==2018.10.15 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
@ -27,40 +26,40 @@ factory-boy==2.11.1
faker==0.9.2
flask==1.0.2 # via pytest-flask
freezegun==0.3.11
future==0.16.0 # via python-jose
future==0.17.1 # via python-jose
idna==2.7 # via cryptography, requests
itsdangerous==1.1.0 # via flask
jinja2==2.10 # via flask, moto
jmespath==0.9.3 # via boto3, botocore
jsondiff==1.1.1 # via moto
jsonpickle==1.0 # via aws-xray-sdk
markupsafe==1.0 # via jinja2
markupsafe==1.1.0 # via jinja2
mock==2.0.0 # via moto
more-itertools==4.3.0 # via pytest
moto==1.3.4
nose==1.3.7
pbr==5.0.0 # via mock
pbr==5.1.0 # via mock
pluggy==0.8.0 # via pytest
py==1.7.0 # via pytest
pyaml==17.12.1 # via moto
pycparser==2.19 # via cffi
pycryptodome==3.6.6 # via python-jose
pycryptodome==3.7.0 # via python-jose
pyflakes==2.0.0
pytest-flask==0.14.0
pytest-mock==1.10.0
pytest==3.9.1
python-dateutil==2.7.3 # via botocore, faker, freezegun, moto
pytest==3.10.0
python-dateutil==2.7.5 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.5 # via moto
pytz==2018.7 # via moto
pyyaml==3.13 # via pyaml
requests-mock==1.5.2
requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses
responses==0.10.1 # via moto
responses==0.10.2 # via moto
s3transfer==0.1.13 # via boto3
six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker
urllib3==1.23 # via botocore, requests
websocket-client==0.53.0 # via docker
urllib3==1.24.1 # via botocore, requests
websocket-client==0.54.0 # via docker
werkzeug==0.14.1 # via flask, moto, pytest-flask
wrapt==1.10.11 # via aws-xray-sdk
xmltodict==0.11.0 # via moto

22
requirements.txt
View File

@ -6,9 +6,9 @@
#
acme==0.27.1
alembic-autogenerate-enums==0.0.2
alembic==1.0.1 # via flask-migrate
alembic==1.0.2 # via flask-migrate
amqp==2.3.2 # via kombu
aniso8601==3.0.2 # via flask-restful
aniso8601==4.0.1 # via flask-restful
arrow==0.12.1
asn1crypto==0.24.0 # via cryptography
asyncpool==1.0
@ -37,7 +37,7 @@ flask-restful==0.3.6
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask==0.12
future==0.16.0
future==0.17.1
gunicorn==19.9.0
idna==2.7 # via cryptography, requests
inflection==0.3.1
@ -49,13 +49,13 @@ jsonlines==1.2.0 # via cloudflare
kombu==4.2.1 # via celery
lockfile==0.12.2
mako==1.0.7 # via alembic
markupsafe==1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.14.1
marshmallow==2.16.0
markupsafe==1.1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.15.0
marshmallow==2.16.3
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.0.0 # via mock
pbr==5.1.0 # via mock
pem==18.2.0
psycopg2==2.7.5
pyasn1-modules==0.2.2 # via python-ldap
@ -65,10 +65,10 @@ pyjwt==1.6.4
pynacl==1.3.0 # via paramiko
pyopenssl==18.0.0
pyrfc3339==1.1 # via acme
python-dateutil==2.7.3 # via alembic, arrow, botocore
python-dateutil==2.7.5 # via alembic, arrow, botocore
python-editor==1.0.3 # via alembic
python-ldap==3.1.0
pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339
pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339
pyyaml==3.13 # via cloudflare
raven[flask]==6.9.0
redis==2.10.6 # via celery
@ -78,9 +78,9 @@ retrying==1.3.3
s3transfer==0.1.13 # via boto3
six==1.11.0
sqlalchemy-utils==0.33.6
sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlalchemy==1.2.13 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2
urllib3==1.24 # via requests
urllib3==1.24.1 # via requests
vine==1.1.4 # via amqp
werkzeug==0.14.1 # via flask
xmltodict==0.11.0