diff --git a/.travis.yml b/.travis.yml index 37ec1434..b540937d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ env: before_script: - psql -c "create database lemur;" -U postgres - psql -c "create user lemur with password 'lemur;'" -U postgres + - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres - npm config set registry https://registry.npmjs.org - npm install -g bower - pip install --upgrade setuptools @@ -45,4 +46,4 @@ after_success: notifications: email: - kglisson@netflix.com + ccastrapel@netflix.com diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 6b438c06..7a1706f4 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -77,6 +77,14 @@ def get_or_increase_name(name, serial): class Certificate(db.Model): __tablename__ = 'certificates' + __table_args__ = ( + Index('ix_certificates_cn', "cn", + postgresql_ops={"cn": "gin_trgm_ops"}, + postgresql_using='gin'), + Index('ix_certificates_name', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True) external_id = Column(String(128)) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index fd6142ee..8fc031c4 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -362,7 +362,8 @@ def render(args): now = arrow.now().format('YYYY-MM-DD') query = query.filter(Certificate.not_after <= to).filter(Certificate.not_after >= now) - return database.sort_and_page(query, Certificate, args) + result = database.sort_and_page(query, Certificate, args) + return result def create_csr(**csr_config): diff --git a/lemur/common/celery.py b/lemur/common/celery.py index 1711b452..69bd9ce1 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -193,8 +193,8 @@ def clean_source(source): @celery.task() def sync_all_sources(): """ - This function will sync certificates from all sources. This function triggers one celery task per source. - """ + This function will sync certificates from all sources. This function triggers one celery task per source. + """ sources = validate_sources("all") for source in sources: current_app.logger.debug("Creating celery task to sync source {}".format(source.label)) diff --git a/lemur/domains/models.py b/lemur/domains/models.py index d0c7fef4..05fccd9c 100644 --- a/lemur/domains/models.py +++ b/lemur/domains/models.py @@ -7,13 +7,18 @@ .. moduleauthor:: Kevin Glisson """ -from sqlalchemy import Column, Integer, String, Boolean +from sqlalchemy import Column, Integer, String, Boolean, Index from lemur.database import db class Domain(db.Model): __tablename__ = 'domains' + __table_args__ = ( + Index('ix_domains_name_gin', "name", + postgresql_ops={"name": "gin_trgm_ops"}, + postgresql_using='gin'), + ) id = Column(Integer, primary_key=True) name = Column(String(256), index=True) sensitive = Column(Boolean, default=False) diff --git a/lemur/migrations/versions/ee827d1e1974_.py b/lemur/migrations/versions/ee827d1e1974_.py new file mode 100644 index 00000000..62ac6222 --- /dev/null +++ b/lemur/migrations/versions/ee827d1e1974_.py @@ -0,0 +1,31 @@ +"""Add pg_trgm indexes on certain attributes used for CN / Name filtering in ILIKE queries. + +Revision ID: ee827d1e1974 +Revises: 7ead443ba911 +Create Date: 2018-11-05 09:49:40.226368 + +""" + +# revision identifiers, used by Alembic. +revision = 'ee827d1e1974' +down_revision = '7ead443ba911' + +from alembic import op +from sqlalchemy.exc import ProgrammingError + +def upgrade(): + connection = op.get_bind() + connection.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") + + op.create_index('ix_certificates_cn', 'certificates', ['cn'], unique=False, postgresql_ops={'cn': 'gin_trgm_ops'}, + postgresql_using='gin') + op.create_index('ix_certificates_name', 'certificates', ['name'], unique=False, + postgresql_ops={'name': 'gin_trgm_ops'}, postgresql_using='gin') + op.create_index('ix_domains_name_gin', 'domains', ['name'], unique=False, postgresql_ops={'name': 'gin_trgm_ops'}, + postgresql_using='gin') + + +def downgrade(): + op.drop_index('ix_domains_name', table_name='domains') + op.drop_index('ix_certificates_name', table_name='certificates') + op.drop_index('ix_certificates_cn', table_name='certificates') diff --git a/requirements-dev.in b/requirements-dev.in index de8b60d3..84104679 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,6 @@ # Run `make up-reqs` to update pinned dependencies in requirement text files -flake8>=3.2,<4.0 +flake8==3.5.0 # flake8 3.6.0 is giving erroneous "W605 invalid escape sequence" errors. pre-commit invoke twine diff --git a/requirements-dev.txt b/requirements-dev.txt index c473aa56..1ded25a2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,32 +8,30 @@ aspy.yaml==1.1.1 # via pre-commit bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.10.15 # via requests -cffi==1.11.5 # via cmarkgfm cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests -cmarkgfm==0.4.2 # via readme-renderer docutils==0.14 # via readme-renderer flake8==3.5.0 -future==0.16.0 # via readme-renderer identify==1.1.7 # via pre-commit idna==2.7 # via requests +importlib-metadata==0.6 # via pre-commit +importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine -pre-commit==1.11.2 +pre-commit==1.12.0 pycodestyle==2.3.1 # via flake8 -pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 pygments==2.2.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit -readme-renderer==22.0 # via twine +readme-renderer==24.0 # via twine requests-toolbelt==0.8.0 # via twine requests==2.20.0 # via requests-toolbelt, twine six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.28.1 # via twine twine==1.12.1 -urllib3==1.24 # via requests -virtualenv==16.0.0 # via pre-commit +urllib3==1.24.1 # via requests +virtualenv==16.1.0 # via pre-commit webencodings==0.5.1 # via bleach diff --git a/requirements-docs.txt b/requirements-docs.txt index 812a947f..8937be9e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -7,9 +7,9 @@ acme==0.27.1 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.1 +alembic==1.0.2 amqp==2.3.2 -aniso8601==3.0.2 +aniso8601==4.0.1 arrow==0.12.1 asn1crypto==0.24.0 asyncpool==1.0 @@ -39,7 +39,7 @@ flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 -future==0.16.0 +future==0.17.1 gunicorn==19.9.0 idna==2.7 imagesize==1.1.0 # via sphinx @@ -52,14 +52,14 @@ jsonlines==1.2.0 kombu==4.2.1 lockfile==0.12.2 mako==1.0.7 -markupsafe==1.0 -marshmallow-sqlalchemy==0.14.1 -marshmallow==2.16.0 +markupsafe==1.1.0 +marshmallow-sqlalchemy==0.15.0 +marshmallow==2.16.3 mock==2.0.0 ndg-httpsclient==0.5.1 packaging==18.0 # via sphinx paramiko==2.4.2 -pbr==5.0.0 +pbr==5.1.0 pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 @@ -69,11 +69,11 @@ pygments==2.2.0 # via sphinx pyjwt==1.6.4 pynacl==1.3.0 pyopenssl==18.0.0 -pyparsing==2.2.2 # via packaging +pyparsing==2.3.0 # via packaging pyrfc3339==1.1 -python-dateutil==2.7.3 +python-dateutil==2.7.5 python-editor==1.0.3 -pytz==2018.5 +pytz==2018.7 pyyaml==3.13 raven[flask]==6.9.0 redis==2.10.6 @@ -88,9 +88,9 @@ sphinx==1.8.1 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 +sqlalchemy==1.2.13 tabulate==0.8.2 -urllib3==1.24 +urllib3==1.24.1 vine==1.1.4 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 2ad412db..64875903 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,10 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -biscuits==0.1.1 # via responses -boto3==1.9.28 # via moto +boto3==1.9.37 # via moto boto==2.49.0 # via moto -botocore==1.12.28 # via boto3, moto, s3transfer +botocore==1.12.37 # via boto3, moto, s3transfer certifi==2018.10.15 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -27,40 +26,40 @@ factory-boy==2.11.1 faker==0.9.2 flask==1.0.2 # via pytest-flask freezegun==0.3.11 -future==0.16.0 # via python-jose +future==0.17.1 # via python-jose idna==2.7 # via cryptography, requests itsdangerous==1.1.0 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto jsonpickle==1.0 # via aws-xray-sdk -markupsafe==1.0 # via jinja2 +markupsafe==1.1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.3.0 # via pytest moto==1.3.4 nose==1.3.7 -pbr==5.0.0 # via mock +pbr==5.1.0 # via mock pluggy==0.8.0 # via pytest py==1.7.0 # via pytest pyaml==17.12.1 # via moto pycparser==2.19 # via cffi -pycryptodome==3.6.6 # via python-jose +pycryptodome==3.7.0 # via python-jose pyflakes==2.0.0 pytest-flask==0.14.0 pytest-mock==1.10.0 -pytest==3.9.1 -python-dateutil==2.7.3 # via botocore, faker, freezegun, moto +pytest==3.10.0 +python-dateutil==2.7.5 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto -pytz==2018.5 # via moto +pytz==2018.7 # via moto pyyaml==3.13 # via pyaml requests-mock==1.5.2 requests==2.20.0 # via aws-xray-sdk, docker, moto, requests-mock, responses -responses==0.10.1 # via moto +responses==0.10.2 # via moto s3transfer==0.1.13 # via boto3 six==1.11.0 # via cryptography, docker, docker-pycreds, faker, freezegun, mock, more-itertools, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client text-unidecode==1.2 # via faker -urllib3==1.23 # via botocore, requests -websocket-client==0.53.0 # via docker +urllib3==1.24.1 # via botocore, requests +websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask wrapt==1.10.11 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.txt b/requirements.txt index ffe6372f..51d93b09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,9 @@ # acme==0.27.1 alembic-autogenerate-enums==0.0.2 -alembic==1.0.1 # via flask-migrate +alembic==1.0.2 # via flask-migrate amqp==2.3.2 # via kombu -aniso8601==3.0.2 # via flask-restful +aniso8601==4.0.1 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 @@ -37,7 +37,7 @@ flask-restful==0.3.6 flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 -future==0.16.0 +future==0.17.1 gunicorn==19.9.0 idna==2.7 # via cryptography, requests inflection==0.3.1 @@ -49,13 +49,13 @@ jsonlines==1.2.0 # via cloudflare kombu==4.2.1 # via celery lockfile==0.12.2 mako==1.0.7 # via alembic -markupsafe==1.0 # via jinja2, mako -marshmallow-sqlalchemy==0.14.1 -marshmallow==2.16.0 +markupsafe==1.1.0 # via jinja2, mako +marshmallow-sqlalchemy==0.15.0 +marshmallow==2.16.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.2 -pbr==5.0.0 # via mock +pbr==5.1.0 # via mock pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap @@ -65,10 +65,10 @@ pyjwt==1.6.4 pynacl==1.3.0 # via paramiko pyopenssl==18.0.0 pyrfc3339==1.1 # via acme -python-dateutil==2.7.3 # via alembic, arrow, botocore +python-dateutil==2.7.5 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 -pytz==2018.5 # via acme, celery, flask-restful, pyrfc3339 +pytz==2018.7 # via acme, celery, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 redis==2.10.6 # via celery @@ -78,9 +78,9 @@ retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 sqlalchemy-utils==0.33.6 -sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.13 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 -urllib3==1.24 # via requests +urllib3==1.24.1 # via requests vine==1.1.4 # via amqp werkzeug==0.14.1 # via flask xmltodict==0.11.0