Merge pull request #1823 from castrapel/indexes_domains_certs

add indexes to domains and certificates tables to optimize load time

lemur/certificates/models.py

@@ -5,41 +5,32 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import arrow
 from datetime import timedelta
 
-from flask import current_app
-
+import arrow
 from cryptography import x509
 from cryptography.hazmat.primitives.asymmetric import rsa
-
+from flask import current_app
 from idna.core import InvalidCodepoint
-
+from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean, Index
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import case, extract
-from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
-
 from sqlalchemy_utils.types.arrow import ArrowType
 from werkzeug.utils import cached_property
 
-from lemur.database import db
-from lemur.extensions import sentry
-
-from lemur.utils import Vault
 from lemur.common import defaults, utils
-
-from lemur.plugins.base import plugins
-
-from lemur.extensions import metrics
 from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
-
+from lemur.database import db
+from lemur.domains.models import Domain
+from lemur.extensions import metrics
+from lemur.extensions import sentry
 from lemur.models import certificate_associations, certificate_source_associations, \
     certificate_destination_associations, certificate_notification_associations, \
     certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations
-
-from lemur.domains.models import Domain
+from lemur.plugins.base import plugins
 from lemur.policies.models import RotationPolicy
+from lemur.utils import Vault
 
 
 def get_sequence(name):
@@ -87,6 +78,7 @@ def get_or_increase_name(name, serial):
 class Certificate(db.Model):
     __tablename__ = 'certificates'
     id = Column(Integer, primary_key=True)
+    ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
     external_id = Column(String(128))
     owner = Column(String(128), nullable=False)
     name = Column(String(256), unique=True)

lemur/domains/models.py

@@ -15,7 +15,7 @@ from lemur.database import db
 class Domain(db.Model):
     __tablename__ = 'domains'
     id = Column(Integer, primary_key=True)
-    name = Column(String(256))
+    name = Column(String(256), index=True)
     sensitive = Column(Boolean, default=False)
 
     def __repr__(self):

lemur/migrations/versions/c87cb989af04_.py

@@ -0,0 +1,19 @@
+"""Create an index on the domains table for the domain name
+Revision ID: c87cb989af04
+Revises: 9392b9f9a805
+Create Date: 2018-10-11 09:44:57.099854
+
+"""
+
+revision = 'c87cb989af04'
+down_revision = '9392b9f9a805'
+
+from alembic import op
+
+
+def upgrade():
+    op.create_index(op.f('ix_domains_name'), 'domains', ['name'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_domains_name'), table_name='domains')

lemur/migrations/versions/f2383bf08fbc_.py

@@ -0,0 +1,23 @@
+"""Create index on certificates table for id desc
+
+
+Revision ID: f2383bf08fbc
+Revises: c87cb989af04
+Create Date: 2018-10-11 11:23:31.195471
+
+"""
+
+revision = 'f2383bf08fbc'
+down_revision = 'c87cb989af04'
+
+import sqlalchemy as sa
+from alembic import op
+
+
+def upgrade():
+    op.create_index('ix_certificates_id_desc', 'certificates', [sa.text('id DESC')], unique=True,
+                    postgresql_using='btree')
+
+
+def downgrade():
+    op.drop_index('ix_certificates_id_desc', table_name='certificates')

requirements-dev.txt

@@ -5,26 +5,35 @@
 #    pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
 #
 aspy.yaml==1.1.1          # via pre-commit
+bleach==3.0.2             # via readme-renderer
 cached-property==1.5.1    # via pre-commit
 certifi==2018.8.24        # via requests
+cffi==1.11.5              # via cmarkgfm
 cfgv==1.1.0               # via pre-commit
 chardet==3.0.4            # via requests
+cmarkgfm==0.4.2           # via readme-renderer
+docutils==0.14            # via readme-renderer
 flake8==3.5.0
-identify==1.1.6           # via pre-commit
+future==0.16.0            # via readme-renderer
+identify==1.1.7           # via pre-commit
 idna==2.7                 # via requests
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.2
 pkginfo==1.4.2            # via twine
-pre-commit==1.11.0
+pre-commit==1.11.2
 pycodestyle==2.3.1        # via flake8
+pycparser==2.19           # via cffi
 pyflakes==1.6.0           # via flake8
+pygments==2.2.0           # via readme-renderer
 pyyaml==3.13              # via aspy.yaml, pre-commit
+readme-renderer==22.0     # via twine
 requests-toolbelt==0.8.0  # via twine
 requests==2.19.1          # via requests-toolbelt, twine
-six==1.11.0               # via cfgv, pre-commit
-toml==0.9.6               # via pre-commit
+six==1.11.0               # via bleach, cfgv, pre-commit, readme-renderer
+toml==0.10.0              # via pre-commit
 tqdm==4.26.0              # via twine
-twine==1.11.0
+twine==1.12.1
 urllib3==1.23             # via requests
 virtualenv==16.0.0        # via pre-commit
+webencodings==0.5.1       # via bleach

requirements-docs.txt

@@ -5,7 +5,7 @@
 #    pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
 #
 acme==0.27.1
-alabaster==0.7.11         # via sphinx
+alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.0
 amqp==2.3.2
@@ -54,10 +54,10 @@ lockfile==0.12.2
 mako==1.0.7
 markupsafe==1.0
 marshmallow-sqlalchemy==0.14.1
-marshmallow==2.15.4
+marshmallow==2.15.5
 mock==2.0.0
 ndg-httpsclient==0.5.1
-packaging==17.1           # via sphinx
+packaging==18.0           # via sphinx
 paramiko==2.4.1
 pbr==4.2.0
 pem==18.1.0
@@ -69,7 +69,7 @@ pygments==2.2.0           # via sphinx
 pyjwt==1.6.4
 pynacl==1.2.1
 pyopenssl==18.0.0
-pyparsing==2.2.0          # via packaging
+pyparsing==2.2.2          # via packaging
 pyrfc3339==1.1
 python-dateutil==2.7.3
 python-editor==1.0.3
@@ -83,8 +83,8 @@ retrying==1.3.3
 s3transfer==0.1.13
 six==1.11.0
 snowballstemmer==1.2.1    # via sphinx
-sphinx-rtd-theme==0.4.1
-sphinx==1.8.0
+sphinx-rtd-theme==0.4.2
+sphinx==1.8.1
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx
 sqlalchemy-utils==0.33.4

requirements-tests.txt

@@ -8,13 +8,13 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.9.4              # via moto
+boto3==1.9.21             # via moto
 boto==2.49.0              # via moto
-botocore==1.12.4          # via boto3, moto, s3transfer
+botocore==1.12.21         # via boto3, moto, s3transfer
 certifi==2018.8.24        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
-click==6.7                # via flask
+click==7.0                # via flask
 cookies==2.2.1            # via moto, responses
 coverage==4.5.1
 cryptography==2.3.1       # via moto
@@ -32,22 +32,22 @@ itsdangerous==0.24        # via flask
 jinja2==2.10              # via flask, moto
 jmespath==0.9.3           # via boto3, botocore
 jsondiff==1.1.1           # via moto
-jsonpickle==0.9.6         # via aws-xray-sdk
+jsonpickle==1.0           # via aws-xray-sdk
 markupsafe==1.0           # via jinja2
 mock==2.0.0               # via moto
 more-itertools==4.3.0     # via pytest
 moto==1.3.4
 nose==1.3.7
-pbr==4.2.0                # via mock
+pbr==4.3.0                # via mock
 pluggy==0.7.1             # via pytest
 py==1.6.0                 # via pytest
 pyaml==17.12.1            # via moto
-pycparser==2.18           # via cffi
+pycparser==2.19           # via cffi
 pycryptodome==3.6.6       # via python-jose
 pyflakes==2.0.0
-pytest-flask==0.12.0
+pytest-flask==0.13.0
 pytest-mock==1.10.0
-pytest==3.8.0
+pytest==3.8.2
 python-dateutil==2.7.3    # via botocore, faker, freezegun, moto
 python-jose==2.0.2        # via moto
 pytz==2018.5              # via moto

requirements.txt

@@ -21,7 +21,7 @@ celery[redis]==4.2.1
 certifi==2018.8.24
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
-click==6.7                # via flask
+click==7.0                # via flask
 cloudflare==2.1.0
 cryptography==2.3.1
 dnspython3==1.15.0
@@ -51,18 +51,18 @@ lockfile==0.12.2
 mako==1.0.7               # via alembic
 markupsafe==1.0           # via jinja2, mako
 marshmallow-sqlalchemy==0.14.1
-marshmallow==2.15.5
+marshmallow==2.16.0
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
-paramiko==2.4.1
-pbr==4.2.0                # via mock
-pem==18.1.0
+paramiko==2.4.2
+pbr==4.3.0                # via mock
+pem==18.2.0
 psycopg2==2.7.5
 pyasn1-modules==0.2.2     # via python-ldap
 pyasn1==0.4.4             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
-pycparser==2.18           # via cffi
+pycparser==2.19           # via cffi
 pyjwt==1.6.4
-pynacl==1.2.1             # via paramiko
+pynacl==1.3.0             # via paramiko
 pyopenssl==18.0.0
 pyrfc3339==1.1            # via acme
 python-dateutil==2.7.3    # via alembic, arrow, botocore
@@ -77,8 +77,8 @@ requests[security]==2.19.1
 retrying==1.3.3
 s3transfer==0.1.13        # via boto3
 six==1.11.0
-sqlalchemy-utils==0.33.4
-sqlalchemy==1.2.11        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy-utils==0.33.5
+sqlalchemy==1.2.12        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 tabulate==0.8.2
 urllib3==1.23             # via requests
 vine==1.1.4               # via amqp