From e91d8ec81b5ecb1569ad0a2aa32ee5dd7fbcce82 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 11 Oct 2018 11:36:50 -0700 Subject: [PATCH] add indexes to domains and certificates tables to optimize load time --- lemur/certificates/models.py | 30 ++++++++-------------- lemur/domains/models.py | 2 +- lemur/migrations/versions/c87cb989af04_.py | 19 ++++++++++++++ lemur/migrations/versions/f2383bf08fbc_.py | 23 +++++++++++++++++ requirements-dev.txt | 19 ++++++++++---- requirements-docs.txt | 12 ++++----- requirements-tests.txt | 16 ++++++------ requirements.txt | 18 ++++++------- 8 files changed, 91 insertions(+), 48 deletions(-) create mode 100644 lemur/migrations/versions/c87cb989af04_.py create mode 100644 lemur/migrations/versions/f2383bf08fbc_.py diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index b337c755..5e0c42cc 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -5,41 +5,32 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ -import arrow from datetime import timedelta -from flask import current_app - +import arrow from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import rsa - +from flask import current_app from idna.core import InvalidCodepoint - +from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean, Index +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import case, extract -from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean - from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property -from lemur.database import db -from lemur.extensions import sentry - -from lemur.utils import Vault from lemur.common import defaults, utils - -from lemur.plugins.base import plugins - -from lemur.extensions import metrics from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS - +from lemur.database import db +from lemur.domains.models import Domain +from lemur.extensions import metrics +from lemur.extensions import sentry from lemur.models import certificate_associations, certificate_source_associations, \ certificate_destination_associations, certificate_notification_associations, \ certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations - -from lemur.domains.models import Domain +from lemur.plugins.base import plugins from lemur.policies.models import RotationPolicy +from lemur.utils import Vault def get_sequence(name): @@ -87,6 +78,7 @@ def get_or_increase_name(name, serial): class Certificate(db.Model): __tablename__ = 'certificates' id = Column(Integer, primary_key=True) + ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True) external_id = Column(String(128)) owner = Column(String(128), nullable=False) name = Column(String(256), unique=True) diff --git a/lemur/domains/models.py b/lemur/domains/models.py index afd348d6..d0c7fef4 100644 --- a/lemur/domains/models.py +++ b/lemur/domains/models.py @@ -15,7 +15,7 @@ from lemur.database import db class Domain(db.Model): __tablename__ = 'domains' id = Column(Integer, primary_key=True) - name = Column(String(256)) + name = Column(String(256), index=True) sensitive = Column(Boolean, default=False) def __repr__(self): diff --git a/lemur/migrations/versions/c87cb989af04_.py b/lemur/migrations/versions/c87cb989af04_.py new file mode 100644 index 00000000..4959e727 --- /dev/null +++ b/lemur/migrations/versions/c87cb989af04_.py @@ -0,0 +1,19 @@ +"""Create an index on the domains table for the domain name +Revision ID: c87cb989af04 +Revises: 9392b9f9a805 +Create Date: 2018-10-11 09:44:57.099854 + +""" + +revision = 'c87cb989af04' +down_revision = '9392b9f9a805' + +from alembic import op + + +def upgrade(): + op.create_index(op.f('ix_domains_name'), 'domains', ['name'], unique=False) + + +def downgrade(): + op.drop_index(op.f('ix_domains_name'), table_name='domains') diff --git a/lemur/migrations/versions/f2383bf08fbc_.py b/lemur/migrations/versions/f2383bf08fbc_.py new file mode 100644 index 00000000..1fa36960 --- /dev/null +++ b/lemur/migrations/versions/f2383bf08fbc_.py @@ -0,0 +1,23 @@ +"""Create index on certificates table for id desc + + +Revision ID: f2383bf08fbc +Revises: c87cb989af04 +Create Date: 2018-10-11 11:23:31.195471 + +""" + +revision = 'f2383bf08fbc' +down_revision = 'c87cb989af04' + +import sqlalchemy as sa +from alembic import op + + +def upgrade(): + op.create_index('ix_certificates_id_desc', 'certificates', [sa.text('id DESC')], unique=True, + postgresql_using='btree') + + +def downgrade(): + op.drop_index('ix_certificates_id_desc', table_name='certificates') diff --git a/requirements-dev.txt b/requirements-dev.txt index eb027066..279724af 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,26 +5,35 @@ # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # aspy.yaml==1.1.1 # via pre-commit +bleach==3.0.2 # via readme-renderer cached-property==1.5.1 # via pre-commit certifi==2018.8.24 # via requests +cffi==1.11.5 # via cmarkgfm cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests +cmarkgfm==0.4.2 # via readme-renderer +docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.1.6 # via pre-commit +future==0.16.0 # via readme-renderer +identify==1.1.7 # via pre-commit idna==2.7 # via requests invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine -pre-commit==1.11.0 +pre-commit==1.11.2 pycodestyle==2.3.1 # via flake8 +pycparser==2.19 # via cffi pyflakes==1.6.0 # via flake8 +pygments==2.2.0 # via readme-renderer pyyaml==3.13 # via aspy.yaml, pre-commit +readme-renderer==22.0 # via twine requests-toolbelt==0.8.0 # via twine requests==2.19.1 # via requests-toolbelt, twine -six==1.11.0 # via cfgv, pre-commit -toml==0.9.6 # via pre-commit +six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer +toml==0.10.0 # via pre-commit tqdm==4.26.0 # via twine -twine==1.11.0 +twine==1.12.1 urllib3==1.23 # via requests virtualenv==16.0.0 # via pre-commit +webencodings==0.5.1 # via bleach diff --git a/requirements-docs.txt b/requirements-docs.txt index e00030c6..b82b2f23 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -5,7 +5,7 @@ # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # acme==0.27.1 -alabaster==0.7.11 # via sphinx +alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.0 amqp==2.3.2 @@ -54,10 +54,10 @@ lockfile==0.12.2 mako==1.0.7 markupsafe==1.0 marshmallow-sqlalchemy==0.14.1 -marshmallow==2.15.4 +marshmallow==2.15.5 mock==2.0.0 ndg-httpsclient==0.5.1 -packaging==17.1 # via sphinx +packaging==18.0 # via sphinx paramiko==2.4.1 pbr==4.2.0 pem==18.1.0 @@ -69,7 +69,7 @@ pygments==2.2.0 # via sphinx pyjwt==1.6.4 pynacl==1.2.1 pyopenssl==18.0.0 -pyparsing==2.2.0 # via packaging +pyparsing==2.2.2 # via packaging pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 @@ -83,8 +83,8 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.4.1 -sphinx==1.8.0 +sphinx-rtd-theme==0.4.2 +sphinx==1.8.1 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.4 diff --git a/requirements-tests.txt b/requirements-tests.txt index 2ed9ae2f..d9807539 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,13 +8,13 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.4 # via moto +boto3==1.9.21 # via moto boto==2.49.0 # via moto -botocore==1.12.4 # via boto3, moto, s3transfer +botocore==1.12.21 # via boto3, moto, s3transfer certifi==2018.8.24 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests -click==6.7 # via flask +click==7.0 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.3.1 # via moto @@ -32,22 +32,22 @@ itsdangerous==0.24 # via flask jinja2==2.10 # via flask, moto jmespath==0.9.3 # via boto3, botocore jsondiff==1.1.1 # via moto -jsonpickle==0.9.6 # via aws-xray-sdk +jsonpickle==1.0 # via aws-xray-sdk markupsafe==1.0 # via jinja2 mock==2.0.0 # via moto more-itertools==4.3.0 # via pytest moto==1.3.4 nose==1.3.7 -pbr==4.2.0 # via mock +pbr==4.3.0 # via mock pluggy==0.7.1 # via pytest py==1.6.0 # via pytest pyaml==17.12.1 # via moto -pycparser==2.18 # via cffi +pycparser==2.19 # via cffi pycryptodome==3.6.6 # via python-jose pyflakes==2.0.0 -pytest-flask==0.12.0 +pytest-flask==0.13.0 pytest-mock==1.10.0 -pytest==3.8.0 +pytest==3.8.2 python-dateutil==2.7.3 # via botocore, faker, freezegun, moto python-jose==2.0.2 # via moto pytz==2018.5 # via moto diff --git a/requirements.txt b/requirements.txt index 9b44bbbe..0d093ced 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ celery[redis]==4.2.1 certifi==2018.8.24 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -click==6.7 # via flask +click==7.0 # via flask cloudflare==2.1.0 cryptography==2.3.1 dnspython3==1.15.0 @@ -51,18 +51,18 @@ lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.0 # via jinja2, mako marshmallow-sqlalchemy==0.14.1 -marshmallow==2.15.5 +marshmallow==2.16.0 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 -paramiko==2.4.1 -pbr==4.2.0 # via mock -pem==18.1.0 +paramiko==2.4.2 +pbr==4.3.0 # via mock +pem==18.2.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap -pycparser==2.18 # via cffi +pycparser==2.19 # via cffi pyjwt==1.6.4 -pynacl==1.2.1 # via paramiko +pynacl==1.3.0 # via paramiko pyopenssl==18.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore @@ -77,8 +77,8 @@ requests[security]==2.19.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.4 -sqlalchemy==1.2.11 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.33.5 +sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 urllib3==1.23 # via requests vine==1.1.4 # via amqp