Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #1253 from Netflix/quicker_count 

Sort and page
This commit is contained in:
Curtis 2018-05-25 11:04:05 -07:00 committed by GitHub
parent 2d61200a05 5e3add0b81
commit c5e7e5ab68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 49 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
lemur/database.py
View File

@ -10,7 +10,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com> .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
""" """
from inflection import underscore from inflection import underscore
from sqlalchemy import exc from sqlalchemy import exc, func
from sqlalchemy.sql import and_, or_ from sqlalchemy.sql import and_, or_
from sqlalchemy.orm import make_transient from sqlalchemy.orm import make_transient
@ -267,6 +267,17 @@ def clone(model):
return model return model
def get_count(q):
"""
Count the number of rows in a table. More efficient than count(*)
:param q:
:return:
"""
count_q = q.statement.with_only_columns([func.count()]).order_by(None)
count = q.session.execute(count_q).scalar()
return count
def sort_and_page(query, model, args): def sort_and_page(query, model, args):
""" """
Helper that allows us to combine sorting and paging Helper that allows us to combine sorting and paging
@ -289,7 +300,7 @@ def sort_and_page(query, model, args):
if sort_by and sort_dir: if sort_by and sort_dir:
query = sort(query, model, sort_by, sort_dir) query = sort(query, model, sort_by, sort_dir)
total = query.count() total = get_count(query)
# offset calculated at zero # offset calculated at zero
page -= 1 page -= 1

12
requirements-dev.txt
View File

@ -4,26 +4,26 @@
# #
# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
# #
aspy.yaml==1.1.0 # via pre-commit aspy.yaml==1.1.1 # via pre-commit
cached-property==1.4.2 # via pre-commit cached-property==1.4.2 # via pre-commit
certifi==2018.4.16 # via requests certifi==2018.4.16 # via requests
cfgv==1.0.0 # via pre-commit cfgv==1.0.0 # via pre-commit
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
flake8==3.5.0 flake8==3.5.0
identify==1.0.13 # via pre-commit identify==1.0.17 # via pre-commit
idna==2.6 # via requests idna==2.6 # via requests
invoke==0.22.1 invoke==1.0.0
mccabe==0.6.1 # via flake8 mccabe==0.6.1 # via flake8
nodeenv==1.3.0 nodeenv==1.3.0
pkginfo==1.4.2 # via twine pkginfo==1.4.2 # via twine
pre-commit==1.8.2 pre-commit==1.9.0
pycodestyle==2.3.1 # via flake8 pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8 pyflakes==1.6.0 # via flake8
pyyaml==3.12 # via aspy.yaml, pre-commit pyyaml==3.12 # via aspy.yaml, pre-commit
requests-toolbelt==0.8.0 # via twine requests-toolbelt==0.8.0 # via twine
requests==2.18.4 # via requests-toolbelt, twine requests==2.18.4 # via requests-toolbelt, twine
six==1.11.0 # via cfgv, pre-commit six==1.11.0 # via cfgv, pre-commit
tqdm==4.23.0 # via twine tqdm==4.23.4 # via twine
twine==1.11.0 twine==1.11.0
urllib3==1.22 # via requests urllib3==1.22 # via requests
virtualenv==15.2.0 # via pre-commit virtualenv==16.0.0 # via pre-commit

4
requirements-docs.txt
View File

@ -68,8 +68,8 @@ retrying==1.3.3
s3transfer==0.1.13 s3transfer==0.1.13
six==1.11.0 six==1.11.0
snowballstemmer==1.2.1 # via sphinx snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.3.0 sphinx-rtd-theme==0.3.1
sphinx==1.7.3 sphinx==1.7.4
sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-httpdomain==1.6.1
sphinxcontrib-websupport==1.0.1 # via sphinx sphinxcontrib-websupport==1.0.1 # via sphinx
sqlalchemy-utils==0.33.2 sqlalchemy-utils==0.33.2

29
requirements-tests.txt
View File

@ -5,11 +5,12 @@
# pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in
# #
asn1crypto==0.24.0 # via cryptography asn1crypto==0.24.0 # via cryptography
attrs==17.4.0 # via pytest atomicwrites==1.1.5 # via pytest
attrs==18.1.0 # via pytest
aws-xray-sdk==0.95 # via moto aws-xray-sdk==0.95 # via moto
boto3==1.7.6 # via moto boto3==1.7.26 # via moto
boto==2.48.0 # via moto boto==2.48.0 # via moto
botocore==1.10.6 # via boto3, moto, s3transfer botocore==1.10.26 # via boto3, moto, s3transfer
certifi==2018.4.16 # via requests certifi==2018.4.16 # via requests
cffi==1.11.5 # via cryptography cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
@ -17,12 +18,12 @@ click==6.7 # via flask
cookies==2.2.1 # via moto, responses cookies==2.2.1 # via moto, responses
coverage==4.5.1 coverage==4.5.1
cryptography==2.2.2 # via moto cryptography==2.2.2 # via moto
docker-pycreds==0.2.2 # via docker docker-pycreds==0.2.3 # via docker
docker==3.2.1 # via moto docker==3.3.0 # via moto
docutils==0.14 # via botocore docutils==0.14 # via botocore
factory-boy==2.10.0 factory-boy==2.11.1
faker==0.8.13 faker==0.8.15
flask==0.12.2 # via pytest-flask flask==1.0.2 # via pytest-flask
freezegun==0.3.10 freezegun==0.3.10
idna==2.6 # via cryptography, requests idna==2.6 # via cryptography, requests
itsdangerous==0.24 # via flask itsdangerous==0.24 # via flask
@ -32,22 +33,22 @@ jsondiff==1.1.1 # via moto
jsonpickle==0.9.6 # via aws-xray-sdk jsonpickle==0.9.6 # via aws-xray-sdk
markupsafe==1.0 # via jinja2 markupsafe==1.0 # via jinja2
mock==2.0.0 # via moto mock==2.0.0 # via moto
more-itertools==4.1.0 # via pytest more-itertools==4.2.0 # via pytest
moto==1.3.3 moto==1.3.3
nose==1.3.7 nose==1.3.7
pbr==4.0.2 # via mock pbr==4.0.3 # via mock
pluggy==0.6.0 # via pytest pluggy==0.6.0 # via pytest
py==1.5.3 # via pytest py==1.5.3 # via pytest
pyaml==17.12.1 # via moto pyaml==17.12.1 # via moto
pycparser==2.18 # via cffi pycparser==2.18 # via cffi
pyflakes==1.6.0 pyflakes==2.0.0
pytest-flask==0.10.0 pytest-flask==0.10.0
pytest-mock==1.9.0 pytest-mock==1.10.0
pytest==3.5.0 pytest==3.6.0
python-dateutil==2.6.1 # via botocore, faker, freezegun, moto python-dateutil==2.6.1 # via botocore, faker, freezegun, moto
pytz==2018.4 # via moto pytz==2018.4 # via moto
pyyaml==3.12 # via pyaml pyyaml==3.12 # via pyaml
requests-mock==1.4.0 requests-mock==1.5.0
requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses
responses==0.9.0 # via moto responses==0.9.0 # via moto
s3transfer==0.1.13 # via boto3 s3transfer==0.1.13 # via boto3

26
requirements.txt
View File

@ -4,7 +4,7 @@
# #
# pip-compile --no-index --output-file requirements.txt requirements.in # pip-compile --no-index --output-file requirements.txt requirements.in
# #
acme==0.23.0 acme==0.24.0
alembic-autogenerate-enums==0.0.2 alembic-autogenerate-enums==0.0.2
alembic==0.9.9 # via flask-migrate alembic==0.9.9 # via flask-migrate
aniso8601==3.0.0 # via flask-restful aniso8601==3.0.0 # via flask-restful
@ -12,14 +12,14 @@ arrow==0.12.1
asn1crypto==0.24.0 # via cryptography asn1crypto==0.24.0 # via cryptography
bcrypt==3.1.4 # via flask-bcrypt, paramiko bcrypt==3.1.4 # via flask-bcrypt, paramiko
blinker==1.4 # via flask-mail, flask-principal, raven blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.7.6 boto3==1.7.26
botocore==1.10.6 # via boto3, s3transfer botocore==1.10.26 # via boto3, s3transfer
cffi==1.11.5 # via bcrypt, cryptography, pynacl cffi==1.11.5 # via bcrypt, cryptography, pynacl
click==6.7 # via flask click==6.7 # via flask
cryptography==2.2.2 cryptography==2.2.2
docutils==0.14 # via botocore docutils==0.14 # via botocore
flask-bcrypt==0.7.1 flask-bcrypt==0.7.1
flask-cors==3.0.3 flask-cors==3.0.4
flask-mail==0.9.1 flask-mail==0.9.1
flask-migrate==2.1.1 flask-migrate==2.1.1
flask-principal==0.4.0 flask-principal==0.4.0
@ -28,7 +28,7 @@ flask-script==2.0.6
flask-sqlalchemy==2.3.2 flask-sqlalchemy==2.3.2
flask==0.12 flask==0.12
future==0.16.0 future==0.16.0
gunicorn==19.7.1 gunicorn==19.8.1
idna==2.6 # via cryptography idna==2.6 # via cryptography
inflection==0.3.1 inflection==0.3.1
itsdangerous==0.24 # via flask itsdangerous==0.24 # via flask
@ -39,30 +39,30 @@ lockfile==0.12.2
mako==1.0.7 # via alembic mako==1.0.7 # via alembic
markupsafe==1.0 # via jinja2, mako markupsafe==1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.13.2 marshmallow-sqlalchemy==0.13.2
marshmallow==2.15.0 marshmallow==2.15.3
mock==2.0.0 # via acme mock==2.0.0 # via acme
ndg-httpsclient==0.4.4 ndg-httpsclient==0.5.0
paramiko==2.4.1 paramiko==2.4.1
pbr==4.0.2 # via mock pbr==4.0.3 # via mock
pem==17.1.0 pem==17.1.0
psycopg2==2.7.4 psycopg2==2.7.4
pyasn1-modules==0.2.1 # via python-ldap pyasn1-modules==0.2.1 # via python-ldap
pyasn1==0.4.2 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap, requests pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap, requests
pycparser==2.18 # via cffi pycparser==2.18 # via cffi
pyjwt==1.6.1 pyjwt==1.6.4
pynacl==1.2.1 # via paramiko pynacl==1.2.1 # via paramiko
pyopenssl==17.2.0 pyopenssl==17.2.0
pyrfc3339==1.0 # via acme pyrfc3339==1.0 # via acme
python-dateutil==2.7.2 # via alembic, arrow, botocore python-dateutil==2.7.3 # via alembic, arrow, botocore
python-editor==1.0.3 # via alembic python-editor==1.0.3 # via alembic
python-ldap==3.0.0 python-ldap==3.0.0
pytz==2018.4 # via acme, flask-restful, pyrfc3339 pytz==2018.4 # via acme, flask-restful, pyrfc3339
raven[flask]==6.7.0 raven[flask]==6.8.0
requests[security]==2.11.1 requests[security]==2.11.1
retrying==1.3.3 retrying==1.3.3
s3transfer==0.1.13 # via boto3 s3transfer==0.1.13 # via boto3
six==1.11.0 six==1.11.0
sqlalchemy-utils==0.33.2 sqlalchemy-utils==0.33.3
sqlalchemy==1.2.7 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils sqlalchemy==1.2.7 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2 tabulate==0.8.2
werkzeug==0.14.1 # via flask werkzeug==0.14.1 # via flask