From 9fc6c9aaf776f16532258f19d4bfbf07fb828f6c Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 24 May 2018 12:55:52 -0700 Subject: [PATCH 1/2] Sort and page --- lemur/database.py | 10 ++++++++-- requirements-dev.txt | 12 ++++++------ requirements-docs.txt | 4 ++-- requirements-tests.txt | 29 +++++++++++++++-------------- requirements.txt | 26 +++++++++++++------------- 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/lemur/database.py b/lemur/database.py index 9198c9ce..13e67710 100644 --- a/lemur/database.py +++ b/lemur/database.py @@ -10,7 +10,7 @@ .. moduleauthor:: Kevin Glisson """ from inflection import underscore -from sqlalchemy import exc +from sqlalchemy import exc, func from sqlalchemy.sql import and_, or_ from sqlalchemy.orm import make_transient @@ -267,6 +267,12 @@ def clone(model): return model +def get_count(q): + count_q = q.statement.with_only_columns([func.count()]).order_by(None) + count = q.session.execute(count_q).scalar() + return count + + def sort_and_page(query, model, args): """ Helper that allows us to combine sorting and paging @@ -289,7 +295,7 @@ def sort_and_page(query, model, args): if sort_by and sort_dir: query = sort(query, model, sort_by, sort_dir) - total = query.count() + total = get_count(query) # offset calculated at zero page -= 1 diff --git a/requirements-dev.txt b/requirements-dev.txt index 356db421..6a084947 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,26 +4,26 @@ # # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # -aspy.yaml==1.1.0 # via pre-commit +aspy.yaml==1.1.1 # via pre-commit cached-property==1.4.2 # via pre-commit certifi==2018.4.16 # via requests cfgv==1.0.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.13 # via pre-commit +identify==1.0.17 # via pre-commit idna==2.6 # via requests -invoke==0.22.1 +invoke==1.0.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.0 pkginfo==1.4.2 # via twine -pre-commit==1.8.2 +pre-commit==1.9.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.12 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.18.4 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit -tqdm==4.23.0 # via twine +tqdm==4.23.4 # via twine twine==1.11.0 urllib3==1.22 # via requests -virtualenv==15.2.0 # via pre-commit +virtualenv==16.0.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 9d388183..22b06f47 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -68,8 +68,8 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.3.0 -sphinx==1.7.3 +sphinx-rtd-theme==0.3.1 +sphinx==1.7.4 sphinxcontrib-httpdomain==1.6.1 sphinxcontrib-websupport==1.0.1 # via sphinx sqlalchemy-utils==0.33.2 diff --git a/requirements-tests.txt b/requirements-tests.txt index 0c74eacb..e4be0eff 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,11 +5,12 @@ # pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography -attrs==17.4.0 # via pytest +atomicwrites==1.1.5 # via pytest +attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.6 # via moto +boto3==1.7.26 # via moto boto==2.48.0 # via moto -botocore==1.10.6 # via boto3, moto, s3transfer +botocore==1.10.26 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -17,12 +18,12 @@ click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 cryptography==2.2.2 # via moto -docker-pycreds==0.2.2 # via docker -docker==3.2.1 # via moto +docker-pycreds==0.2.3 # via docker +docker==3.3.0 # via moto docutils==0.14 # via botocore -factory-boy==2.10.0 -faker==0.8.13 -flask==0.12.2 # via pytest-flask +factory-boy==2.11.1 +faker==0.8.15 +flask==1.0.2 # via pytest-flask freezegun==0.3.10 idna==2.6 # via cryptography, requests itsdangerous==0.24 # via flask @@ -32,22 +33,22 @@ jsondiff==1.1.1 # via moto jsonpickle==0.9.6 # via aws-xray-sdk markupsafe==1.0 # via jinja2 mock==2.0.0 # via moto -more-itertools==4.1.0 # via pytest +more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.0.2 # via mock +pbr==4.0.3 # via mock pluggy==0.6.0 # via pytest py==1.5.3 # via pytest pyaml==17.12.1 # via moto pycparser==2.18 # via cffi -pyflakes==1.6.0 +pyflakes==2.0.0 pytest-flask==0.10.0 -pytest-mock==1.9.0 -pytest==3.5.0 +pytest-mock==1.10.0 +pytest==3.6.0 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.4 # via moto pyyaml==3.12 # via pyaml -requests-mock==1.4.0 +requests-mock==1.5.0 requests==2.18.4 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 diff --git a/requirements.txt b/requirements.txt index 23f1f28f..2c986910 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.23.0 +acme==0.24.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.9 # via flask-migrate aniso8601==3.0.0 # via flask-restful @@ -12,14 +12,14 @@ arrow==0.12.1 asn1crypto==0.24.0 # via cryptography bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.6 -botocore==1.10.6 # via boto3, s3transfer +boto3==1.7.26 +botocore==1.10.26 # via boto3, s3transfer cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask cryptography==2.2.2 docutils==0.14 # via botocore flask-bcrypt==0.7.1 -flask-cors==3.0.3 +flask-cors==3.0.4 flask-mail==0.9.1 flask-migrate==2.1.1 flask-principal==0.4.0 @@ -28,7 +28,7 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 -gunicorn==19.7.1 +gunicorn==19.8.1 idna==2.6 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask @@ -39,30 +39,30 @@ lockfile==0.12.2 mako==1.0.7 # via alembic markupsafe==1.0 # via jinja2, mako marshmallow-sqlalchemy==0.13.2 -marshmallow==2.15.0 +marshmallow==2.15.3 mock==2.0.0 # via acme -ndg-httpsclient==0.4.4 +ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.0.2 # via mock +pbr==4.0.3 # via mock pem==17.1.0 psycopg2==2.7.4 pyasn1-modules==0.2.1 # via python-ldap -pyasn1==0.4.2 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap, requests +pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap, requests pycparser==2.18 # via cffi -pyjwt==1.6.1 +pyjwt==1.6.4 pynacl==1.2.1 # via paramiko pyopenssl==17.2.0 pyrfc3339==1.0 # via acme -python-dateutil==2.7.2 # via alembic, arrow, botocore +python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.0.0 pytz==2018.4 # via acme, flask-restful, pyrfc3339 -raven[flask]==6.7.0 +raven[flask]==6.8.0 requests[security]==2.11.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 -sqlalchemy-utils==0.33.2 +sqlalchemy-utils==0.33.3 sqlalchemy==1.2.7 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 werkzeug==0.14.1 # via flask From 5e3add0b81d8923d6c72dd649c1829ea57f4f5f1 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 24 May 2018 15:21:38 -0700 Subject: [PATCH 2/2] docstring --- lemur/database.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lemur/database.py b/lemur/database.py index 13e67710..8efd4b95 100644 --- a/lemur/database.py +++ b/lemur/database.py @@ -268,6 +268,11 @@ def clone(model): def get_count(q): + """ + Count the number of rows in a table. More efficient than count(*) + :param q: + :return: + """ count_q = q.statement.with_only_columns([func.count()]).order_by(None) count = q.session.execute(count_q).scalar() return count