diff --git a/docs/guide/create_dns_provider.png b/docs/production/create_dns_provider.png similarity index 100% rename from docs/guide/create_dns_provider.png rename to docs/production/create_dns_provider.png diff --git a/docs/guide/letsencrypt_authority_1.png b/docs/production/letsencrypt_authority_1.png similarity index 100% rename from docs/guide/letsencrypt_authority_1.png rename to docs/production/letsencrypt_authority_1.png diff --git a/docs/guide/letsencrypt_authority_2.png b/docs/production/letsencrypt_authority_2.png similarity index 100% rename from docs/guide/letsencrypt_authority_2.png rename to docs/production/letsencrypt_authority_2.png diff --git a/docs/guide/letsencrypt_flow.png b/docs/production/letsencrypt_flow.png similarity index 100% rename from docs/guide/letsencrypt_flow.png rename to docs/production/letsencrypt_flow.png diff --git a/lemur/common/celery.py b/lemur/common/celery.py index d3cc7621..67780957 100644 --- a/lemur/common/celery.py +++ b/lemur/common/celery.py @@ -9,6 +9,7 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B """ import copy import sys +import time from datetime import datetime, timezone, timedelta from celery import Celery @@ -16,6 +17,7 @@ from celery.exceptions import SoftTimeLimitExceeded from flask import current_app from lemur.authorities.service import get as get_authority +from lemur.common.redis import RedisHandler from lemur.destinations import service as destinations_service from lemur.extensions import metrics, sentry from lemur.factory import create_app @@ -30,6 +32,8 @@ if current_app: else: flask_app = create_app() +red = RedisHandler().redis() + def make_celery(app): celery = Celery( @@ -68,6 +72,30 @@ def is_task_active(fun, task_id, args): return False +@celery.task() +def report_celery_last_success_metrics(): + """ + For each celery task, this will determine the number of seconds since it has last been successful. + + Celery tasks should be emitting redis stats with a deterministic key (In our case, `f"{task}.last_success"`. + report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful. + Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted + from this function. + + """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" + current_time = int(time.time()) + schedule = current_app.config.get('CELERYBEAT_SCHEDULE') + for _, t in schedule.items(): + task = t.get("task") + last_success = int(red.get(f"{task}.last_success") or 0) + metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success) + red.set( + f"{function}.last_success", int(time.time()) + ) # Alert if this metric is not seen + metrics.send(f"{function}.success", 'counter', 1) + + @celery.task(soft_time_limit=600) def fetch_acme_cert(id): """ @@ -80,8 +108,9 @@ def fetch_acme_cert(id): if celery.current_task: task_id = celery.current_task.request.id + function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { - "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "function": function, "message": "Resolving pending certificate {}".format(id), "task_id": task_id, "id": id, @@ -165,11 +194,15 @@ def fetch_acme_cert(id): log_data["failed"] = failed log_data["wrong_issuer"] = wrong_issuer current_app.logger.debug(log_data) + metrics.send(f"{function}.resolved", 'gauge', new) + metrics.send(f"{function}.failed", 'gauge', failed) + metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( new=new, failed=failed, wrong_issuer=wrong_issuer ) ) + red.set(f'{function}.last_success', int(time.time())) @celery.task() @@ -177,8 +210,9 @@ def fetch_all_pending_acme_certs(): """Instantiate celery workers to resolve all pending Acme certificates""" pending_certs = pending_certificate_service.get_unresolved_pending_certs() + function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { - "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name), + "function": function, "message": "Starting job.", } @@ -195,11 +229,18 @@ def fetch_all_pending_acme_certs(): current_app.logger.debug(log_data) fetch_acme_cert.delay(cert.id) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task() def remove_old_acme_certs(): """Prune old pending acme certificates from the database""" - log_data = {"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)} + function = f"{__name__}.{sys._getframe().f_code.co_name}" + log_data = { + "function": function, + "message": "Starting job.", + } pending_certs = pending_certificate_service.get_pending_certs("all") # Delete pending certs more than a week old @@ -211,6 +252,9 @@ def remove_old_acme_certs(): current_app.logger.debug(log_data) pending_certificate_service.delete(cert) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task() def clean_all_sources(): @@ -218,6 +262,7 @@ def clean_all_sources(): This function will clean unused certificates from sources. This is a destructive operation and should only be ran periodically. This function triggers one celery task per source. """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" sources = validate_sources("all") for source in sources: current_app.logger.debug( @@ -225,6 +270,9 @@ def clean_all_sources(): ) clean_source.delay(source.label) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task() def clean_source(source): @@ -244,6 +292,7 @@ def sync_all_sources(): """ This function will sync certificates from all sources. This function triggers one celery task per source. """ + function = f"{__name__}.{sys._getframe().f_code.co_name}" sources = validate_sources("all") for source in sources: current_app.logger.debug( @@ -251,6 +300,9 @@ def sync_all_sources(): ) sync_source.delay(source.label) + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) + @celery.task(soft_time_limit=7200) def sync_source(source): @@ -261,7 +313,7 @@ def sync_source(source): :return: """ - function = "{}.{}".format(__name__, sys._getframe().f_code.co_name) + function = f"{__name__}.{sys._getframe().f_code.co_name}" task_id = None if celery.current_task: task_id = celery.current_task.request.id @@ -279,6 +331,7 @@ def sync_source(source): return try: sync([source]) + metrics.send(f"{function}.success", 'counter', '1', metric_tags={"source": source}) except SoftTimeLimitExceeded: log_data["message"] = "Error syncing source: Time limit exceeded." current_app.logger.error(log_data) @@ -290,6 +343,8 @@ def sync_source(source): log_data["message"] = "Done syncing source" current_app.logger.debug(log_data) + metrics.send(f"{function}.success", 'counter', 1, metric_tags=source) + red.set(f'{function}.last_success', int(time.time())) @celery.task() @@ -302,9 +357,12 @@ def sync_source_destination(): We rely on account numbers to avoid duplicates. """ current_app.logger.debug("Syncing AWS destinations and sources") + function = f"{__name__}.{sys._getframe().f_code.co_name}" for dst in destinations_service.get_all(): if add_aws_destination_to_sources(dst): current_app.logger.debug("Source: %s added", dst.label) current_app.logger.debug("Completed Syncing AWS destinations and sources") + red.set(f'{function}.last_success', int(time.time())) + metrics.send(f"{function}.success", 'counter', 1) diff --git a/lemur/common/redis.py b/lemur/common/redis.py new file mode 100644 index 00000000..34a8778f --- /dev/null +++ b/lemur/common/redis.py @@ -0,0 +1,38 @@ +""" +Helper Class for Redis + +""" +import redis +from flask import current_app +from lemur.factory import create_app + +if current_app: + flask_app = current_app +else: + flask_app = create_app() + + +class RedisHandler: + def __init__(self, host=flask_app.config.get('REDIS_HOST', 'localhost'), + port=flask_app.config.get('REDIS_PORT', 6379), + db=flask_app.config.get('REDIS_DB', 0)): + self.host = host + self.port = port + self.db = db + + def redis(self, db=0): + # The decode_responses flag here directs the client to convert the responses from Redis into Python strings + # using the default encoding utf-8. This is client specific. + red = redis.StrictRedis(host=self.host, port=self.port, db=self.db, encoding="utf-8", decode_responses=True) + return red + + +def redis_get(key, default=None): + red = RedisHandler().redis() + try: + v = red.get(key) + except redis.exceptions.ConnectionError: + v = None + if not v: + return default + return v diff --git a/lemur/tests/test_redis.py b/lemur/tests/test_redis.py new file mode 100644 index 00000000..aab2e397 --- /dev/null +++ b/lemur/tests/test_redis.py @@ -0,0 +1,13 @@ +import fakeredis +import time +import sys + + +def test_write_and_read_from_redis(): + function = f"{__name__}.{sys._getframe().f_code.co_name}" + + red = fakeredis.FakeStrictRedis() + key = f"{function}.last_success" + value = int(time.time()) + assert red.set(key, value) is True + assert (int(red.get(key)) == value) is True diff --git a/requirements-dev.txt b/requirements-dev.txt index 030c3f93..5e7b36f2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,31 +6,34 @@ # aspy.yaml==1.3.0 # via pre-commit bleach==3.1.0 # via readme-renderer -certifi==2019.3.9 # via requests +certifi==2019.6.16 # via requests cfgv==2.0.0 # via pre-commit chardet==3.0.4 # via requests docutils==0.14 # via readme-renderer flake8==3.5.0 -identify==1.4.3 # via pre-commit +identify==1.4.5 # via pre-commit idna==2.8 # via requests -importlib-metadata==0.17 # via pre-commit +importlib-metadata==0.18 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 pkginfo==1.5.0.1 # via twine -pre-commit==1.16.1 +pre-commit==1.17.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.4.2 # via readme-renderer -pyyaml==5.1 +pyyaml==5.1.1 readme-renderer==24.0 # via twine requests-toolbelt==0.9.1 # via twine requests==2.22.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit -tqdm==4.32.1 # via twine +tqdm==4.32.2 # via twine twine==1.13.0 urllib3==1.25.3 # via requests -virtualenv==16.6.0 # via pre-commit +virtualenv==16.6.1 # via pre-commit webencodings==0.5.1 # via bleach -zipp==0.5.1 # via importlib-metadata +zipp==0.5.2 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index c0fe427e..6222687e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,23 +4,23 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.34.2 +acme==0.36.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.0.10 +alembic==1.0.11 amqp==2.5.0 -aniso8601==6.0.0 +aniso8601==7.0.0 arrow==0.14.2 asn1crypto==0.24.0 asyncpool==1.0 babel==2.7.0 # via sphinx -bcrypt==3.1.6 +bcrypt==3.1.7 billiard==3.6.0.0 blinker==1.4 -boto3==1.9.160 -botocore==1.12.160 +boto3==1.9.187 +botocore==1.12.187 celery[redis]==4.3.0 -certifi==2019.3.9 +certifi==2019.6.16 certsrv==2.1.1 cffi==1.12.3 chardet==3.0.4 @@ -32,7 +32,7 @@ dnspython==1.15.0 docutils==0.14 dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.7 +flask-cors==3.0.8 flask-mail==0.9.1 flask-migrate==2.5.2 flask-principal==0.4.0 @@ -40,10 +40,10 @@ flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.0 -flask==1.0.3 +flask==1.1.1 future==0.17.1 gunicorn==19.9.0 -hvac==0.9.1 +hvac==0.9.3 idna==2.8 imagesize==1.1.0 # via sphinx inflection==0.3.1 @@ -51,21 +51,21 @@ itsdangerous==1.1.0 javaobj-py3==0.3.0 jinja2==2.10.1 jmespath==0.9.4 -josepy==1.1.0 +josepy==1.2.0 jsonlines==1.2.0 kombu==4.5.0 lockfile==0.12.2 logmatic-python==0.1.7 -mako==1.0.11 +mako==1.0.13 markupsafe==1.1.1 -marshmallow-sqlalchemy==0.16.3 -marshmallow==2.19.2 +marshmallow-sqlalchemy==0.17.0 +marshmallow==2.19.5 mock==3.0.5 ndg-httpsclient==0.5.1 packaging==19.0 # via sphinx -paramiko==2.4.2 +paramiko==2.6.0 pem==19.1.0 -psycopg2==2.8.2 +psycopg2==2.8.3 pyasn1-modules==0.2.5 pyasn1==0.4.5 pycparser==2.19 @@ -81,17 +81,17 @@ python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.11 pytz==2019.1 -pyyaml==5.1 +pyyaml==5.1.1 raven[flask]==6.10.0 redis==3.2.1 requests-toolbelt==0.9.1 requests[security]==2.22.0 retrying==1.3.3 -s3transfer==0.2.0 +s3transfer==0.2.1 six==1.12.0 -snowballstemmer==1.2.1 # via sphinx +snowballstemmer==1.9.0 # via sphinx sphinx-rtd-theme==0.4.3 -sphinx==2.1.0 +sphinx==2.1.2 sphinxcontrib-applehelp==1.0.1 # via sphinx sphinxcontrib-devhelp==1.0.1 # via sphinx sphinxcontrib-htmlhelp==1.0.2 # via sphinx @@ -99,11 +99,14 @@ sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx -sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.4 +sqlalchemy-utils==0.34.0 +sqlalchemy==1.3.5 tabulate==0.8.3 twofish==0.3.0 urllib3==1.25.3 vine==1.3.0 werkzeug==0.15.4 xmltodict==0.12.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via acme, josepy, sphinx diff --git a/requirements-tests.in b/requirements-tests.in index d315cf7a..610f26f9 100644 --- a/requirements-tests.in +++ b/requirements-tests.in @@ -5,6 +5,7 @@ black coverage factory-boy Faker +fakeredis freezegun moto nose diff --git a/requirements-tests.txt b/requirements-tests.txt index 77bc92af..fee45cf5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -7,33 +7,35 @@ appdirs==1.4.3 # via black asn1crypto==0.24.0 # via cryptography atomicwrites==1.3.0 # via pytest -attrs==19.1.0 # via black, pytest -aws-sam-translator==1.11.0 # via cfn-lint +attrs==19.1.0 # via black, jsonschema, pytest +aws-sam-translator==1.12.0 # via cfn-lint aws-xray-sdk==2.4.2 # via moto -bandit==1.6.0 +bandit==1.6.2 black==19.3b0 -boto3==1.9.160 # via aws-sam-translator, moto +boto3==1.9.187 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.160 # via aws-xray-sdk, boto3, moto, s3transfer -certifi==2019.3.9 # via requests +botocore==1.12.187 # via aws-xray-sdk, boto3, moto, s3transfer +certifi==2019.6.16 # via requests cffi==1.12.3 # via cryptography -cfn-lint==0.21.4 # via moto +cfn-lint==0.22.2 # via moto chardet==3.0.4 # via requests click==7.0 # via black, flask coverage==4.5.3 -cryptography==2.7 # via moto -docker==4.0.1 # via moto +cryptography==2.7 # via moto, sshpubkeys +datetime==4.3 # via moto +docker==4.0.2 # via moto docutils==0.14 # via botocore -ecdsa==0.13.2 # via python-jose +ecdsa==0.13.2 # via python-jose, sshpubkeys factory-boy==2.12.0 faker==1.0.7 -flask==1.0.3 # via pytest-flask +fakeredis==1.0.3 +flask==1.1.1 # via pytest-flask freezegun==0.3.12 future==0.17.1 # via aws-xray-sdk, python-jose gitdb2==2.0.5 # via gitpython gitpython==2.1.11 # via bandit idna==2.8 # via moto, requests -importlib-metadata==0.17 # via pluggy, pytest +importlib-metadata==0.18 # via pluggy, pytest itsdangerous==1.1.0 # via flask jinja2==2.10.1 # via flask, moto jmespath==0.9.4 # via boto3, botocore @@ -41,34 +43,38 @@ jsondiff==1.1.2 # via moto jsonpatch==1.23 # via cfn-lint jsonpickle==1.2 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch -jsonschema==2.6.0 # via aws-sam-translator, cfn-lint +jsonschema==3.0.1 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 mock==3.0.5 # via moto -more-itertools==7.0.0 # via pytest -moto==1.3.8 +more-itertools==7.1.0 # via pytest +moto==1.3.11 nose==1.3.7 packaging==19.0 # via pytest -pbr==5.2.1 # via stevedore +pbr==5.4.0 # via stevedore pluggy==0.12.0 # via pytest py==1.8.0 # via pytest pyasn1==0.4.5 # via rsa pycparser==2.19 # via cffi pyflakes==2.1.1 pyparsing==2.4.0 # via packaging +pyrsistent==0.15.3 # via jsonschema pytest-flask==0.15.0 pytest-mock==1.10.4 -pytest==4.6.2 +pytest==5.0.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto -pytz==2019.1 # via moto -pyyaml==5.1 +pytz==2019.1 # via datetime, moto +pyyaml==5.1.1 +redis==3.2.1 # via fakeredis requests-mock==1.6.0 requests==2.22.0 # via cfn-lint, docker, moto, requests-mock, responses responses==0.10.6 # via moto rsa==4.0 # via python-jose -s3transfer==0.2.0 # via boto3 -six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, freezegun, mock, moto, packaging, pytest, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client +s3transfer==0.2.1 # via boto3 +six==1.12.0 # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, faker, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client smmap2==2.0.5 # via gitdb2 +sshpubkeys==3.1.0 # via moto +sortedcontainers==2.1.0 # via fakeredis stevedore==1.30.1 # via bandit text-unidecode==1.2 # via faker toml==0.10.0 # via black @@ -76,6 +82,10 @@ urllib3==1.25.3 # via botocore, requests wcwidth==0.1.7 # via pytest websocket-client==0.56.0 # via docker werkzeug==0.15.4 # via flask, moto, pytest-flask -wrapt==1.11.1 # via aws-xray-sdk +wrapt==1.11.2 # via aws-xray-sdk xmltodict==0.12.0 # via moto -zipp==0.5.1 # via importlib-metadata +zipp==0.5.2 # via importlib-metadata +zope.interface==4.6.0 # via datetime + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via cfn-lint, jsonschema, zope.interface diff --git a/requirements.txt b/requirements.txt index c19c7b6e..7635c29d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,21 +4,21 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.34.2 +acme==0.36.0 alembic-autogenerate-enums==0.0.2 -alembic==1.0.10 # via flask-migrate +alembic==1.0.11 # via flask-migrate amqp==2.5.0 # via kombu -aniso8601==6.0.0 # via flask-restful +aniso8601==7.0.0 # via flask-restful arrow==0.14.2 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 -bcrypt==3.1.6 # via flask-bcrypt, paramiko +bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.0.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.160 -botocore==1.12.160 +boto3==1.9.187 +botocore==1.12.187 celery[redis]==4.3.0 -certifi==2019.3.9 +certifi==2019.6.16 certsrv==2.1.1 cffi==1.12.3 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -30,7 +30,7 @@ dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore dyn==1.8.1 flask-bcrypt==0.7.1 -flask-cors==3.0.7 +flask-cors==3.0.8 flask-mail==0.9.1 flask-migrate==2.5.2 flask-principal==0.4.0 @@ -38,32 +38,32 @@ flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 flask-sqlalchemy==2.4.0 -flask==1.0.3 +flask==1.1.1 future==0.17.1 gunicorn==19.9.0 -hvac==0.9.1 +hvac==0.9.3 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask javaobj-py3==0.3.0 # via pyjks jinja2==2.10.1 jmespath==0.9.4 # via boto3, botocore -josepy==1.1.0 # via acme +josepy==1.2.0 # via acme jsonlines==1.2.0 # via cloudflare kombu==4.5.0 lockfile==0.12.2 logmatic-python==0.1.7 -mako==1.0.11 # via alembic +mako==1.0.13 # via alembic markupsafe==1.1.1 # via jinja2, mako -marshmallow-sqlalchemy==0.16.3 -marshmallow==2.19.2 +marshmallow-sqlalchemy==0.17.0 +marshmallow==2.19.5 mock==3.0.5 # via acme ndg-httpsclient==0.5.1 -paramiko==2.4.2 +paramiko==2.6.0 pem==19.1.0 -psycopg2==2.8.2 +psycopg2==2.8.3 pyasn1-modules==0.2.5 # via pyjks, python-ldap -pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, pyjks, python-ldap +pyasn1==0.4.5 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.19 # via cffi pycryptodomex==3.8.2 # via pyjks pyjks==19.0.0 @@ -76,19 +76,22 @@ python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python python-ldap==3.2.0 pytz==2019.1 # via acme, celery, flask-restful, pyrfc3339 -pyyaml==5.1 +pyyaml==5.1.1 raven[flask]==6.10.0 redis==3.2.1 requests-toolbelt==0.9.1 # via acme requests[security]==2.22.0 retrying==1.3.3 -s3transfer==0.2.0 # via boto3 +s3transfer==0.2.1 # via boto3 six==1.12.0 -sqlalchemy-utils==0.33.11 -sqlalchemy==1.3.4 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy-utils==0.34.0 +sqlalchemy==1.3.5 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.3 twofish==0.3.0 # via pyjks urllib3==1.25.3 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==0.15.4 # via flask xmltodict==0.12.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.0.1 # via acme, josepy