From 86082009b97917320c4f57105a38af8c941f0205 Mon Sep 17 00:00:00 2001 From: Justin P Date: Mon, 9 Jul 2018 23:24:35 -0500 Subject: [PATCH 01/31] Sinful Use of `$` Using the `$` sign within any block of text already marked as a code block is a grievous sin due to the fact that it makes it 100% pointless for you to have USED THE CODE BLOCK IN THE FIRST PLACE! The `$` becomes included in the text we're trying to highlight for us to be able to actually use in our own projects. Why post the info if you don't want us to use it. Thank you. --- docs/quickstart/index.rst | 56 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 32abcc9c..70ca1312 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -26,19 +26,19 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack .. code-block:: bash - $ sudo apt-get update - $ sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql + sudo apt-get update + sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver. npm is needed if you're installing Lemur from the source (e.g., from git). .. note:: Installing node from a package manager may creat the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following - $ sudo ln -s /user/bin/nodejs /usr/bin/node + sudo ln -s /user/bin/nodejs /usr/bin/node Now, install Python ``virtualenv`` package: .. code-block:: bash - $ sudo pip install -U virtualenv + sudo pip install -U virtualenv Setting up an Environment @@ -48,28 +48,28 @@ In this guide, Lemur will be installed in ``/www``, so you need to create that s .. code-block:: bash - $ sudo mkdir /www - $ cd /www + sudo mkdir /www + cd /www Clone Lemur inside the just created directory and give yourself write permission (we assume ``lemur`` is the user): .. code-block:: bash - $ sudo useradd lemur - $ sudo passwd lemur - $ sudo mkdir /home/lemur - $ sudo chown lemur:lemur /home/lemur - $ sudo git clone https://github.com/Netflix/lemur - $ sudo chown -R lemur lemur/ + sudo useradd lemur + sudo passwd lemur + sudo mkdir /home/lemur + sudo chown lemur:lemur /home/lemur + sudo git clone https://github.com/Netflix/lemur + sudo chown -R lemur lemur/ Create the virtual environment, activate it and enter the Lemur's directory: .. code-block:: bash - $ su lemur - $ virtualenv -p python3 lemur - $ source /www/lemur/bin/activate - $ cd lemur + su lemur + virtualenv -p python3 lemur + source /www/lemur/bin/activate + cd lemur .. note:: Activating the environment adjusts your PATH, so that things like pip now install into the virtualenv by default. @@ -81,13 +81,13 @@ Once your system is prepared, ensure that you are in the virtualenv: .. code-block:: bash - $ which python + which python And then run: .. code-block:: bash - $ make release + make release .. note:: This command will install npm dependencies as well as compile static assets. @@ -101,7 +101,7 @@ You may also run with the urlContextPath variable set. If this is set it will ad .. code-block:: bash - $ make release urlContextPath={desired context path} + make release urlContextPath={desired context path} Creating a configuration @@ -113,7 +113,7 @@ Simply run: .. code-block:: bash - $ lemur create_config + lemur create_config .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command. @@ -127,7 +127,7 @@ Once created, you will need to update the configuration file with information ab .. code-block:: bash - $ vi ~/.lemur/lemur.conf.py + vi ~/.lemur/lemur.conf.py .. note:: If you are unfamiliar with the SQLALCHEMY_DATABASE_URI string it can be broken up like so: ``postgresql://userame:password@:/`` @@ -153,8 +153,8 @@ First, set a password for the postgres user. For this guide, we will use ``lemu .. code-block:: bash - $ sudo -u postgres -i - $ psql + sudo -u postgres -i + psql postgres=# CREATE USER lemur WITH PASSWORD 'lemur'; Once successful, type CTRL-D to exit the Postgres shell. @@ -163,7 +163,7 @@ Next, we will create our new database: .. code-block:: bash - $ sudo -u postgres createdb lemur + sudo -u postgres createdb lemur .. _InitializingLemur: @@ -186,8 +186,8 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. code-block:: bash - $ cd /www/lemur/lemur - $ lemur init + cd /www/lemur/lemur + lemur init .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. @@ -228,7 +228,7 @@ After making these changes, restart Nginx service to apply them: .. code-block:: bash - $ sudo service nginx restart + sudo service nginx restart Starting the Web Service @@ -284,7 +284,7 @@ Lemur uses periodic sync tasks to make sure it is up-to-date with its environmen .. code-block:: bash - $ crontab -e + crontab -e */15 * * * * lemur sync -s all 0 22 * * * lemur check_revoked 0 22 * * * lemur notify From 8dc52b859ba2e2183ebe52db510dd546dd13131c Mon Sep 17 00:00:00 2001 From: root Date: Wed, 11 Jul 2018 11:57:36 -0500 Subject: [PATCH 02/31] initial commit --- lemur/dns_providers/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lemur/dns_providers/__init__.py diff --git a/lemur/dns_providers/__init__.py b/lemur/dns_providers/__init__.py new file mode 100644 index 00000000..e69de29b From 2f32014c75d4707e8fcd48ee8865e7eb3fb290db Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Tue, 19 Jun 2018 18:41:12 +0300 Subject: [PATCH 03/31] Cache parsed certificate instead of re-parsing for each field Use @cached_property decorator to cache the results of parse_certificate(). This significantly cuts down on the number of times certs need to be parsed for a list view. --- lemur/certificates/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index f5b60ad5..87ee3b93 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -23,6 +23,8 @@ from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property +import lemur.common.utils + from lemur.database import db from lemur.extensions import sentry @@ -186,7 +188,7 @@ class Certificate(db.Model): @cached_property def parsed_cert(self): assert self.body, "Certificate body not set" - return utils.parse_certificate(self.body) + return lemur.common.utils.parse_certificate(self.body) @property def active(self): From ead374db5fa575881f92550921a26d1a19c99818 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Wed, 27 Jun 2018 13:56:39 +0300 Subject: [PATCH 04/31] Clean up module imports Example: * import lemur.common.utils -> from lemur.common import utils * import sqlalchemy.types as types -> from sqlalchemy import types --- lemur/certificates/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 87ee3b93..f5b60ad5 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -23,8 +23,6 @@ from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property -import lemur.common.utils - from lemur.database import db from lemur.extensions import sentry @@ -188,7 +186,7 @@ class Certificate(db.Model): @cached_property def parsed_cert(self): assert self.body, "Certificate body not set" - return lemur.common.utils.parse_certificate(self.body) + return utils.parse_certificate(self.body) @property def active(self): From a19a47dba1d8a22ae41e2e6401b7c642c735dca4 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 12 Jul 2018 13:24:44 -0700 Subject: [PATCH 05/31] updated requirements --- requirements-dev.txt | 8 ++++---- requirements-docs.txt | 2 +- requirements-tests.txt | 14 +++++++------- requirements.txt | 12 ++++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4f6d3603..58d6e277 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,16 +10,16 @@ certifi==2018.4.16 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.1.0 # via pre-commit +identify==1.1.3 # via pre-commit idna==2.7 # via requests -invoke==1.0.0 +invoke==1.1.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.1 pkginfo==1.4.2 # via twine -pre-commit==1.10.2 +pre-commit==1.10.3 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 -pyyaml==3.12 # via aspy.yaml, pre-commit +pyyaml==3.13 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.19.1 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 146ddf25..9cfa1b7b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -79,7 +79,7 @@ six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.0 sphinx==1.7.5 -sphinxcontrib-httpdomain==1.6.1 +sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 sqlalchemy==1.2.9 diff --git a/requirements-tests.txt b/requirements-tests.txt index 1da5269f..219c445a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.48 # via moto -boto==2.48.0 # via moto -botocore==1.10.48 # via boto3, moto, s3transfer +boto3==1.7.56 # via moto +boto==2.49.0 # via moto +botocore==1.10.56 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -22,7 +22,7 @@ docker-pycreds==0.3.0 # via docker docker==3.4.1 # via moto docutils==0.14 # via botocore factory-boy==2.11.1 -faker==0.8.16 +faker==0.8.17 flask==1.0.2 # via pytest-flask freezegun==0.3.10 idna==2.7 # via cryptography, requests @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.0.4 # via mock +pbr==4.1.0 # via mock pluggy==0.6.0 # via pytest py==1.5.4 # via pytest pyaml==17.12.1 # via moto @@ -44,10 +44,10 @@ pycparser==2.18 # via cffi pyflakes==2.0.0 pytest-flask==0.10.0 pytest-mock==1.10.0 -pytest==3.6.2 +pytest==3.6.3 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.5 # via moto -pyyaml==3.12 # via pyaml +pyyaml==3.13 # via pyaml requests-mock==1.5.0 requests==2.19.1 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto diff --git a/requirements.txt b/requirements.txt index dd6ce10f..d80d7e09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.25.1 +acme==0.26.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.10 # via flask-migrate aniso8601==3.0.2 # via flask-restful @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.48 -botocore==1.10.48 # via boto3, s3transfer +boto3==1.7.56 +botocore==1.10.56 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -34,7 +34,7 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 -gunicorn==19.8.1 +gunicorn==19.9.0 idna==2.7 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask @@ -50,7 +50,7 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.0.4 # via mock +pbr==4.1.0 # via mock pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap @@ -64,7 +64,7 @@ python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 pytz==2018.5 # via acme, flask-restful, pyrfc3339 -pyyaml==3.12 # via cloudflare +pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 requests-toolbelt==0.8.0 # via acme requests[security]==2.11.1 From 7f3454128d379da8733c4d528ebd4c2a587c861a Mon Sep 17 00:00:00 2001 From: Steven Reiling Date: Fri, 13 Jul 2018 13:34:43 -0700 Subject: [PATCH 06/31] Adds an optional interval variable to notification service's create_default_expiration_notifications and introduces a new optional configuration variable, LEMUR_SECURITY_TEAM_EMAIL_INTERVALS, to allow admins control over the centralized email notification defaults. --- docs/administration.rst | 10 +++++++++- lemur/certificates/schemas.py | 4 +++- lemur/notifications/service.py | 10 ++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index aab7cd58..eec01cc5 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -274,7 +274,6 @@ Lemur supports sending certification expiration notifications through SES and SM LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com'] - .. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS :noindex: @@ -284,6 +283,15 @@ Lemur supports sending certification expiration notifications through SES and SM LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2] +.. data:: LEMUR_SECURITY_TEAM_EMAIL_INTERVALS + :noindex: + + Alternate notification interval set for security team notifications. Use this if you would like the default security team notification interval for new certificates to differ from the global default as specified in LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS. If unspecified, the value of LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS is used. Security team default notifications for new certificates can effectively be disabled by setting this value to an empty array. + + :: + + LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2] + Authentication Options ---------------------- diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 72b42fb9..e88b6e73 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -48,9 +48,11 @@ class CertificateCreationSchema(CertificateSchema): "DEFAULT_{0}".format(data['owner'].split('@')[0].upper()), [data['owner']], ) + data['notifications'] += notification_service.create_default_expiration_notifications( 'DEFAULT_SECURITY', - current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL') + current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'), + current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL_INTERVALS', None) ) return data diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 466c680b..957757bd 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -16,10 +16,11 @@ from lemur.common.utils import truthiness from lemur.notifications.models import Notification -def create_default_expiration_notifications(name, recipients): +def create_default_expiration_notifications(name, recipients, intervals=None): """ - Will create standard 30, 10 and 2 day notifications for a given owner. If standard notifications - already exist these will be returned instead of new notifications. + Will create standard 30, 10 and 2 day notifications for a given owner unless an alternate set of + intervals is supplied. If standard notifications already exist these will be returned instead of + new notifications. :param name: :param recipients: @@ -48,7 +49,8 @@ def create_default_expiration_notifications(name, recipients): }, ] - intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", [30, 15, 2]) + if intervals is None: + intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", [30, 15, 2]) notifications = [] for i in intervals: From be9be6d3cd283351486992c51bf0e41114d88b78 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 17 Jul 2018 18:38:15 -0700 Subject: [PATCH 07/31] Unpinning requests --- requirements-docs.txt | 14 +++++++------- requirements-tests.txt | 4 ++-- requirements.in | 4 ++-- requirements.txt | 20 +++++++++++--------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 9cfa1b7b..bc930c58 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.25.1 +acme==0.26.0 alabaster==0.7.11 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==0.9.10 @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.48 -botocore==1.10.48 +boto3==1.7.56 +botocore==1.10.56 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -36,7 +36,7 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 -gunicorn==19.8.1 +gunicorn==19.9.0 idna==2.7 imagesize==1.0.0 # via sphinx inflection==0.3.1 @@ -54,7 +54,7 @@ mock==2.0.0 ndg-httpsclient==0.5.0 packaging==17.1 # via sphinx paramiko==2.4.1 -pbr==4.0.4 +pbr==4.1.0 pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 @@ -69,7 +69,7 @@ pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 pytz==2018.5 -pyyaml==3.12 +pyyaml==3.13 raven[flask]==6.9.0 requests-toolbelt==0.8.0 requests[security]==2.11.1 @@ -78,7 +78,7 @@ s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.0 -sphinx==1.7.5 +sphinx==1.7.6 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 diff --git a/requirements-tests.txt b/requirements-tests.txt index 219c445a..55836e9a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.56 # via moto +boto3==1.7.59 # via moto boto==2.49.0 # via moto -botocore==1.10.56 # via boto3, moto, s3transfer +botocore==1.10.59 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests diff --git a/requirements.in b/requirements.in index 2a5051a7..0e028261 100644 --- a/requirements.in +++ b/requirements.in @@ -31,10 +31,10 @@ paramiko # required for the SFTP destination plugin pem psycopg2 pyjwt -pyOpenSSL==17.2.0 # PINNED for a specific reason. This needs to be merged in before upgrade: https://github.com/shazow/urllib3/pull/1246 +pyOpenSSL python_ldap raven[flask] -requests==2.11.1 # PINNED for a specific reason. This needs to be merged in before upgrade: https://github.com/shazow/urllib3/pull/1246 +requests retrying six SQLAlchemy-Utils diff --git a/requirements.txt b/requirements.txt index d80d7e09..82817e7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,20 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.26.0 +acme==0.26.1 alembic-autogenerate-enums==0.0.2 -alembic==0.9.10 # via flask-migrate +alembic==1.0.0 # via flask-migrate aniso8601==3.0.2 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.56 -botocore==1.10.56 # via boto3, s3transfer +boto3==1.7.59 +botocore==1.10.59 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl +chardet==3.0.4 # via requests click==6.7 # via flask cloudflare==2.1.0 cryptography==2.2.2 @@ -35,7 +36,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.9.0 -idna==2.7 # via cryptography +idna==2.7 # via cryptography, requests inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 @@ -54,11 +55,11 @@ pbr==4.1.0 # via mock pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap -pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap, requests +pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.18 # via cffi pyjwt==1.6.4 pynacl==1.2.1 # via paramiko -pyopenssl==17.2.0 +pyopenssl==18.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic @@ -67,12 +68,13 @@ pytz==2018.5 # via acme, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 requests-toolbelt==0.8.0 # via acme -requests[security]==2.11.1 +requests[security]==2.19.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 sqlalchemy-utils==0.33.3 -sqlalchemy==1.2.9 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.10 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 +urllib3==1.23 # via requests werkzeug==0.14.1 # via flask xmltodict==0.11.0 From 3ac440b6b5b7f4c4303c5bd5115eb16aae3edc3b Mon Sep 17 00:00:00 2001 From: Justin P Date: Mon, 9 Jul 2018 23:24:35 -0500 Subject: [PATCH 08/31] Sinful Use of `$` Using the `$` sign within any block of text already marked as a code block is a grievous sin due to the fact that it makes it 100% pointless for you to have USED THE CODE BLOCK IN THE FIRST PLACE! The `$` becomes included in the text we're trying to highlight for us to be able to actually use in our own projects. Why post the info if you don't want us to use it. Thank you. --- docs/quickstart/index.rst | 56 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 32abcc9c..70ca1312 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -26,19 +26,19 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack .. code-block:: bash - $ sudo apt-get update - $ sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql + sudo apt-get update + sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver. npm is needed if you're installing Lemur from the source (e.g., from git). .. note:: Installing node from a package manager may creat the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following - $ sudo ln -s /user/bin/nodejs /usr/bin/node + sudo ln -s /user/bin/nodejs /usr/bin/node Now, install Python ``virtualenv`` package: .. code-block:: bash - $ sudo pip install -U virtualenv + sudo pip install -U virtualenv Setting up an Environment @@ -48,28 +48,28 @@ In this guide, Lemur will be installed in ``/www``, so you need to create that s .. code-block:: bash - $ sudo mkdir /www - $ cd /www + sudo mkdir /www + cd /www Clone Lemur inside the just created directory and give yourself write permission (we assume ``lemur`` is the user): .. code-block:: bash - $ sudo useradd lemur - $ sudo passwd lemur - $ sudo mkdir /home/lemur - $ sudo chown lemur:lemur /home/lemur - $ sudo git clone https://github.com/Netflix/lemur - $ sudo chown -R lemur lemur/ + sudo useradd lemur + sudo passwd lemur + sudo mkdir /home/lemur + sudo chown lemur:lemur /home/lemur + sudo git clone https://github.com/Netflix/lemur + sudo chown -R lemur lemur/ Create the virtual environment, activate it and enter the Lemur's directory: .. code-block:: bash - $ su lemur - $ virtualenv -p python3 lemur - $ source /www/lemur/bin/activate - $ cd lemur + su lemur + virtualenv -p python3 lemur + source /www/lemur/bin/activate + cd lemur .. note:: Activating the environment adjusts your PATH, so that things like pip now install into the virtualenv by default. @@ -81,13 +81,13 @@ Once your system is prepared, ensure that you are in the virtualenv: .. code-block:: bash - $ which python + which python And then run: .. code-block:: bash - $ make release + make release .. note:: This command will install npm dependencies as well as compile static assets. @@ -101,7 +101,7 @@ You may also run with the urlContextPath variable set. If this is set it will ad .. code-block:: bash - $ make release urlContextPath={desired context path} + make release urlContextPath={desired context path} Creating a configuration @@ -113,7 +113,7 @@ Simply run: .. code-block:: bash - $ lemur create_config + lemur create_config .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command. @@ -127,7 +127,7 @@ Once created, you will need to update the configuration file with information ab .. code-block:: bash - $ vi ~/.lemur/lemur.conf.py + vi ~/.lemur/lemur.conf.py .. note:: If you are unfamiliar with the SQLALCHEMY_DATABASE_URI string it can be broken up like so: ``postgresql://userame:password@:/`` @@ -153,8 +153,8 @@ First, set a password for the postgres user. For this guide, we will use ``lemu .. code-block:: bash - $ sudo -u postgres -i - $ psql + sudo -u postgres -i + psql postgres=# CREATE USER lemur WITH PASSWORD 'lemur'; Once successful, type CTRL-D to exit the Postgres shell. @@ -163,7 +163,7 @@ Next, we will create our new database: .. code-block:: bash - $ sudo -u postgres createdb lemur + sudo -u postgres createdb lemur .. _InitializingLemur: @@ -186,8 +186,8 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. code-block:: bash - $ cd /www/lemur/lemur - $ lemur init + cd /www/lemur/lemur + lemur init .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. @@ -228,7 +228,7 @@ After making these changes, restart Nginx service to apply them: .. code-block:: bash - $ sudo service nginx restart + sudo service nginx restart Starting the Web Service @@ -284,7 +284,7 @@ Lemur uses periodic sync tasks to make sure it is up-to-date with its environmen .. code-block:: bash - $ crontab -e + crontab -e */15 * * * * lemur sync -s all 0 22 * * * lemur check_revoked 0 22 * * * lemur notify From 2a5f713f97103cad37f76ce3c4a29f026a9815de Mon Sep 17 00:00:00 2001 From: root Date: Wed, 11 Jul 2018 11:57:36 -0500 Subject: [PATCH 09/31] initial commit --- lemur/dns_providers/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lemur/dns_providers/__init__.py diff --git a/lemur/dns_providers/__init__.py b/lemur/dns_providers/__init__.py new file mode 100644 index 00000000..e69de29b From 5a0184078430d3bc1c8636f7f0525070d8e08346 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 20 Jul 2018 10:47:19 -0700 Subject: [PATCH 10/31] Explicit capture exception during create failure --- lemur/certificates/service.py | 9 +++++++-- requirements-dev.txt | 2 +- requirements-docs.txt | 16 +++++++++------- requirements-tests.txt | 8 ++++---- requirements.txt | 8 ++++---- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index e7a5afd1..16383d61 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -15,7 +15,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from lemur import database -from lemur.extensions import metrics, signals +from lemur.extensions import metrics, sentry, signals from lemur.plugins.base import plugins from lemur.common.utils import generate_private_key, truthiness @@ -247,7 +247,12 @@ def create(**kwargs): """ Creates a new certificate. """ - cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) + try: + cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) + except: + current_app.logger.error("Exception minting certificate", exc_info=True) + sentry.captureException() + raise kwargs['body'] = cert_body kwargs['private_key'] = private_key kwargs['chain'] = cert_chain diff --git a/requirements-dev.txt b/requirements-dev.txt index 58d6e277..3909600c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,7 +14,7 @@ identify==1.1.3 # via pre-commit idna==2.7 # via requests invoke==1.1.0 mccabe==0.6.1 # via flake8 -nodeenv==1.3.1 +nodeenv==1.3.2 pkginfo==1.4.2 # via twine pre-commit==1.10.3 pycodestyle==2.3.1 # via flake8 diff --git a/requirements-docs.txt b/requirements-docs.txt index bc930c58..2de29229 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,10 +4,10 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.26.0 +acme==0.26.1 alabaster==0.7.11 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==0.9.10 +alembic==1.0.0 aniso8601==3.0.2 arrow==0.12.1 asn1crypto==0.24.0 @@ -15,10 +15,11 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.56 -botocore==1.10.56 +boto3==1.7.59 +botocore==1.10.59 certifi==2018.4.16 cffi==1.11.5 +chardet==3.0.4 click==6.7 cloudflare==2.1.0 cryptography==2.2.2 @@ -63,7 +64,7 @@ pycparser==2.18 pygments==2.2.0 # via sphinx pyjwt==1.6.4 pynacl==1.2.1 -pyopenssl==17.2.0 +pyopenssl==18.0.0 pyparsing==2.2.0 # via packaging pyrfc3339==1.1 python-dateutil==2.7.3 @@ -72,7 +73,7 @@ pytz==2018.5 pyyaml==3.13 raven[flask]==6.9.0 requests-toolbelt==0.8.0 -requests[security]==2.11.1 +requests[security]==2.19.1 retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 @@ -82,7 +83,8 @@ sphinx==1.7.6 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 -sqlalchemy==1.2.9 +sqlalchemy==1.2.10 tabulate==0.8.2 +urllib3==1.23 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 55836e9a..57292b12 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,16 +8,16 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.59 # via moto +boto3==1.7.61 # via moto boto==2.49.0 # via moto -botocore==1.10.59 # via boto3, moto, s3transfer +botocore==1.10.61 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 -cryptography==2.2.2 # via moto +cryptography==2.3 # via moto docker-pycreds==0.3.0 # via docker docker==3.4.1 # via moto docutils==0.14 # via botocore @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.1.0 # via mock +pbr==4.1.1 # via mock pluggy==0.6.0 # via pytest py==1.5.4 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.txt b/requirements.txt index 82817e7e..3012ea6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,14 +13,14 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.59 -botocore==1.10.59 # via boto3, s3transfer +boto3==1.7.61 +botocore==1.10.61 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==6.7 # via flask cloudflare==2.1.0 -cryptography==2.2.2 +cryptography==2.3 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore @@ -51,7 +51,7 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.1.0 # via mock +pbr==4.1.1 # via mock pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap From f93e938cda078fa3d137bf19368d55f8ef49c042 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 20 Jul 2018 10:53:47 -0700 Subject: [PATCH 11/31] no bare except --- lemur/certificates/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 16383d61..9b250fc3 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -249,7 +249,7 @@ def create(**kwargs): """ try: cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) - except: + except Exception: current_app.logger.error("Exception minting certificate", exc_info=True) sentry.captureException() raise From 7146c4cb710b1c6921496fadf2b3567afc32df06 Mon Sep 17 00:00:00 2001 From: Justin P Date: Mon, 9 Jul 2018 23:24:35 -0500 Subject: [PATCH 12/31] Sinful Use of `$` Using the `$` sign within any block of text already marked as a code block is a grievous sin due to the fact that it makes it 100% pointless for you to have USED THE CODE BLOCK IN THE FIRST PLACE! The `$` becomes included in the text we're trying to highlight for us to be able to actually use in our own projects. Why post the info if you don't want us to use it. Thank you. --- docs/quickstart/index.rst | 56 +++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/quickstart/index.rst b/docs/quickstart/index.rst index 32abcc9c..70ca1312 100644 --- a/docs/quickstart/index.rst +++ b/docs/quickstart/index.rst @@ -26,19 +26,19 @@ If installing Lemur on a bare Ubuntu OS you will need to grab the following pack .. code-block:: bash - $ sudo apt-get update - $ sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql + sudo apt-get update + sudo apt-get install nodejs nodejs-legacy python-pip python-dev python3-dev libpq-dev build-essential libssl-dev libffi-dev libsasl2-dev libldap2-dev nginx git supervisor npm postgresql .. note:: PostgreSQL is only required if your database is going to be on the same host as the webserver. npm is needed if you're installing Lemur from the source (e.g., from git). .. note:: Installing node from a package manager may creat the nodejs bin at /usr/bin/nodejs instead of /usr/bin/node If that is the case run the following - $ sudo ln -s /user/bin/nodejs /usr/bin/node + sudo ln -s /user/bin/nodejs /usr/bin/node Now, install Python ``virtualenv`` package: .. code-block:: bash - $ sudo pip install -U virtualenv + sudo pip install -U virtualenv Setting up an Environment @@ -48,28 +48,28 @@ In this guide, Lemur will be installed in ``/www``, so you need to create that s .. code-block:: bash - $ sudo mkdir /www - $ cd /www + sudo mkdir /www + cd /www Clone Lemur inside the just created directory and give yourself write permission (we assume ``lemur`` is the user): .. code-block:: bash - $ sudo useradd lemur - $ sudo passwd lemur - $ sudo mkdir /home/lemur - $ sudo chown lemur:lemur /home/lemur - $ sudo git clone https://github.com/Netflix/lemur - $ sudo chown -R lemur lemur/ + sudo useradd lemur + sudo passwd lemur + sudo mkdir /home/lemur + sudo chown lemur:lemur /home/lemur + sudo git clone https://github.com/Netflix/lemur + sudo chown -R lemur lemur/ Create the virtual environment, activate it and enter the Lemur's directory: .. code-block:: bash - $ su lemur - $ virtualenv -p python3 lemur - $ source /www/lemur/bin/activate - $ cd lemur + su lemur + virtualenv -p python3 lemur + source /www/lemur/bin/activate + cd lemur .. note:: Activating the environment adjusts your PATH, so that things like pip now install into the virtualenv by default. @@ -81,13 +81,13 @@ Once your system is prepared, ensure that you are in the virtualenv: .. code-block:: bash - $ which python + which python And then run: .. code-block:: bash - $ make release + make release .. note:: This command will install npm dependencies as well as compile static assets. @@ -101,7 +101,7 @@ You may also run with the urlContextPath variable set. If this is set it will ad .. code-block:: bash - $ make release urlContextPath={desired context path} + make release urlContextPath={desired context path} Creating a configuration @@ -113,7 +113,7 @@ Simply run: .. code-block:: bash - $ lemur create_config + lemur create_config .. note:: This command will create a default configuration under ``~/.lemur/lemur.conf.py`` you can specify this location by passing the ``config_path`` parameter to the ``create_config`` command. @@ -127,7 +127,7 @@ Once created, you will need to update the configuration file with information ab .. code-block:: bash - $ vi ~/.lemur/lemur.conf.py + vi ~/.lemur/lemur.conf.py .. note:: If you are unfamiliar with the SQLALCHEMY_DATABASE_URI string it can be broken up like so: ``postgresql://userame:password@:/`` @@ -153,8 +153,8 @@ First, set a password for the postgres user. For this guide, we will use ``lemu .. code-block:: bash - $ sudo -u postgres -i - $ psql + sudo -u postgres -i + psql postgres=# CREATE USER lemur WITH PASSWORD 'lemur'; Once successful, type CTRL-D to exit the Postgres shell. @@ -163,7 +163,7 @@ Next, we will create our new database: .. code-block:: bash - $ sudo -u postgres createdb lemur + sudo -u postgres createdb lemur .. _InitializingLemur: @@ -186,8 +186,8 @@ Additional notifications can be created through the UI or API. See :ref:`Creati .. code-block:: bash - $ cd /www/lemur/lemur - $ lemur init + cd /www/lemur/lemur + lemur init .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users ` and :ref:`Command Line Interface ` for details. @@ -228,7 +228,7 @@ After making these changes, restart Nginx service to apply them: .. code-block:: bash - $ sudo service nginx restart + sudo service nginx restart Starting the Web Service @@ -284,7 +284,7 @@ Lemur uses periodic sync tasks to make sure it is up-to-date with its environmen .. code-block:: bash - $ crontab -e + crontab -e */15 * * * * lemur sync -s all 0 22 * * * lemur check_revoked 0 22 * * * lemur notify From 56372c55b4f7399e98b81c0ba2a320bad63ebc3b Mon Sep 17 00:00:00 2001 From: root Date: Wed, 11 Jul 2018 11:57:36 -0500 Subject: [PATCH 13/31] initial commit --- lemur/dns_providers/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lemur/dns_providers/__init__.py diff --git a/lemur/dns_providers/__init__.py b/lemur/dns_providers/__init__.py new file mode 100644 index 00000000..e69de29b From 04ee1656ee263575faa2da5f23ef58010d4bd6ec Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Tue, 19 Jun 2018 18:41:12 +0300 Subject: [PATCH 14/31] Cache parsed certificate instead of re-parsing for each field Use @cached_property decorator to cache the results of parse_certificate(). This significantly cuts down on the number of times certs need to be parsed for a list view. --- lemur/certificates/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index f5b60ad5..87ee3b93 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -23,6 +23,8 @@ from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property +import lemur.common.utils + from lemur.database import db from lemur.extensions import sentry @@ -186,7 +188,7 @@ class Certificate(db.Model): @cached_property def parsed_cert(self): assert self.body, "Certificate body not set" - return utils.parse_certificate(self.body) + return lemur.common.utils.parse_certificate(self.body) @property def active(self): From d071d85486b32fd6e83206c383c4d1b4400f33f9 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Wed, 27 Jun 2018 13:56:39 +0300 Subject: [PATCH 15/31] Clean up module imports Example: * import lemur.common.utils -> from lemur.common import utils * import sqlalchemy.types as types -> from sqlalchemy import types --- lemur/certificates/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 87ee3b93..f5b60ad5 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -23,8 +23,6 @@ from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, from sqlalchemy_utils.types.arrow import ArrowType from werkzeug.utils import cached_property -import lemur.common.utils - from lemur.database import db from lemur.extensions import sentry @@ -188,7 +186,7 @@ class Certificate(db.Model): @cached_property def parsed_cert(self): assert self.body, "Certificate body not set" - return lemur.common.utils.parse_certificate(self.body) + return utils.parse_certificate(self.body) @property def active(self): From af8cf2d550678dc6d8714f6d8d7894f2323e80d4 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Thu, 12 Jul 2018 13:24:44 -0700 Subject: [PATCH 16/31] updated requirements --- requirements-dev.txt | 8 ++++---- requirements-docs.txt | 2 +- requirements-tests.txt | 14 +++++++------- requirements.txt | 12 ++++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4f6d3603..58d6e277 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,16 +10,16 @@ certifi==2018.4.16 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.1.0 # via pre-commit +identify==1.1.3 # via pre-commit idna==2.7 # via requests -invoke==1.0.0 +invoke==1.1.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.1 pkginfo==1.4.2 # via twine -pre-commit==1.10.2 +pre-commit==1.10.3 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 -pyyaml==3.12 # via aspy.yaml, pre-commit +pyyaml==3.13 # via aspy.yaml, pre-commit requests-toolbelt==0.8.0 # via twine requests==2.19.1 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 146ddf25..9cfa1b7b 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -79,7 +79,7 @@ six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.0 sphinx==1.7.5 -sphinxcontrib-httpdomain==1.6.1 +sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 sqlalchemy==1.2.9 diff --git a/requirements-tests.txt b/requirements-tests.txt index 1da5269f..219c445a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.48 # via moto -boto==2.48.0 # via moto -botocore==1.10.48 # via boto3, moto, s3transfer +boto3==1.7.56 # via moto +boto==2.49.0 # via moto +botocore==1.10.56 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -22,7 +22,7 @@ docker-pycreds==0.3.0 # via docker docker==3.4.1 # via moto docutils==0.14 # via botocore factory-boy==2.11.1 -faker==0.8.16 +faker==0.8.17 flask==1.0.2 # via pytest-flask freezegun==0.3.10 idna==2.7 # via cryptography, requests @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.0.4 # via mock +pbr==4.1.0 # via mock pluggy==0.6.0 # via pytest py==1.5.4 # via pytest pyaml==17.12.1 # via moto @@ -44,10 +44,10 @@ pycparser==2.18 # via cffi pyflakes==2.0.0 pytest-flask==0.10.0 pytest-mock==1.10.0 -pytest==3.6.2 +pytest==3.6.3 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.5 # via moto -pyyaml==3.12 # via pyaml +pyyaml==3.13 # via pyaml requests-mock==1.5.0 requests==2.19.1 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto diff --git a/requirements.txt b/requirements.txt index dd6ce10f..d80d7e09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.25.1 +acme==0.26.0 alembic-autogenerate-enums==0.0.2 alembic==0.9.10 # via flask-migrate aniso8601==3.0.2 # via flask-restful @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.48 -botocore==1.10.48 # via boto3, s3transfer +boto3==1.7.56 +botocore==1.10.56 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -34,7 +34,7 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 -gunicorn==19.8.1 +gunicorn==19.9.0 idna==2.7 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask @@ -50,7 +50,7 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.0.4 # via mock +pbr==4.1.0 # via mock pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap @@ -64,7 +64,7 @@ python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic python-ldap==3.1.0 pytz==2018.5 # via acme, flask-restful, pyrfc3339 -pyyaml==3.12 # via cloudflare +pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 requests-toolbelt==0.8.0 # via acme requests[security]==2.11.1 From bd9203fcbc5a88d11149c0c689a93b02a877d679 Mon Sep 17 00:00:00 2001 From: Steven Reiling Date: Fri, 13 Jul 2018 13:34:43 -0700 Subject: [PATCH 17/31] Adds an optional interval variable to notification service's create_default_expiration_notifications and introduces a new optional configuration variable, LEMUR_SECURITY_TEAM_EMAIL_INTERVALS, to allow admins control over the centralized email notification defaults. --- docs/administration.rst | 10 +++++++++- lemur/certificates/schemas.py | 4 +++- lemur/notifications/service.py | 10 ++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index aab7cd58..eec01cc5 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -274,7 +274,6 @@ Lemur supports sending certification expiration notifications through SES and SM LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com'] - .. data:: LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS :noindex: @@ -284,6 +283,15 @@ Lemur supports sending certification expiration notifications through SES and SM LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2] +.. data:: LEMUR_SECURITY_TEAM_EMAIL_INTERVALS + :noindex: + + Alternate notification interval set for security team notifications. Use this if you would like the default security team notification interval for new certificates to differ from the global default as specified in LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS. If unspecified, the value of LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS is used. Security team default notifications for new certificates can effectively be disabled by setting this value to an empty array. + + :: + + LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2] + Authentication Options ---------------------- diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index 72b42fb9..e88b6e73 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -48,9 +48,11 @@ class CertificateCreationSchema(CertificateSchema): "DEFAULT_{0}".format(data['owner'].split('@')[0].upper()), [data['owner']], ) + data['notifications'] += notification_service.create_default_expiration_notifications( 'DEFAULT_SECURITY', - current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL') + current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL'), + current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL_INTERVALS', None) ) return data diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 466c680b..957757bd 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -16,10 +16,11 @@ from lemur.common.utils import truthiness from lemur.notifications.models import Notification -def create_default_expiration_notifications(name, recipients): +def create_default_expiration_notifications(name, recipients, intervals=None): """ - Will create standard 30, 10 and 2 day notifications for a given owner. If standard notifications - already exist these will be returned instead of new notifications. + Will create standard 30, 10 and 2 day notifications for a given owner unless an alternate set of + intervals is supplied. If standard notifications already exist these will be returned instead of + new notifications. :param name: :param recipients: @@ -48,7 +49,8 @@ def create_default_expiration_notifications(name, recipients): }, ] - intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", [30, 15, 2]) + if intervals is None: + intervals = current_app.config.get("LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS", [30, 15, 2]) notifications = [] for i in intervals: From 0bb7a6e125c1bcdf191b895ead7d325b193b1ef7 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Tue, 17 Jul 2018 18:38:15 -0700 Subject: [PATCH 18/31] Unpinning requests --- requirements-docs.txt | 14 +++++++------- requirements-tests.txt | 4 ++-- requirements.in | 4 ++-- requirements.txt | 20 +++++++++++--------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index 9cfa1b7b..bc930c58 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.25.1 +acme==0.26.0 alabaster==0.7.11 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==0.9.10 @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.48 -botocore==1.10.48 +boto3==1.7.56 +botocore==1.10.56 certifi==2018.4.16 cffi==1.11.5 click==6.7 @@ -36,7 +36,7 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 -gunicorn==19.8.1 +gunicorn==19.9.0 idna==2.7 imagesize==1.0.0 # via sphinx inflection==0.3.1 @@ -54,7 +54,7 @@ mock==2.0.0 ndg-httpsclient==0.5.0 packaging==17.1 # via sphinx paramiko==2.4.1 -pbr==4.0.4 +pbr==4.1.0 pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 @@ -69,7 +69,7 @@ pyrfc3339==1.1 python-dateutil==2.7.3 python-editor==1.0.3 pytz==2018.5 -pyyaml==3.12 +pyyaml==3.13 raven[flask]==6.9.0 requests-toolbelt==0.8.0 requests[security]==2.11.1 @@ -78,7 +78,7 @@ s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx sphinx-rtd-theme==0.4.0 -sphinx==1.7.5 +sphinx==1.7.6 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 diff --git a/requirements-tests.txt b/requirements-tests.txt index 219c445a..55836e9a 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.56 # via moto +boto3==1.7.59 # via moto boto==2.49.0 # via moto -botocore==1.10.56 # via boto3, moto, s3transfer +botocore==1.10.59 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests diff --git a/requirements.in b/requirements.in index 2a5051a7..0e028261 100644 --- a/requirements.in +++ b/requirements.in @@ -31,10 +31,10 @@ paramiko # required for the SFTP destination plugin pem psycopg2 pyjwt -pyOpenSSL==17.2.0 # PINNED for a specific reason. This needs to be merged in before upgrade: https://github.com/shazow/urllib3/pull/1246 +pyOpenSSL python_ldap raven[flask] -requests==2.11.1 # PINNED for a specific reason. This needs to be merged in before upgrade: https://github.com/shazow/urllib3/pull/1246 +requests retrying six SQLAlchemy-Utils diff --git a/requirements.txt b/requirements.txt index d80d7e09..82817e7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,19 +4,20 @@ # # pip-compile --no-index --output-file requirements.txt requirements.in # -acme==0.26.0 +acme==0.26.1 alembic-autogenerate-enums==0.0.2 -alembic==0.9.10 # via flask-migrate +alembic==1.0.0 # via flask-migrate aniso8601==3.0.2 # via flask-restful arrow==0.12.1 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.56 -botocore==1.10.56 # via boto3, s3transfer +boto3==1.7.59 +botocore==1.10.59 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl +chardet==3.0.4 # via requests click==6.7 # via flask cloudflare==2.1.0 cryptography==2.2.2 @@ -35,7 +36,7 @@ flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 gunicorn==19.9.0 -idna==2.7 # via cryptography +idna==2.7 # via cryptography, requests inflection==0.3.1 itsdangerous==0.24 # via flask jinja2==2.10 @@ -54,11 +55,11 @@ pbr==4.1.0 # via mock pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap -pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap, requests +pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.18 # via cffi pyjwt==1.6.4 pynacl==1.2.1 # via paramiko -pyopenssl==17.2.0 +pyopenssl==18.0.0 pyrfc3339==1.1 # via acme python-dateutil==2.7.3 # via alembic, arrow, botocore python-editor==1.0.3 # via alembic @@ -67,12 +68,13 @@ pytz==2018.5 # via acme, flask-restful, pyrfc3339 pyyaml==3.13 # via cloudflare raven[flask]==6.9.0 requests-toolbelt==0.8.0 # via acme -requests[security]==2.11.1 +requests[security]==2.19.1 retrying==1.3.3 s3transfer==0.1.13 # via boto3 six==1.11.0 sqlalchemy-utils==0.33.3 -sqlalchemy==1.2.9 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +sqlalchemy==1.2.10 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils tabulate==0.8.2 +urllib3==1.23 # via requests werkzeug==0.14.1 # via flask xmltodict==0.11.0 From c78077d8d62a6da2de22e9c8a7e51b71cfd8c5fd Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 20 Jul 2018 10:47:19 -0700 Subject: [PATCH 19/31] Explicit capture exception during create failure --- lemur/certificates/service.py | 9 +++++++-- requirements-dev.txt | 2 +- requirements-docs.txt | 16 +++++++++------- requirements-tests.txt | 8 ++++---- requirements.txt | 8 ++++---- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index e7a5afd1..16383d61 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -15,7 +15,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization from lemur import database -from lemur.extensions import metrics, signals +from lemur.extensions import metrics, sentry, signals from lemur.plugins.base import plugins from lemur.common.utils import generate_private_key, truthiness @@ -247,7 +247,12 @@ def create(**kwargs): """ Creates a new certificate. """ - cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) + try: + cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) + except: + current_app.logger.error("Exception minting certificate", exc_info=True) + sentry.captureException() + raise kwargs['body'] = cert_body kwargs['private_key'] = private_key kwargs['chain'] = cert_chain diff --git a/requirements-dev.txt b/requirements-dev.txt index 58d6e277..3909600c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,7 +14,7 @@ identify==1.1.3 # via pre-commit idna==2.7 # via requests invoke==1.1.0 mccabe==0.6.1 # via flake8 -nodeenv==1.3.1 +nodeenv==1.3.2 pkginfo==1.4.2 # via twine pre-commit==1.10.3 pycodestyle==2.3.1 # via flake8 diff --git a/requirements-docs.txt b/requirements-docs.txt index bc930c58..2de29229 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,10 +4,10 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.26.0 +acme==0.26.1 alabaster==0.7.11 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==0.9.10 +alembic==1.0.0 aniso8601==3.0.2 arrow==0.12.1 asn1crypto==0.24.0 @@ -15,10 +15,11 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.56 -botocore==1.10.56 +boto3==1.7.59 +botocore==1.10.59 certifi==2018.4.16 cffi==1.11.5 +chardet==3.0.4 click==6.7 cloudflare==2.1.0 cryptography==2.2.2 @@ -63,7 +64,7 @@ pycparser==2.18 pygments==2.2.0 # via sphinx pyjwt==1.6.4 pynacl==1.2.1 -pyopenssl==17.2.0 +pyopenssl==18.0.0 pyparsing==2.2.0 # via packaging pyrfc3339==1.1 python-dateutil==2.7.3 @@ -72,7 +73,7 @@ pytz==2018.5 pyyaml==3.13 raven[flask]==6.9.0 requests-toolbelt==0.8.0 -requests[security]==2.11.1 +requests[security]==2.19.1 retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 @@ -82,7 +83,8 @@ sphinx==1.7.6 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx sqlalchemy-utils==0.33.3 -sqlalchemy==1.2.9 +sqlalchemy==1.2.10 tabulate==0.8.2 +urllib3==1.23 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 55836e9a..57292b12 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,16 +8,16 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.59 # via moto +boto3==1.7.61 # via moto boto==2.49.0 # via moto -botocore==1.10.59 # via boto3, moto, s3transfer +botocore==1.10.61 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests click==6.7 # via flask cookies==2.2.1 # via moto, responses coverage==4.5.1 -cryptography==2.2.2 # via moto +cryptography==2.3 # via moto docker-pycreds==0.3.0 # via docker docker==3.4.1 # via moto docutils==0.14 # via botocore @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.1.0 # via mock +pbr==4.1.1 # via mock pluggy==0.6.0 # via pytest py==1.5.4 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.txt b/requirements.txt index 82817e7e..3012ea6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,14 +13,14 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.59 -botocore==1.10.59 # via boto3, s3transfer +boto3==1.7.61 +botocore==1.10.61 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==6.7 # via flask cloudflare==2.1.0 -cryptography==2.2.2 +cryptography==2.3 dnspython3==1.15.0 dnspython==1.15.0 # via dnspython3 docutils==0.14 # via botocore @@ -51,7 +51,7 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.0 paramiko==2.4.1 -pbr==4.1.0 # via mock +pbr==4.1.1 # via mock pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap From 2f51fea743d04c1360726813a8639a9ff0a12e80 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 20 Jul 2018 10:53:47 -0700 Subject: [PATCH 20/31] no bare except --- lemur/certificates/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index 16383d61..9b250fc3 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -249,7 +249,7 @@ def create(**kwargs): """ try: cert_body, private_key, cert_chain, external_id, csr = mint(**kwargs) - except: + except Exception: current_app.logger.error("Exception minting certificate", exc_info=True) sentry.captureException() raise From 9b29f9f8190a0a35e064b46fa812ac6fea2bddd2 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 23 Jul 2018 10:57:22 -0700 Subject: [PATCH 21/31] Adding pessimistic sqlalchemy disconnection handling --- lemur/extensions.py | 10 +++++++++- requirements-dev.txt | 2 +- requirements-docs.txt | 8 ++++---- requirements-tests.txt | 6 +++--- requirements.txt | 6 +++--- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lemur/extensions.py b/lemur/extensions.py index 17a8e6e7..a54df6c7 100644 --- a/lemur/extensions.py +++ b/lemur/extensions.py @@ -3,7 +3,15 @@ :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more :license: Apache, see LICENSE for more details. """ -from flask_sqlalchemy import SQLAlchemy +from flask_sqlalchemy import SQLAlchemy as SA + + +class SQLAlchemy(SA): + def apply_pool_defaults(self, app, options): + SA.apply_pool_defaults(self, app, options) + options["pool_pre_ping"] = True + + db = SQLAlchemy() from flask_migrate import Migrate diff --git a/requirements-dev.txt b/requirements-dev.txt index 3909600c..0a7369d9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ invoke==1.1.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.2 pkginfo==1.4.2 # via twine -pre-commit==1.10.3 +pre-commit==1.10.4 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pyyaml==3.13 # via aspy.yaml, pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 2de29229..5d779a84 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,14 +15,14 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.59 -botocore==1.10.59 +boto3==1.7.61 +botocore==1.10.61 certifi==2018.4.16 cffi==1.11.5 chardet==3.0.4 click==6.7 cloudflare==2.1.0 -cryptography==2.2.2 +cryptography==2.3 dnspython3==1.15.0 dnspython==1.15.0 docutils==0.14 @@ -55,7 +55,7 @@ mock==2.0.0 ndg-httpsclient==0.5.0 packaging==17.1 # via sphinx paramiko==2.4.1 -pbr==4.1.0 +pbr==4.1.1 pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 diff --git a/requirements-tests.txt b/requirements-tests.txt index 57292b12..22361c6f 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.61 # via moto +boto3==1.7.62 # via moto boto==2.49.0 # via moto -botocore==1.10.61 # via boto3, moto, s3transfer +botocore==1.10.62 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -48,7 +48,7 @@ pytest==3.6.3 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.5 # via moto pyyaml==3.13 # via pyaml -requests-mock==1.5.0 +requests-mock==1.5.2 requests==2.19.1 # via aws-xray-sdk, docker, moto, requests-mock, responses responses==0.9.0 # via moto s3transfer==0.1.13 # via boto3 diff --git a/requirements.txt b/requirements.txt index 3012ea6d..6af5cd92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.61 -botocore==1.10.61 # via boto3, s3transfer +boto3==1.7.62 +botocore==1.10.62 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -49,7 +49,7 @@ markupsafe==1.0 # via jinja2, mako marshmallow-sqlalchemy==0.14.0 marshmallow==2.15.3 mock==2.0.0 # via acme -ndg-httpsclient==0.5.0 +ndg-httpsclient==0.5.1 paramiko==2.4.1 pbr==4.1.1 # via mock pem==18.1.0 From 2a6dda07ebb57814a994a9f6428edb2c5afff410 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 27 Jul 2018 14:15:14 -0700 Subject: [PATCH 22/31] Show and send error for pending certs --- lemur/notifications/messaging.py | 39 +++++ lemur/pending_certificates/cli.py | 29 ++++ lemur/plugins/lemur_acme/plugin.py | 3 +- .../plugins/lemur_email/templates/failed.html | 161 ++++++++++++++++++ .../pending_certificates/view/view.tpl.html | 6 + 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 lemur/plugins/lemur_email/templates/failed.html diff --git a/lemur/notifications/messaging.py b/lemur/notifications/messaging.py index 4600ac61..cd4ff0f1 100644 --- a/lemur/notifications/messaging.py +++ b/lemur/notifications/messaging.py @@ -24,6 +24,7 @@ from lemur.common.utils import windowed_query from lemur.certificates.schemas import certificate_notification_output_schema from lemur.certificates.models import Certificate +from lemur.pending_certificates.schemas import pending_certificate_output_schema from lemur.plugins import plugins from lemur.plugins.utils import get_plugin_option @@ -172,6 +173,44 @@ def send_rotation_notification(certificate, notification_plugin=None): return True +def send_pending_failure_notification(pending_cert, notify_owner=True, notify_security=True, notification_plugin=None): + """ + Sends a report to certificate owners when their pending certificate failed to be created. + + :param pending_cert: + :param notification_plugin: + :return: + """ + status = FAILURE_METRIC_STATUS + + if not notification_plugin: + notification_plugin = plugins.get( + current_app.config.get('LEMUR_DEFAULT_NOTIFICATION_PLUGIN', 'email-notification') + ) + + data = pending_certificate_output_schema.dump(pending_cert).data + data["security_email"] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL') + + if notify_owner: + try: + notification_plugin.send('failed', data, [data['owner']], pending_cert) + status = SUCCESS_METRIC_STATUS + except Exception as e: + sentry.captureException() + + if notify_security: + try: + notification_plugin.send('failed', data, data["security_email"], pending_cert) + status = SUCCESS_METRIC_STATUS + except Exception as e: + sentry.captureException() + + metrics.send('notification', 'counter', 1, metric_tags={'status': status, 'event_type': 'rotation'}) + + if status == SUCCESS_METRIC_STATUS: + return True + + def needs_notification(certificate): """ Determine if notifications for a given certificate should diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index 6e12c53b..fd7591b1 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -4,9 +4,15 @@ .. moduleauthor:: James Chuong .. moduleauthor:: Curtis Castrapel """ + +import copy +import sys + +from flask import current_app from flask_script import Manager from lemur.authorities.service import get as get_authority +from lemur.notifications.messaging import send_pending_failure_notification from lemur.pending_certificates import service as pending_certificate_service from lemur.plugins.base import plugins from lemur.users import service as user_service @@ -56,6 +62,10 @@ def fetch_all_acme(): for acme-issued certificates because it will configure all of the DNS challenges prior to resolving any certificates. """ + + log_data = { + "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name) + } pending_certs = pending_certificate_service.get_pending_certs('all') user = user_service.get_by_username('lemur') new = 0 @@ -88,7 +98,26 @@ def fetch_all_acme(): new += 1 else: pending_certificate_service.increment_attempt(pending_cert) + pending_certificate_service.update( + cert.get("pending_cert").id, + status=str(cert.get("last_error"))[0:128] + ) failed += 1 + if pending_cert.number_attempts > 0: + error_log = copy.deepcopy(log_data) + error_log["message"] = "Deleting pending certificate" + error_log["pending_cert_id"] = pending_cert.id + error_log["last_error"] = cert.get("last_error") + error_log["cn"] = pending_cert.cn + current_app.logger.error(error_log) + if 1 == 0: + send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) + pending_certificate_service.delete_by_id(pending_cert.id) + log_data["message"] = "Complete" + log_data["new"] = new + log_data["failed"] = failed + log_data["wrong_issuer"] = wrong_issuer + current_app.logger.debug(log_data) print( "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format( new=new, diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f472a965..0d3e9c2a 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -300,11 +300,12 @@ class ACMEIssuerPlugin(IssuerPlugin): "order": order, "dns_provider_options": dns_provider_options, }) - except (ClientError, ValueError, Exception): + except (ClientError, ValueError, Exception) as e: current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) certs.append({ "cert": False, "pending_cert": pending_cert, + "last_error": e, }) for entry in pending: diff --git a/lemur/plugins/lemur_email/templates/failed.html b/lemur/plugins/lemur_email/templates/failed.html new file mode 100644 index 00000000..63e37fb5 --- /dev/null +++ b/lemur/plugins/lemur_email/templates/failed.html @@ -0,0 +1,161 @@ + + + + + + + + Lemur + + +
+ + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + +
+ Lemur +
+
+ + + + + + + + + + + + + + +
+ Your certificate request has failed! +
+
+ + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ Hi, +
This is a Lemur certificate failure notice. We were unable to create or rotate your certificate. Please retry your request. The reason for the failure is listed below. +
+ + + + + + + + +
+ {{ message.certificates.name }} +
+ +
{{ message.certificates.owner }} +
{{ message.certificates.status }} + +
+
+ If you are having any trouble, please reach out to {{ ", ".join(message.certificates.security_email) }}. +
+
Best,
Lemur +
+ + + + + + +
*All times are in UTC
+
+
+
+ + + + + + + + + +
You received this mandatory email announcement to update you about + important changes to your TLS certificate. +
+
© 2016 Lemur
+
+
+
+
diff --git a/lemur/static/app/angular/pending_certificates/view/view.tpl.html b/lemur/static/app/angular/pending_certificates/view/view.tpl.html index 8aaf9f47..d480cc2d 100644 --- a/lemur/static/app/angular/pending_certificates/view/view.tpl.html +++ b/lemur/static/app/angular/pending_certificates/view/view.tpl.html @@ -80,6 +80,12 @@ {{ pendingCertificate.numberAttempts }} +
  • + Latest Status + + {{ pendingCertificate.status }} + +
  • Date Created From e16c1de00171c112d131da5c92663692bf5758a0 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 27 Jul 2018 14:17:50 -0700 Subject: [PATCH 23/31] Error logging --- lemur/pending_certificates/cli.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index fd7591b1..647c5407 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -103,16 +103,17 @@ def fetch_all_acme(): status=str(cert.get("last_error"))[0:128] ) failed += 1 - if pending_cert.number_attempts > 0: - error_log = copy.deepcopy(log_data) + error_log = copy.deepcopy(log_data) + error_log["message"] = "Pending certificate creation failure" + error_log["pending_cert_id"] = pending_cert.id + error_log["last_error"] = cert.get("last_error") + error_log["cn"] = pending_cert.cn + + if pending_cert.number_attempts > 4: error_log["message"] = "Deleting pending certificate" - error_log["pending_cert_id"] = pending_cert.id - error_log["last_error"] = cert.get("last_error") - error_log["cn"] = pending_cert.cn - current_app.logger.error(error_log) - if 1 == 0: - send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) - pending_certificate_service.delete_by_id(pending_cert.id) + send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) + pending_certificate_service.delete_by_id(pending_cert.id) + current_app.logger.error(error_log) log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed From 2bb00bc666c2c237b018b8bfe47cc7bc4a3b0982 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 27 Jul 2018 14:20:22 -0700 Subject: [PATCH 24/31] requirements --- requirements-dev.txt | 4 ++-- requirements-docs.txt | 8 ++++---- requirements-tests.txt | 6 +++--- requirements.txt | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0a7369d9..2abd7a43 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ certifi==2018.4.16 # via requests cfgv==1.1.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.1.3 # via pre-commit +identify==1.1.4 # via pre-commit idna==2.7 # via requests invoke==1.1.0 mccabe==0.6.1 # via flake8 @@ -24,7 +24,7 @@ requests-toolbelt==0.8.0 # via twine requests==2.19.1 # via requests-toolbelt, twine six==1.11.0 # via cfgv, pre-commit toml==0.9.4 # via pre-commit -tqdm==4.23.4 # via twine +tqdm==4.24.0 # via twine twine==1.11.0 urllib3==1.23 # via requests virtualenv==16.0.0 # via pre-commit diff --git a/requirements-docs.txt b/requirements-docs.txt index 5d779a84..2a688d6f 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.61 -botocore==1.10.61 +boto3==1.7.62 +botocore==1.10.62 certifi==2018.4.16 cffi==1.11.5 chardet==3.0.4 @@ -52,7 +52,7 @@ markupsafe==1.0 marshmallow-sqlalchemy==0.14.0 marshmallow==2.15.3 mock==2.0.0 -ndg-httpsclient==0.5.0 +ndg-httpsclient==0.5.1 packaging==17.1 # via sphinx paramiko==2.4.1 pbr==4.1.1 @@ -78,7 +78,7 @@ retrying==1.3.3 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 # via sphinx -sphinx-rtd-theme==0.4.0 +sphinx-rtd-theme==0.4.1 sphinx==1.7.6 sphinxcontrib-httpdomain==1.7.0 sphinxcontrib-websupport==1.1.0 # via sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 22361c6f..c3405365 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.62 # via moto +boto3==1.7.65 # via moto boto==2.49.0 # via moto -botocore==1.10.62 # via boto3, moto, s3transfer +botocore==1.10.65 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -36,7 +36,7 @@ mock==2.0.0 # via moto more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 -pbr==4.1.1 # via mock +pbr==4.2.0 # via mock pluggy==0.6.0 # via pytest py==1.5.4 # via pytest pyaml==17.12.1 # via moto diff --git a/requirements.txt b/requirements.txt index 6af5cd92..0474e651 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.62 -botocore==1.10.62 # via boto3, s3transfer +boto3==1.7.65 +botocore==1.10.65 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests @@ -51,11 +51,11 @@ marshmallow==2.15.3 mock==2.0.0 # via acme ndg-httpsclient==0.5.1 paramiko==2.4.1 -pbr==4.1.1 # via mock +pbr==4.2.0 # via mock pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 # via python-ldap -pyasn1==0.4.3 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap +pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap pycparser==2.18 # via cffi pyjwt==1.6.4 pynacl==1.2.1 # via paramiko From caf99d36d62b8f97a162ff874a08c3cc2bb68a74 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 27 Jul 2018 15:52:22 -0700 Subject: [PATCH 25/31] fix deletion --- lemur/pending_certificates/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index 647c5407..6dc01679 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -97,11 +97,6 @@ def fetch_all_acme(): # add metrics to metrics extension new += 1 else: - pending_certificate_service.increment_attempt(pending_cert) - pending_certificate_service.update( - cert.get("pending_cert").id, - status=str(cert.get("last_error"))[0:128] - ) failed += 1 error_log = copy.deepcopy(log_data) error_log["message"] = "Pending certificate creation failure" @@ -114,6 +109,11 @@ def fetch_all_acme(): send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) pending_certificate_service.delete_by_id(pending_cert.id) current_app.logger.error(error_log) + pending_certificate_service.increment_attempt(pending_cert) + pending_certificate_service.update( + cert.get("pending_cert").id, + status=str(cert.get("last_error"))[0:128] + ) log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed From d6b482755b4912a247444ddbda5a66a5eff7e5b8 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 30 Jul 2018 13:49:41 -0700 Subject: [PATCH 26/31] Proper flask_restful boolean parsing. This is documented here: https://github.com/flask-restful/flask-restful/issues/488 --- lemur/certificates/views.py | 10 +++++----- lemur/notifications/views.py | 4 ++-- lemur/pending_certificates/views.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index cdfd06a3..72b38bab 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -9,7 +9,7 @@ import base64 from builtins import str from flask import Blueprint, make_response, jsonify, g -from flask_restful import reqparse, Api +from flask_restful import reqparse, Api, inputs from lemur.common.schema import validate_schema from lemur.common.utils import paginated_parser @@ -132,9 +132,9 @@ class CertificatesList(AuthenticatedResource): """ parser = paginated_parser.copy() parser.add_argument('timeRange', type=int, dest='time_range', location='args') - parser.add_argument('owner', type=bool, location='args') + parser.add_argument('owner', type=inputs.boolean, location='args') parser.add_argument('id', type=str, location='args') - parser.add_argument('active', type=bool, location='args') + parser.add_argument('active', type=inputs.boolean, location='args') parser.add_argument('destinationId', type=int, dest="destination_id", location='args') parser.add_argument('creator', type=str, location='args') parser.add_argument('show', type=str, location='args') @@ -756,9 +756,9 @@ class NotificationCertificatesList(AuthenticatedResource): """ parser = paginated_parser.copy() parser.add_argument('timeRange', type=int, dest='time_range', location='args') - parser.add_argument('owner', type=bool, location='args') + parser.add_argument('owner', type=inputs.boolean, location='args') parser.add_argument('id', type=str, location='args') - parser.add_argument('active', type=bool, location='args') + parser.add_argument('active', type=inputs.boolean, location='args') parser.add_argument('destinationId', type=int, dest="destination_id", location='args') parser.add_argument('creator', type=str, location='args') parser.add_argument('show', type=str, location='args') diff --git a/lemur/notifications/views.py b/lemur/notifications/views.py index df75aaa9..4a2d82a8 100644 --- a/lemur/notifications/views.py +++ b/lemur/notifications/views.py @@ -7,7 +7,7 @@ .. moduleauthor:: Kevin Glisson """ from flask import Blueprint -from flask_restful import Api, reqparse +from flask_restful import Api, reqparse, inputs from lemur.notifications import service from lemur.notifications.schemas import notification_input_schema, notification_output_schema, notifications_output_schema @@ -103,7 +103,7 @@ class NotificationsList(AuthenticatedResource): :statuscode 200: no error """ parser = paginated_parser.copy() - parser.add_argument('active', type=bool, location='args') + parser.add_argument('active', type=inputs.boolean, location='args') args = parser.parse_args() return service.render(args) diff --git a/lemur/pending_certificates/views.py b/lemur/pending_certificates/views.py index 81b0671e..13598040 100644 --- a/lemur/pending_certificates/views.py +++ b/lemur/pending_certificates/views.py @@ -5,7 +5,7 @@ .. moduleauthor:: James Chuong """ from flask import Blueprint, g, make_response, jsonify -from flask_restful import Api, reqparse +from flask_restful import Api, reqparse, inputs from lemur.auth.service import AuthenticatedResource from lemur.auth.permissions import CertificatePermission @@ -110,9 +110,9 @@ class PendingCertificatesList(AuthenticatedResource): """ parser = paginated_parser.copy() parser.add_argument('timeRange', type=int, dest='time_range', location='args') - parser.add_argument('owner', type=bool, location='args') + parser.add_argument('owner', type=inputs.boolean, location='args') parser.add_argument('id', type=str, location='args') - parser.add_argument('active', type=bool, location='args') + parser.add_argument('active', type=inputs.boolean, location='args') parser.add_argument('destinationId', type=int, dest="destination_id", location='args') parser.add_argument('creator', type=str, location='args') parser.add_argument('show', type=str, location='args') From 0889076d3ba1ccbb2cbd05888aab0a189ea8bd5d Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 30 Jul 2018 15:25:02 -0700 Subject: [PATCH 27/31] Support LetsEncrypt accounts --- lemur/pending_certificates/cli.py | 11 ++-- lemur/plugins/lemur_acme/plugin.py | 65 ++++++++++++++++++--- lemur/plugins/lemur_acme/tests/test_acme.py | 1 + requirements-docs.txt | 8 +-- requirements-tests.txt | 8 +-- requirements.txt | 4 +- 6 files changed, 75 insertions(+), 22 deletions(-) diff --git a/lemur/pending_certificates/cli.py b/lemur/pending_certificates/cli.py index 6dc01679..0deeaf68 100644 --- a/lemur/pending_certificates/cli.py +++ b/lemur/pending_certificates/cli.py @@ -108,12 +108,13 @@ def fetch_all_acme(): error_log["message"] = "Deleting pending certificate" send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify) pending_certificate_service.delete_by_id(pending_cert.id) + else: + pending_certificate_service.increment_attempt(pending_cert) + pending_certificate_service.update( + cert.get("pending_cert").id, + status=str(cert.get("last_error"))[0:128] + ) current_app.logger.error(error_log) - pending_certificate_service.increment_attempt(pending_cert) - pending_certificate_service.update( - cert.get("pending_cert").id, - status=str(cert.get("last_error"))[0:128] - ) log_data["message"] = "Complete" log_data["new"] = new log_data["failed"] = failed diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 0d3e9c2a..1cfac059 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -140,14 +140,27 @@ def setup_acme_client(authority): tel = options.get('telephone', current_app.config.get('ACME_TEL')) directory_url = options.get('acme_url', current_app.config.get('ACME_DIRECTORY_URL')) - key = jose.JWKRSA(key=generate_private_key('RSA2048')) + existing_key = options.get('acme_private_key', current_app.config.get('ACME_PRIVATE_KEY')) + existing_regr = options.get('acme_regr', current_app.config.get('ACME_REGR')) + print(existing_key) + if existing_key and existing_regr: + # Reuse the same account for each certificate issuance + key = jose.JWK.json_loads(existing_key) + regr = messages.RegistrationResource.json_loads(existing_regr) + current_app.logger.debug("Connecting with directory at {0}".format(directory_url)) + net = ClientNetwork(key, account=regr) + client = BackwardsCompatibleClientV2(net, key, directory_url) + return client, {} + else: + # Create an account for each certificate issuance + key = jose.JWKRSA(key=generate_private_key('RSA2048')) - current_app.logger.debug("Connecting with directory at {0}".format(directory_url)) + current_app.logger.debug("Connecting with directory at {0}".format(directory_url)) - net = ClientNetwork(key, account=None) - client = BackwardsCompatibleClientV2(net, key, directory_url) - registration = client.new_account_and_tos(messages.NewRegistration.from_data(email=email)) - current_app.logger.debug("Connected: {0}".format(registration.uri)) + net = ClientNetwork(key, account=None) + client = BackwardsCompatibleClientV2(net, key, directory_url) + registration = client.new_account_and_tos(messages.NewRegistration.from_data(email=email)) + current_app.logger.debug("Connected: {0}".format(registration.uri)) return client, registration @@ -196,6 +209,35 @@ def finalize_authorizations(acme_client, account_number, dns_provider, authoriza return authorizations +def cleanup_dns_challenges(acme_client, account_number, dns_provider, authorizations, dns_provider_options): + """ + Best effort attempt to delete DNS challenges that may not have been deleted previously. This is usually called + on an exception + + :param acme_client: + :param account_number: + :param dns_provider: + :param authorizations: + :param dns_provider_options: + :return: + """ + for authz_record in authorizations: + dns_challenges = authz_record.dns_challenge + host_to_validate = maybe_remove_wildcard(authz_record.host) + host_to_validate = maybe_add_extension(host_to_validate, dns_provider_options) + for dns_challenge in dns_challenges: + try: + dns_provider.delete_txt_record( + authz_record.change_id, + account_number, + dns_challenge.validation_domain_name(host_to_validate), + dns_challenge.validation(acme_client.client.net.key) + ) + except Exception: + # If this fails, it's most likely because the record doesn't exist or we're not authorized to modify it. + pass + + class ACMEIssuerPlugin(IssuerPlugin): title = 'Acme' slug = 'acme-issuer' @@ -333,12 +375,21 @@ class ACMEIssuerPlugin(IssuerPlugin): "cert": cert, "pending_cert": entry["pending_cert"], }) - except (PollError, AcmeError, Exception): + except (PollError, AcmeError, Exception) as e: current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True) certs.append({ "cert": False, "pending_cert": entry["pending_cert"], + "last_error": e, }) + # Ensure DNS records get deleted + cleanup_dns_challenges( + entry["acme_client"], + entry["account_number"], + entry["dns_provider_type"], + entry["authorizations"], + entry["dns_provider_options"], + ) return certs def create_certificate(self, csr, issuer_options): diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index 2a358a63..69f4e438 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -134,6 +134,7 @@ class TestAcme(unittest.TestCase): mock_client.register = mock_registration mock_client.agree_to_tos = Mock(return_value=True) mock_acme.return_value = mock_client + mock_current_app.config = {} result_client, result_registration = plugin.setup_acme_client(mock_authority) assert result_client assert result_registration diff --git a/requirements-docs.txt b/requirements-docs.txt index 2a688d6f..3244c58e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -15,8 +15,8 @@ asyncpool==1.0 babel==2.6.0 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.62 -botocore==1.10.62 +boto3==1.7.65 +botocore==1.10.65 certifi==2018.4.16 cffi==1.11.5 chardet==3.0.4 @@ -55,11 +55,11 @@ mock==2.0.0 ndg-httpsclient==0.5.1 packaging==17.1 # via sphinx paramiko==2.4.1 -pbr==4.1.1 +pbr==4.2.0 pem==18.1.0 psycopg2==2.7.5 pyasn1-modules==0.2.2 -pyasn1==0.4.3 +pyasn1==0.4.4 pycparser==2.18 pygments==2.2.0 # via sphinx pyjwt==1.6.4 diff --git a/requirements-tests.txt b/requirements-tests.txt index c3405365..a024eb18 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.1.5 # via pytest attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.65 # via moto +boto3==1.7.66 # via moto boto==2.49.0 # via moto -botocore==1.10.65 # via boto3, moto, s3transfer +botocore==1.10.66 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -37,14 +37,14 @@ more-itertools==4.2.0 # via pytest moto==1.3.3 nose==1.3.7 pbr==4.2.0 # via mock -pluggy==0.6.0 # via pytest +pluggy==0.7.1 # via pytest py==1.5.4 # via pytest pyaml==17.12.1 # via moto pycparser==2.18 # via cffi pyflakes==2.0.0 pytest-flask==0.10.0 pytest-mock==1.10.0 -pytest==3.6.3 +pytest==3.6.4 python-dateutil==2.6.1 # via botocore, faker, freezegun, moto pytz==2018.5 # via moto pyyaml==3.13 # via pyaml diff --git a/requirements.txt b/requirements.txt index 0474e651..a88d2309 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.65 -botocore==1.10.65 # via boto3, s3transfer +boto3==1.7.66 +botocore==1.10.66 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests From 44192d44941022bf25a6bfcd713d8c507abfe44c Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 30 Jul 2018 15:27:23 -0700 Subject: [PATCH 28/31] remove debug print --- lemur/plugins/lemur_acme/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index 1cfac059..6d07c063 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -142,7 +142,7 @@ def setup_acme_client(authority): existing_key = options.get('acme_private_key', current_app.config.get('ACME_PRIVATE_KEY')) existing_regr = options.get('acme_regr', current_app.config.get('ACME_REGR')) - print(existing_key) + if existing_key and existing_regr: # Reuse the same account for each certificate issuance key = jose.JWK.json_loads(existing_key) From bb0c229d7ef8cca02e4426c6947ac29a1d218df8 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Fri, 3 Aug 2018 18:12:05 +0300 Subject: [PATCH 29/31] Add .pytest_cache dir to .gitignore (#1512) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f6268f3d..100161ae 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ package-lock.json docs/_build .editorconfig .idea +.pytest_cache lemur/tests/tmp /lemur/plugins/lemur_email/tests/expiration-rendered.html From acd2701fa26d910874bb353a7e327596de9ead23 Mon Sep 17 00:00:00 2001 From: Marti Raudsepp Date: Fri, 3 Aug 2018 18:21:55 +0300 Subject: [PATCH 30/31] Delete dead code in unit tests (#1510) --- lemur/tests/plugins/issuer_plugin.py | 1 - lemur/tests/test_authorities.py | 1 - lemur/tests/test_certificates.py | 19 ------------------- 3 files changed, 21 deletions(-) diff --git a/lemur/tests/plugins/issuer_plugin.py b/lemur/tests/plugins/issuer_plugin.py index acc7dcab..7dbc46de 100644 --- a/lemur/tests/plugins/issuer_plugin.py +++ b/lemur/tests/plugins/issuer_plugin.py @@ -38,7 +38,6 @@ class TestAsyncIssuerPlugin(IssuerPlugin): return "", "", 12345 def get_ordered_certificate(self, pending_cert): - order_id = pending_cert.external_id return INTERNAL_VALID_LONG_STR, INTERNAL_VALID_SAN_STR, 54321 @staticmethod diff --git a/lemur/tests/test_authorities.py b/lemur/tests/test_authorities.py index 78399e47..e865ab41 100644 --- a/lemur/tests/test_authorities.py +++ b/lemur/tests/test_authorities.py @@ -28,7 +28,6 @@ def test_authority_input_schema(client, role, issuer_plugin, logged_in_user): def test_user_authority(session, client, authority, role, user, issuer_plugin): - resp = client.get(api.url_for(AuthoritiesList), headers=user['token']).json u = user['user'] u.roles.append(role) authority.roles.append(role) diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 3368f88d..fff6e253 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -44,25 +44,6 @@ def test_get_certificate_primitives(certificate): names = [x509.DNSName(x.name) for x in certificate.domains] - data = { - 'common_name': certificate.cn, - 'owner': certificate.owner, - 'authority': certificate.authority, - 'description': certificate.description, - 'extensions': { - 'sub_alt_names': x509.SubjectAlternativeName(names) - }, - 'destinations': [], - 'roles': [], - 'validity_end': arrow.get(2021, 5, 7), - 'validity_start': arrow.get(2016, 10, 30), - 'country': 'US', - 'location': 'A place', - 'organization': 'Example', - 'organizational_unit': 'Operations', - 'state': 'CA' - } - with freeze_time(datetime.date(year=2016, month=10, day=30)): primitives = get_certificate_primitives(certificate) assert len(primitives) == 24 From 2869042f38d40edfe5d99182995b10a4b5c1cce8 Mon Sep 17 00:00:00 2001 From: Cyril Dangerville Date: Sat, 4 Aug 2018 00:26:48 +0200 Subject: [PATCH 31/31] Fixed invalid JSON payloads (making API requests fail in particular) (#1522) --- lemur/authorities/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lemur/authorities/views.py b/lemur/authorities/views.py index 6bf7e2a8..b85c9b70 100644 --- a/lemur/authorities/views.py +++ b/lemur/authorities/views.py @@ -90,7 +90,7 @@ class AuthoritiesList(AuthenticatedResource): "owner": "secure@example.com", "id": 43, "description": "This is the ROOT certificate for the TestAuthority certificate authority." - } + }], "total": 1 } @@ -136,7 +136,7 @@ class AuthoritiesList(AuthenticatedResource): "sensitivity": "medium", "keyType": "RSA2048", "plugin": { - "slug": "cloudca-issuer", + "slug": "cloudca-issuer" }, "name": "TimeTestAuthority5", "owner": "secure@example.com", @@ -148,6 +148,7 @@ class AuthoritiesList(AuthenticatedResource): "names": [] }, "custom": [] + } } **Example response**: @@ -499,6 +500,7 @@ class AuthorityVisualizations(AuthenticatedResource): {"name": "MergeEdge", "size": 743} ] } + ] } ]} """